diff --git a/hw/intc/s390_flic_kvm.c b/hw/intc/s390_flic_kvm.c
index 46c9e61..b471e7a 100644
--- a/hw/intc/s390_flic_kvm.c
+++ b/hw/intc/s390_flic_kvm.c
@@ -117,6 +117,16 @@
     return rc ? -errno : 0;
 }
 
+int kvm_s390_inject_flic(struct kvm_s390_irq *irq)
+{
+    static KVMS390FLICState *flic;
+
+    if (unlikely(!flic)) {
+        flic = KVM_S390_FLIC(s390_get_flic());
+    }
+    return flic_enqueue_irqs(irq, sizeof(*irq), flic);
+}
+
 /**
  * __get_all_irqs - store all pending irqs in buffer
  * @flic: pointer to flic device state
@@ -170,7 +180,8 @@
     };
 
     if (!kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING)) {
-        return -ENOSYS;
+        /* nothing to do */
+        return 0;
     }
 
     r = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
@@ -195,7 +206,8 @@
     int r;
 
     if (!kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING)) {
-        return -ENOSYS;
+        /* nothing to do */
+        return 0;
     }
 
     r = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
diff --git a/hw/s390x/css.c b/hw/s390x/css.c
index 2678e44..e758890 100644
--- a/hw/s390x/css.c
+++ b/hw/s390x/css.c
@@ -128,13 +128,11 @@
 
 static void css_inject_io_interrupt(SubchDev *sch)
 {
-    S390CPU *cpu = s390_cpu_addr2state(0);
     uint8_t isc = (sch->curr_status.pmcw.flags & PMCW_FLAGS_MASK_ISC) >> 11;
 
     trace_css_io_interrupt(sch->cssid, sch->ssid, sch->schid,
                            sch->curr_status.pmcw.intparm, isc, "");
-    s390_io_interrupt(cpu,
-                      css_build_subchannel_id(sch),
+    s390_io_interrupt(css_build_subchannel_id(sch),
                       sch->schid,
                       sch->curr_status.pmcw.intparm,
                       isc << 27);
@@ -147,7 +145,6 @@
      * with alert status.
      */
     if (!(sch->curr_status.scsw.ctrl & SCSW_STCTL_STATUS_PEND)) {
-        S390CPU *cpu = s390_cpu_addr2state(0);
         uint8_t isc = (sch->curr_status.pmcw.flags & PMCW_FLAGS_MASK_ISC) >> 11;
 
         trace_css_io_interrupt(sch->cssid, sch->ssid, sch->schid,
@@ -157,8 +154,7 @@
         sch->curr_status.scsw.ctrl |=
             SCSW_STCTL_ALERT | SCSW_STCTL_STATUS_PEND;
         /* Inject an I/O interrupt. */
-        s390_io_interrupt(cpu,
-                          css_build_subchannel_id(sch),
+        s390_io_interrupt(css_build_subchannel_id(sch),
                           sch->schid,
                           sch->curr_status.pmcw.intparm,
                           isc << 27);
@@ -167,11 +163,10 @@
 
 void css_adapter_interrupt(uint8_t isc)
 {
-    S390CPU *cpu = s390_cpu_addr2state(0);
     uint32_t io_int_word = (isc << 27) | IO_INT_WORD_AI;
 
     trace_css_adapter_interrupt(isc);
-    s390_io_interrupt(cpu, 0, 0, 0, io_int_word);
+    s390_io_interrupt(0, 0, 0, io_int_word);
 }
 
 static void sch_handle_clear_func(SubchDev *sch)
@@ -779,9 +774,11 @@
     return ret;
 }
 
-static void copy_irb_to_guest(IRB *dest, const IRB *src)
+static void copy_irb_to_guest(IRB *dest, const IRB *src, PMCW *pmcw)
 {
     int i;
+    uint16_t stctl = src->scsw.ctrl & SCSW_CTRL_MASK_STCTL;
+    uint16_t actl = src->scsw.ctrl & SCSW_CTRL_MASK_ACTL;
 
     copy_scsw_to_guest(&dest->scsw, &src->scsw);
 
@@ -791,8 +788,22 @@
     for (i = 0; i < ARRAY_SIZE(dest->ecw); i++) {
         dest->ecw[i] = cpu_to_be32(src->ecw[i]);
     }
-    for (i = 0; i < ARRAY_SIZE(dest->emw); i++) {
-        dest->emw[i] = cpu_to_be32(src->emw[i]);
+    /* extended measurements enabled? */
+    if ((src->scsw.flags & SCSW_FLAGS_MASK_ESWF) ||
+        !(pmcw->flags & PMCW_FLAGS_MASK_TF) ||
+        !(pmcw->chars & PMCW_CHARS_MASK_XMWME)) {
+        return;
+    }
+    /* extended measurements pending? */
+    if (!(stctl & SCSW_STCTL_STATUS_PEND)) {
+        return;
+    }
+    if ((stctl & SCSW_STCTL_PRIMARY) ||
+        (stctl == SCSW_STCTL_SECONDARY) ||
+        ((stctl & SCSW_STCTL_INTERMEDIATE) && (actl & SCSW_ACTL_SUSP))) {
+        for (i = 0; i < ARRAY_SIZE(dest->emw); i++) {
+            dest->emw[i] = cpu_to_be32(src->emw[i]);
+        }
     }
 }
 
@@ -838,7 +849,7 @@
         }
     }
     /* Store the irb to the guest. */
-    copy_irb_to_guest(target_irb, &irb);
+    copy_irb_to_guest(target_irb, &irb, p);
 
     /* Clear conditions on subchannel, if applicable. */
     if (stctl & SCSW_STCTL_STATUS_PEND) {
@@ -1215,11 +1226,9 @@
     QTAILQ_INSERT_TAIL(&channel_subsys->pending_crws, crw_cont, sibling);
 
     if (channel_subsys->do_crw_mchk) {
-        S390CPU *cpu = s390_cpu_addr2state(0);
-
         channel_subsys->do_crw_mchk = false;
         /* Inject crw pending machine check. */
-        s390_crw_mchk(cpu);
+        s390_crw_mchk();
     }
 }
 
@@ -1277,6 +1286,117 @@
     return 0;
 }
 
+void subch_device_save(SubchDev *s, QEMUFile *f)
+{
+    int i;
+
+    qemu_put_byte(f, s->cssid);
+    qemu_put_byte(f, s->ssid);
+    qemu_put_be16(f, s->schid);
+    qemu_put_be16(f, s->devno);
+    qemu_put_byte(f, s->thinint_active);
+    /* SCHIB */
+    /*     PMCW */
+    qemu_put_be32(f, s->curr_status.pmcw.intparm);
+    qemu_put_be16(f, s->curr_status.pmcw.flags);
+    qemu_put_be16(f, s->curr_status.pmcw.devno);
+    qemu_put_byte(f, s->curr_status.pmcw.lpm);
+    qemu_put_byte(f, s->curr_status.pmcw.pnom);
+    qemu_put_byte(f, s->curr_status.pmcw.lpum);
+    qemu_put_byte(f, s->curr_status.pmcw.pim);
+    qemu_put_be16(f, s->curr_status.pmcw.mbi);
+    qemu_put_byte(f, s->curr_status.pmcw.pom);
+    qemu_put_byte(f, s->curr_status.pmcw.pam);
+    qemu_put_buffer(f, s->curr_status.pmcw.chpid, 8);
+    qemu_put_be32(f, s->curr_status.pmcw.chars);
+    /*     SCSW */
+    qemu_put_be16(f, s->curr_status.scsw.flags);
+    qemu_put_be16(f, s->curr_status.scsw.ctrl);
+    qemu_put_be32(f, s->curr_status.scsw.cpa);
+    qemu_put_byte(f, s->curr_status.scsw.dstat);
+    qemu_put_byte(f, s->curr_status.scsw.cstat);
+    qemu_put_be16(f, s->curr_status.scsw.count);
+    qemu_put_be64(f, s->curr_status.mba);
+    qemu_put_buffer(f, s->curr_status.mda, 4);
+    /* end SCHIB */
+    qemu_put_buffer(f, s->sense_data, 32);
+    qemu_put_be64(f, s->channel_prog);
+    /* last cmd */
+    qemu_put_byte(f, s->last_cmd.cmd_code);
+    qemu_put_byte(f, s->last_cmd.flags);
+    qemu_put_be16(f, s->last_cmd.count);
+    qemu_put_be32(f, s->last_cmd.cda);
+    qemu_put_byte(f, s->last_cmd_valid);
+    qemu_put_byte(f, s->id.reserved);
+    qemu_put_be16(f, s->id.cu_type);
+    qemu_put_byte(f, s->id.cu_model);
+    qemu_put_be16(f, s->id.dev_type);
+    qemu_put_byte(f, s->id.dev_model);
+    qemu_put_byte(f, s->id.unused);
+    for (i = 0; i < ARRAY_SIZE(s->id.ciw); i++) {
+        qemu_put_byte(f, s->id.ciw[i].type);
+        qemu_put_byte(f, s->id.ciw[i].command);
+        qemu_put_be16(f, s->id.ciw[i].count);
+    }
+    return;
+}
+
+int subch_device_load(SubchDev *s, QEMUFile *f)
+{
+    int i;
+
+    s->cssid = qemu_get_byte(f);
+    s->ssid = qemu_get_byte(f);
+    s->schid = qemu_get_be16(f);
+    s->devno = qemu_get_be16(f);
+    s->thinint_active = qemu_get_byte(f);
+    /* SCHIB */
+    /*     PMCW */
+    s->curr_status.pmcw.intparm = qemu_get_be32(f);
+    s->curr_status.pmcw.flags = qemu_get_be16(f);
+    s->curr_status.pmcw.devno = qemu_get_be16(f);
+    s->curr_status.pmcw.lpm = qemu_get_byte(f);
+    s->curr_status.pmcw.pnom  = qemu_get_byte(f);
+    s->curr_status.pmcw.lpum = qemu_get_byte(f);
+    s->curr_status.pmcw.pim = qemu_get_byte(f);
+    s->curr_status.pmcw.mbi = qemu_get_be16(f);
+    s->curr_status.pmcw.pom = qemu_get_byte(f);
+    s->curr_status.pmcw.pam = qemu_get_byte(f);
+    qemu_get_buffer(f, s->curr_status.pmcw.chpid, 8);
+    s->curr_status.pmcw.chars = qemu_get_be32(f);
+    /*     SCSW */
+    s->curr_status.scsw.flags = qemu_get_be16(f);
+    s->curr_status.scsw.ctrl = qemu_get_be16(f);
+    s->curr_status.scsw.cpa = qemu_get_be32(f);
+    s->curr_status.scsw.dstat = qemu_get_byte(f);
+    s->curr_status.scsw.cstat = qemu_get_byte(f);
+    s->curr_status.scsw.count = qemu_get_be16(f);
+    s->curr_status.mba = qemu_get_be64(f);
+    qemu_get_buffer(f, s->curr_status.mda, 4);
+    /* end SCHIB */
+    qemu_get_buffer(f, s->sense_data, 32);
+    s->channel_prog = qemu_get_be64(f);
+    /* last cmd */
+    s->last_cmd.cmd_code = qemu_get_byte(f);
+    s->last_cmd.flags = qemu_get_byte(f);
+    s->last_cmd.count = qemu_get_be16(f);
+    s->last_cmd.cda = qemu_get_be32(f);
+    s->last_cmd_valid = qemu_get_byte(f);
+    s->id.reserved = qemu_get_byte(f);
+    s->id.cu_type = qemu_get_be16(f);
+    s->id.cu_model = qemu_get_byte(f);
+    s->id.dev_type = qemu_get_be16(f);
+    s->id.dev_model = qemu_get_byte(f);
+    s->id.unused = qemu_get_byte(f);
+    for (i = 0; i < ARRAY_SIZE(s->id.ciw); i++) {
+        s->id.ciw[i].type = qemu_get_byte(f);
+        s->id.ciw[i].command = qemu_get_byte(f);
+        s->id.ciw[i].count = qemu_get_be16(f);
+    }
+    return 0;
+}
+
+
 static void css_init(void)
 {
     channel_subsys = g_malloc0(sizeof(*channel_subsys));
diff --git a/hw/s390x/css.h b/hw/s390x/css.h
index 6586106..c864ea7 100644
--- a/hw/s390x/css.h
+++ b/hw/s390x/css.h
@@ -85,6 +85,8 @@
 
 typedef SubchDev *(*css_subch_cb_func)(uint8_t m, uint8_t cssid, uint8_t ssid,
                                        uint16_t schid);
+void subch_device_save(SubchDev *s, QEMUFile *f);
+int subch_device_load(SubchDev *s, QEMUFile *f);
 int css_create_css_image(uint8_t cssid, bool default_image);
 bool css_devno_used(uint8_t cssid, uint8_t ssid, uint16_t devno);
 void css_subch_assign(uint8_t cssid, uint8_t ssid, uint16_t schid,
diff --git a/hw/s390x/s390-virtio-bus.c b/hw/s390x/s390-virtio-bus.c
index 9c71afa..7c8c81b 100644
--- a/hw/s390x/s390-virtio-bus.c
+++ b/hw/s390x/s390-virtio-bus.c
@@ -45,8 +45,6 @@
     do { } while (0)
 #endif
 
-#define VIRTIO_EXT_CODE   0x2603
-
 static void virtio_s390_bus_new(VirtioBusState *bus, size_t bus_size,
                                 VirtIOS390Device *dev);
 
@@ -113,15 +111,6 @@
     return bus;
 }
 
-static void s390_virtio_irq(S390CPU *cpu, int config_change, uint64_t token)
-{
-    if (kvm_enabled()) {
-        kvm_s390_virtio_irq(cpu, config_change, token);
-    } else {
-        cpu_inject_ext(cpu, VIRTIO_EXT_CODE, config_change, token);
-    }
-}
-
 static int s390_virtio_device_init(VirtIOS390Device *dev, VirtIODevice *vdev)
 {
     VirtIOS390Bus *bus;
@@ -144,8 +133,7 @@
     s390_virtio_device_sync(dev);
     s390_virtio_reset_idx(dev);
     if (dev->qdev.hotplugged) {
-        S390CPU *cpu = s390_cpu_addr2state(0);
-        s390_virtio_irq(cpu, VIRTIO_PARAM_DEV_ADD, dev->dev_offs);
+        s390_virtio_irq(VIRTIO_PARAM_DEV_ADD, dev->dev_offs);
     }
 
     return 0;
@@ -489,9 +477,8 @@
 {
     VirtIOS390Device *dev = to_virtio_s390_device_fast(d);
     uint64_t token = s390_virtio_device_vq_token(dev, vector);
-    S390CPU *cpu = s390_cpu_addr2state(0);
 
-    s390_virtio_irq(cpu, 0, token);
+    s390_virtio_irq(0, token);
 }
 
 static unsigned virtio_s390_get_features(DeviceState *d)
diff --git a/hw/s390x/virtio-ccw.c b/hw/s390x/virtio-ccw.c
index c4f21d3..05656a2 100644
--- a/hw/s390x/virtio-ccw.c
+++ b/hw/s390x/virtio-ccw.c
@@ -1275,6 +1275,97 @@
     return r;
 }
 
+static void virtio_ccw_save_queue(DeviceState *d, int n, QEMUFile *f)
+{
+    VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+    VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
+
+    qemu_put_be16(f, virtio_queue_vector(vdev, n));
+}
+
+static int virtio_ccw_load_queue(DeviceState *d, int n, QEMUFile *f)
+{
+    VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+    VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
+    uint16_t vector;
+
+    qemu_get_be16s(f, &vector);
+    virtio_queue_set_vector(vdev, n , vector);
+
+    return 0;
+}
+
+static void virtio_ccw_save_config(DeviceState *d, QEMUFile *f)
+{
+    VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+    SubchDev *s = dev->sch;
+
+    subch_device_save(s, f);
+    if (dev->indicators != NULL) {
+        qemu_put_be32(f, dev->indicators->len);
+        qemu_put_be64(f, dev->indicators->addr);
+    } else {
+        qemu_put_be32(f, 0);
+        qemu_put_be64(f, 0UL);
+    }
+    if (dev->indicators2 != NULL) {
+        qemu_put_be32(f, dev->indicators2->len);
+        qemu_put_be64(f, dev->indicators2->addr);
+    } else {
+        qemu_put_be32(f, 0);
+        qemu_put_be64(f, 0UL);
+    }
+    if (dev->summary_indicator != NULL) {
+        qemu_put_be32(f, dev->summary_indicator->len);
+        qemu_put_be64(f, dev->summary_indicator->addr);
+    } else {
+        qemu_put_be32(f, 0);
+        qemu_put_be64(f, 0UL);
+    }
+    qemu_put_be64(f, dev->routes.adapter.ind_offset);
+    qemu_put_byte(f, dev->thinint_isc);
+}
+
+static int virtio_ccw_load_config(DeviceState *d, QEMUFile *f)
+{
+    VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+    SubchDev *s = dev->sch;
+    int len;
+
+    s->driver_data = dev;
+    subch_device_load(s, f);
+    len = qemu_get_be32(f);
+    if (len != 0) {
+        dev->indicators = get_indicator(qemu_get_be64(f), len);
+    } else {
+        qemu_get_be64(f);
+        dev->indicators = NULL;
+    }
+    len = qemu_get_be32(f);
+    if (len != 0) {
+        dev->indicators2 = get_indicator(qemu_get_be64(f), len);
+    } else {
+        qemu_get_be64(f);
+        dev->indicators2 = NULL;
+    }
+    len = qemu_get_be32(f);
+    if (len != 0) {
+        dev->summary_indicator = get_indicator(qemu_get_be64(f), len);
+    } else {
+        qemu_get_be64(f);
+        dev->summary_indicator = NULL;
+    }
+    dev->routes.adapter.ind_offset = qemu_get_be64(f);
+    dev->thinint_isc = qemu_get_byte(f);
+    if (s->thinint_active) {
+        return css_register_io_adapter(CSS_IO_ADAPTER_VIRTIO,
+                                       dev->thinint_isc, true, false,
+                                       &dev->routes.adapter.adapter_id);
+    }
+
+    return 0;
+}
+
 /**************** Virtio-ccw Bus Device Descriptions *******************/
 
 static Property virtio_ccw_net_properties[] = {
@@ -1597,6 +1688,10 @@
     k->query_guest_notifiers = virtio_ccw_query_guest_notifiers;
     k->set_host_notifier = virtio_ccw_set_host_notifier;
     k->set_guest_notifiers = virtio_ccw_set_guest_notifiers;
+    k->save_queue = virtio_ccw_save_queue;
+    k->load_queue = virtio_ccw_load_queue;
+    k->save_config = virtio_ccw_save_config;
+    k->load_config = virtio_ccw_load_config;
 }
 
 static const TypeInfo virtio_ccw_bus_info = {
diff --git a/target-s390x/cpu.h b/target-s390x/cpu.h
index 06454d6..b13761d 100644
--- a/target-s390x/cpu.h
+++ b/target-s390x/cpu.h
@@ -357,30 +357,23 @@
 void s390x_cpu_timer(void *opaque);
 
 int s390_virtio_hypercall(CPUS390XState *env);
+void s390_virtio_irq(int config_change, uint64_t token);
 
 #ifdef CONFIG_KVM
 void kvm_s390_reset_vcpu(S390CPU *cpu);
-void kvm_s390_interrupt(S390CPU *cpu, int type, uint32_t code);
-void kvm_s390_virtio_irq(S390CPU *cpu, int config_change, uint64_t token);
-void kvm_s390_interrupt_internal(S390CPU *cpu, int type, uint32_t parm,
-                                 uint64_t parm64, int vm);
+void kvm_s390_virtio_irq(int config_change, uint64_t token);
+void kvm_s390_service_interrupt(uint32_t parm);
+void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq);
+void kvm_s390_floating_interrupt(struct kvm_s390_irq *irq);
+int kvm_s390_inject_flic(struct kvm_s390_irq *irq);
 #else
 static inline void kvm_s390_reset_vcpu(S390CPU *cpu)
 {
 }
-
-static inline void kvm_s390_interrupt(S390CPU *cpu, int type, uint32_t code)
+static inline void kvm_s390_virtio_irq(int config_change, uint64_t token)
 {
 }
-
-static inline void kvm_s390_virtio_irq(S390CPU *cpu, int config_change,
-                                       uint64_t token)
-{
-}
-
-static inline void kvm_s390_interrupt_internal(S390CPU *cpu, int type,
-                                               uint32_t parm, uint64_t parm64,
-                                               int vm)
+static inline void kvm_s390_service_interrupt(uint32_t parm)
 {
 }
 #endif
@@ -1066,23 +1059,23 @@
                                      uintptr_t retaddr);
 
 #ifdef CONFIG_KVM
-void kvm_s390_io_interrupt(S390CPU *cpu, uint16_t subchannel_id,
+void kvm_s390_io_interrupt(uint16_t subchannel_id,
                            uint16_t subchannel_nr, uint32_t io_int_parm,
                            uint32_t io_int_word);
-void kvm_s390_crw_mchk(S390CPU *cpu);
+void kvm_s390_crw_mchk(void);
 void kvm_s390_enable_css_support(S390CPU *cpu);
 int kvm_s390_assign_subch_ioeventfd(EventNotifier *notifier, uint32_t sch,
                                     int vq, bool assign);
 int kvm_s390_cpu_restart(S390CPU *cpu);
+void kvm_s390_clear_cmma_callback(void *opaque);
 #else
-static inline void kvm_s390_io_interrupt(S390CPU *cpu,
-                                        uint16_t subchannel_id,
+static inline void kvm_s390_io_interrupt(uint16_t subchannel_id,
                                         uint16_t subchannel_nr,
                                         uint32_t io_int_parm,
                                         uint32_t io_int_word)
 {
 }
-static inline void kvm_s390_crw_mchk(S390CPU *cpu)
+static inline void kvm_s390_crw_mchk(void)
 {
 }
 static inline void kvm_s390_enable_css_support(S390CPU *cpu)
@@ -1098,8 +1091,19 @@
 {
     return -ENOSYS;
 }
+static inline void kvm_s390_clear_cmma_callback(void *opaque)
+{
+}
 #endif
 
+static inline void cmma_reset(S390CPU *cpu)
+{
+    if (kvm_enabled()) {
+        CPUState *cs = CPU(cpu);
+        kvm_s390_clear_cmma_callback(cs->kvm_state);
+    }
+}
+
 static inline int s390_cpu_restart(S390CPU *cpu)
 {
     if (kvm_enabled()) {
@@ -1108,29 +1112,9 @@
     return -ENOSYS;
 }
 
-static inline void s390_io_interrupt(S390CPU *cpu,
-                                     uint16_t subchannel_id,
-                                     uint16_t subchannel_nr,
-                                     uint32_t io_int_parm,
-                                     uint32_t io_int_word)
-{
-    if (kvm_enabled()) {
-        kvm_s390_io_interrupt(cpu, subchannel_id, subchannel_nr, io_int_parm,
-                              io_int_word);
-    } else {
-        cpu_inject_io(cpu, subchannel_id, subchannel_nr, io_int_parm,
-                      io_int_word);
-    }
-}
-
-static inline void s390_crw_mchk(S390CPU *cpu)
-{
-    if (kvm_enabled()) {
-        kvm_s390_crw_mchk(cpu);
-    } else {
-        cpu_inject_crw_mchk(cpu);
-    }
-}
+void s390_io_interrupt(uint16_t subchannel_id, uint16_t subchannel_nr,
+                       uint32_t io_int_parm, uint32_t io_int_word);
+void s390_crw_mchk(void);
 
 static inline int s390_assign_subch_ioeventfd(EventNotifier *notifier,
                                               uint32_t sch_id, int vq,
diff --git a/target-s390x/interrupt.c b/target-s390x/interrupt.c
index 6d6580d..23a9114 100644
--- a/target-s390x/interrupt.c
+++ b/target-s390x/interrupt.c
@@ -1,7 +1,7 @@
 /*
  * QEMU S/390 Interrupt support
  *
- * Copyright IBM, Corp. 2012
+ * Copyright IBM Corp. 2012, 2014
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or (at your
  * option) any later version.  See the COPYING file in the top-level directory.
@@ -10,21 +10,59 @@
 #include "cpu.h"
 #include "sysemu/kvm.h"
 
+/*
+ * All of the following interrupts are floating, i.e. not per-vcpu.
+ * We just need a dummy cpustate in order to be able to inject in the
+ * non-kvm case.
+ */
 #if !defined(CONFIG_USER_ONLY)
-/* service interrupts are floating therefore we must not pass an cpustate */
 void s390_sclp_extint(uint32_t parm)
 {
-    S390CPU *dummy_cpu = s390_cpu_addr2state(0);
-    CPUS390XState *env = &dummy_cpu->env;
-
     if (kvm_enabled()) {
-#ifdef CONFIG_KVM
-        kvm_s390_interrupt_internal(dummy_cpu, KVM_S390_INT_SERVICE, parm,
-                                    0, 1);
-#endif
+        kvm_s390_service_interrupt(parm);
     } else {
+        S390CPU *dummy_cpu = s390_cpu_addr2state(0);
+        CPUS390XState *env = &dummy_cpu->env;
+
         env->psw.addr += 4;
         cpu_inject_ext(dummy_cpu, EXT_SERVICE, parm, 0);
     }
 }
+
+void s390_virtio_irq(int config_change, uint64_t token)
+{
+    if (kvm_enabled()) {
+        kvm_s390_virtio_irq(config_change, token);
+    } else {
+        S390CPU *dummy_cpu = s390_cpu_addr2state(0);
+
+        cpu_inject_ext(dummy_cpu, EXT_VIRTIO, config_change, token);
+    }
+}
+
+void s390_io_interrupt(uint16_t subchannel_id, uint16_t subchannel_nr,
+                       uint32_t io_int_parm, uint32_t io_int_word)
+{
+    if (kvm_enabled()) {
+        kvm_s390_io_interrupt(subchannel_id, subchannel_nr, io_int_parm,
+                              io_int_word);
+    } else {
+        S390CPU *dummy_cpu = s390_cpu_addr2state(0);
+
+        cpu_inject_io(dummy_cpu, subchannel_id, subchannel_nr, io_int_parm,
+                      io_int_word);
+    }
+}
+
+void s390_crw_mchk(void)
+{
+    if (kvm_enabled()) {
+        kvm_s390_crw_mchk();
+    } else {
+        S390CPU *dummy_cpu = s390_cpu_addr2state(0);
+
+        cpu_inject_crw_mchk(dummy_cpu);
+    }
+}
+
 #endif
diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index 7a07f9d..a1a4cc2 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -32,6 +32,7 @@
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
+#include "hw/hw.h"
 #include "cpu.h"
 #include "sysemu/device_tree.h"
 #include "qapi/qmp/qjson.h"
@@ -82,6 +83,8 @@
 #define DIAG_KVM_BREAKPOINT             0x501
 
 #define ICPT_INSTRUCTION                0x04
+#define ICPT_PROGRAM                    0x08
+#define ICPT_EXT_INT                    0x14
 #define ICPT_WAITPSW                    0x1c
 #define ICPT_SOFT_INTERCEPT             0x24
 #define ICPT_CPU_STOP                   0x28
@@ -104,10 +107,67 @@
 
 static void *legacy_s390_alloc(size_t size);
 
+static int kvm_s390_check_clear_cmma(KVMState *s)
+{
+    struct kvm_device_attr attr = {
+        .group = KVM_S390_VM_MEM_CTRL,
+        .attr = KVM_S390_VM_MEM_CLR_CMMA,
+    };
+
+    return kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attr);
+}
+
+static int kvm_s390_check_enable_cmma(KVMState *s)
+{
+    struct kvm_device_attr attr = {
+        .group = KVM_S390_VM_MEM_CTRL,
+        .attr = KVM_S390_VM_MEM_ENABLE_CMMA,
+    };
+
+    return kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attr);
+}
+
+void kvm_s390_clear_cmma_callback(void *opaque)
+{
+    int rc;
+    KVMState *s = opaque;
+    struct kvm_device_attr attr = {
+        .group = KVM_S390_VM_MEM_CTRL,
+        .attr = KVM_S390_VM_MEM_CLR_CMMA,
+    };
+
+    rc = kvm_vm_ioctl(s, KVM_SET_DEVICE_ATTR, &attr);
+    trace_kvm_clear_cmma(rc);
+}
+
+static void kvm_s390_enable_cmma(KVMState *s)
+{
+    int rc;
+    struct kvm_device_attr attr = {
+        .group = KVM_S390_VM_MEM_CTRL,
+        .attr = KVM_S390_VM_MEM_ENABLE_CMMA,
+    };
+
+    if (kvm_s390_check_enable_cmma(s) || kvm_s390_check_clear_cmma(s)) {
+        return;
+    }
+
+    rc = kvm_vm_ioctl(s, KVM_SET_DEVICE_ATTR, &attr);
+    if (!rc) {
+        qemu_register_reset(kvm_s390_clear_cmma_callback, s);
+    }
+    trace_kvm_enable_cmma(rc);
+}
+
 int kvm_arch_init(KVMState *s)
 {
     cap_sync_regs = kvm_check_extension(s, KVM_CAP_SYNC_REGS);
     cap_async_pf = kvm_check_extension(s, KVM_CAP_ASYNC_PF);
+
+    if (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES)) {
+        kvm_s390_enable_cmma(s);
+    }
+
     if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
         || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
         phys_mem_set_alloc(legacy_s390_alloc);
@@ -495,47 +555,138 @@
     return cs->halted;
 }
 
-void kvm_s390_interrupt_internal(S390CPU *cpu, int type, uint32_t parm,
-                                 uint64_t parm64, int vm)
+static int s390_kvm_irq_to_interrupt(struct kvm_s390_irq *irq,
+                                     struct kvm_s390_interrupt *interrupt)
 {
+    int r = 0;
+
+    interrupt->type = irq->type;
+    switch (irq->type) {
+    case KVM_S390_INT_VIRTIO:
+        interrupt->parm = irq->u.ext.ext_params;
+        /* fall through */
+    case KVM_S390_INT_PFAULT_INIT:
+    case KVM_S390_INT_PFAULT_DONE:
+        interrupt->parm64 = irq->u.ext.ext_params2;
+        break;
+    case KVM_S390_PROGRAM_INT:
+        interrupt->parm = irq->u.pgm.code;
+        break;
+    case KVM_S390_SIGP_SET_PREFIX:
+        interrupt->parm = irq->u.prefix.address;
+        break;
+    case KVM_S390_INT_SERVICE:
+        interrupt->parm = irq->u.ext.ext_params;
+        break;
+    case KVM_S390_MCHK:
+        interrupt->parm = irq->u.mchk.cr14;
+        interrupt->parm64 = irq->u.mchk.mcic;
+        break;
+    case KVM_S390_INT_EXTERNAL_CALL:
+        interrupt->parm = irq->u.extcall.code;
+        break;
+    case KVM_S390_INT_EMERGENCY:
+        interrupt->parm = irq->u.emerg.code;
+        break;
+    case KVM_S390_SIGP_STOP:
+    case KVM_S390_RESTART:
+        break; /* These types have no parameters */
+    case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
+        interrupt->parm = irq->u.io.subchannel_id << 16;
+        interrupt->parm |= irq->u.io.subchannel_nr;
+        interrupt->parm64 = (uint64_t)irq->u.io.io_int_parm << 32;
+        interrupt->parm64 |= irq->u.io.io_int_word;
+        break;
+    default:
+        r = -EINVAL;
+        break;
+    }
+    return r;
+}
+
+void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq)
+{
+    struct kvm_s390_interrupt kvmint = {};
     CPUState *cs = CPU(cpu);
-    struct kvm_s390_interrupt kvmint;
     int r;
 
-    if (!cs->kvm_state) {
-        return;
+    r = s390_kvm_irq_to_interrupt(irq, &kvmint);
+    if (r < 0) {
+        fprintf(stderr, "%s called with bogus interrupt\n", __func__);
+        exit(1);
     }
 
-    kvmint.type = type;
-    kvmint.parm = parm;
-    kvmint.parm64 = parm64;
-
-    if (vm) {
-        r = kvm_vm_ioctl(cs->kvm_state, KVM_S390_INTERRUPT, &kvmint);
-    } else {
-        r = kvm_vcpu_ioctl(cs, KVM_S390_INTERRUPT, &kvmint);
-    }
-
+    r = kvm_vcpu_ioctl(cs, KVM_S390_INTERRUPT, &kvmint);
     if (r < 0) {
         fprintf(stderr, "KVM failed to inject interrupt\n");
         exit(1);
     }
 }
 
-void kvm_s390_virtio_irq(S390CPU *cpu, int config_change, uint64_t token)
+static void __kvm_s390_floating_interrupt(struct kvm_s390_irq *irq)
 {
-    kvm_s390_interrupt_internal(cpu, KVM_S390_INT_VIRTIO, config_change,
-                                token, 1);
+    struct kvm_s390_interrupt kvmint = {};
+    int r;
+
+    r = s390_kvm_irq_to_interrupt(irq, &kvmint);
+    if (r < 0) {
+        fprintf(stderr, "%s called with bogus interrupt\n", __func__);
+        exit(1);
+    }
+
+    r = kvm_vm_ioctl(kvm_state, KVM_S390_INTERRUPT, &kvmint);
+    if (r < 0) {
+        fprintf(stderr, "KVM failed to inject interrupt\n");
+        exit(1);
+    }
 }
 
-void kvm_s390_interrupt(S390CPU *cpu, int type, uint32_t code)
+void kvm_s390_floating_interrupt(struct kvm_s390_irq *irq)
 {
-    kvm_s390_interrupt_internal(cpu, type, code, 0, 0);
+    static bool use_flic = true;
+    int r;
+
+    if (use_flic) {
+        r = kvm_s390_inject_flic(irq);
+        if (r == -ENOSYS) {
+            use_flic = false;
+        }
+        if (!r) {
+            return;
+        }
+    }
+    __kvm_s390_floating_interrupt(irq);
+}
+
+void kvm_s390_virtio_irq(int config_change, uint64_t token)
+{
+    struct kvm_s390_irq irq = {
+        .type = KVM_S390_INT_VIRTIO,
+        .u.ext.ext_params = config_change,
+        .u.ext.ext_params2 = token,
+    };
+
+    kvm_s390_floating_interrupt(&irq);
+}
+
+void kvm_s390_service_interrupt(uint32_t parm)
+{
+    struct kvm_s390_irq irq = {
+        .type = KVM_S390_INT_SERVICE,
+        .u.ext.ext_params = parm,
+    };
+
+    kvm_s390_floating_interrupt(&irq);
 }
 
 static void enter_pgmcheck(S390CPU *cpu, uint16_t code)
 {
-    kvm_s390_interrupt(cpu, KVM_S390_PROGRAM_INT, code);
+    struct kvm_s390_irq irq = {
+        .type = KVM_S390_PROGRAM_INT,
+        .u.pgm.code = code,
+    };
+
+    kvm_s390_vcpu_interrupt(cpu, &irq);
 }
 
 static int kvm_sclp_service_call(S390CPU *cpu, struct kvm_run *run,
@@ -751,7 +902,11 @@
 
 int kvm_s390_cpu_restart(S390CPU *cpu)
 {
-    kvm_s390_interrupt(cpu, KVM_S390_RESTART, 0);
+    struct kvm_s390_irq irq = {
+        .type = KVM_S390_RESTART,
+    };
+
+    kvm_s390_vcpu_interrupt(cpu, &irq);
     s390_add_running_cpu(cpu);
     qemu_cpu_kick(CPU(cpu));
     DPRINTF("DONE: KVM cpu restart: %p\n", &cpu->env);
@@ -872,6 +1027,28 @@
     return cs->kvm_run->psw_addr == 0xfffUL;
 }
 
+static void guest_panicked(void)
+{
+    QObject *data;
+
+    data = qobject_from_jsonf("{ 'action': %s }", "pause");
+    monitor_protocol_event(QEVENT_GUEST_PANICKED, data);
+    qobject_decref(data);
+
+    vm_stop(RUN_STATE_GUEST_PANICKED);
+}
+
+static void unmanageable_intercept(S390CPU *cpu, const char *str, int pswoffset)
+{
+    CPUState *cs = CPU(cpu);
+
+    error_report("Unmanageable %s! CPU%i new PSW: 0x%016lx:%016lx",
+                 str, cs->cpu_index, ldq_phys(cs->as, cpu->env.psa + pswoffset),
+                 ldq_phys(cs->as, cpu->env.psa + pswoffset + 8));
+    s390_del_running_cpu(cpu);
+    guest_panicked();
+}
+
 static int handle_intercept(S390CPU *cpu)
 {
     CPUState *cs = CPU(cpu);
@@ -885,18 +1062,23 @@
         case ICPT_INSTRUCTION:
             r = handle_instruction(cpu, run);
             break;
+        case ICPT_PROGRAM:
+            unmanageable_intercept(cpu, "program interrupt",
+                                   offsetof(LowCore, program_new_psw));
+            r = EXCP_HALTED;
+            break;
+        case ICPT_EXT_INT:
+            unmanageable_intercept(cpu, "external interrupt",
+                                   offsetof(LowCore, external_new_psw));
+            r = EXCP_HALTED;
+            break;
         case ICPT_WAITPSW:
             /* disabled wait, since enabled wait is handled in kernel */
             if (s390_del_running_cpu(cpu) == 0) {
                 if (is_special_wait_psw(cs)) {
                     qemu_system_shutdown_request();
                 } else {
-                    QObject *data;
-
-                    data = qobject_from_jsonf("{ 'action': %s }", "pause");
-                    monitor_protocol_event(QEVENT_GUEST_PANICKED, data);
-                    qobject_decref(data);
-                    vm_stop(RUN_STATE_GUEST_PANICKED);
+                    guest_panicked();
                 }
             }
             r = EXCP_HALTED;
@@ -944,18 +1126,10 @@
          * If an I/O interrupt had been dequeued, we have to reinject it.
          */
         if (run->s390_tsch.dequeued) {
-            uint16_t subchannel_id = run->s390_tsch.subchannel_id;
-            uint16_t subchannel_nr = run->s390_tsch.subchannel_nr;
-            uint32_t io_int_parm = run->s390_tsch.io_int_parm;
-            uint32_t io_int_word = run->s390_tsch.io_int_word;
-            uint32_t type = ((subchannel_id & 0xff00) << 24) |
-                ((subchannel_id & 0x00060) << 22) | (subchannel_nr << 16);
-
-            kvm_s390_interrupt_internal(cpu, type,
-                                        ((uint32_t)subchannel_id << 16)
-                                        | subchannel_nr,
-                                        ((uint64_t)io_int_parm << 32)
-                                        | io_int_word, 1);
+            kvm_s390_io_interrupt(run->s390_tsch.subchannel_id,
+                                  run->s390_tsch.subchannel_nr,
+                                  run->s390_tsch.io_int_parm,
+                                  run->s390_tsch.io_int_word);
         }
         ret = 0;
     }
@@ -1040,27 +1214,34 @@
     return 1;
 }
 
-void kvm_s390_io_interrupt(S390CPU *cpu, uint16_t subchannel_id,
+void kvm_s390_io_interrupt(uint16_t subchannel_id,
                            uint16_t subchannel_nr, uint32_t io_int_parm,
                            uint32_t io_int_word)
 {
-    uint32_t type;
+    struct kvm_s390_irq irq = {
+        .u.io.subchannel_id = subchannel_id,
+        .u.io.subchannel_nr = subchannel_nr,
+        .u.io.io_int_parm = io_int_parm,
+        .u.io.io_int_word = io_int_word,
+    };
 
     if (io_int_word & IO_INT_WORD_AI) {
-        type = KVM_S390_INT_IO(1, 0, 0, 0);
+        irq.type = KVM_S390_INT_IO(1, 0, 0, 0);
     } else {
-        type = ((subchannel_id & 0xff00) << 24) |
+        irq.type = ((subchannel_id & 0xff00) << 24) |
             ((subchannel_id & 0x00060) << 22) | (subchannel_nr << 16);
     }
-    kvm_s390_interrupt_internal(cpu, type,
-                                ((uint32_t)subchannel_id << 16) | subchannel_nr,
-                                ((uint64_t)io_int_parm << 32) | io_int_word, 1);
+    kvm_s390_floating_interrupt(&irq);
 }
 
-void kvm_s390_crw_mchk(S390CPU *cpu)
+void kvm_s390_crw_mchk(void)
 {
-    kvm_s390_interrupt_internal(cpu, KVM_S390_MCHK, 1 << 28,
-                                0x00400f1d40330000, 1);
+    struct kvm_s390_irq irq = {
+        .type = KVM_S390_MCHK,
+        .u.mchk.cr14 = 1 << 28,
+        .u.mchk.mcic = 0x00400f1d40330000,
+    };
+    kvm_s390_floating_interrupt(&irq);
 }
 
 void kvm_s390_enable_css_support(S390CPU *cpu)
diff --git a/target-s390x/misc_helper.c b/target-s390x/misc_helper.c
index 9dae025..0b62582 100644
--- a/target-s390x/misc_helper.c
+++ b/target-s390x/misc_helper.c
@@ -85,7 +85,12 @@
 
     if (kvm_enabled()) {
 #ifdef CONFIG_KVM
-        kvm_s390_interrupt(cpu, KVM_S390_PROGRAM_INT, code);
+        struct kvm_s390_irq irq = {
+            .type = KVM_S390_PROGRAM_INT,
+            .u.pgm.code = code,
+        };
+
+        kvm_s390_vcpu_interrupt(cpu, &irq);
 #endif
     } else {
         CPUState *cs = CPU(cpu);
@@ -136,6 +141,7 @@
     pause_all_vcpus();
     cpu_synchronize_all_states();
     cpu_full_reset_all();
+    cmma_reset(cpu);
     io_subsystem_reset();
     scc->load_normal(CPU(cpu));
     cpu_synchronize_all_post_reset();
@@ -150,6 +156,7 @@
     pause_all_vcpus();
     cpu_synchronize_all_states();
     cpu_reset_all();
+    cmma_reset(cpu);
     io_subsystem_reset();
     scc->initial_cpu_reset(CPU(cpu));
     scc->load_normal(CPU(cpu));
diff --git a/trace-events b/trace-events
index ffe6e62..e984e76 100644
--- a/trace-events
+++ b/trace-events
@@ -1258,3 +1258,7 @@
 # hw/pci/pci_host.c
 pci_cfg_read(const char *dev, unsigned devid, unsigned fnid, unsigned offs, unsigned val) "%s %02u:%u @0x%x -> 0x%x"
 pci_cfg_write(const char *dev, unsigned devid, unsigned fnid, unsigned offs, unsigned val) "%s %02u:%u @0x%x <- 0x%x"
+
+# target-s390x/kvm.c
+kvm_enable_cmma(int rc) "CMMA: enabling with result code %d"
+kvm_clear_cmma(int rc) "CMMA: clearing with result code %d"