Add facility to get notification of domain suspend by event channel.

This event channel will be notified when the domain transitions to the
suspended state, which can be much faster than raising VIRQ_DOM_EXC
and waiting for the notification to be propagated via xenstore.

No attempt is made here to prevent multiple subscribers (last one
wins), or to detect that the subscriber has gone away. Userspace tools
should take care.

Signed-off-by: Brendan Cully <brendan@cs.ubc.ca>

diff --git a/tools/libxc/xc_domain.c b/tools/libxc/xc_domain.c
index 4afec53bd5580cfb82046830fb63bb4a8d4a11c9..b4d7895801c943e567f3ea7093847c264a17b9b9 100644 (file)
--- a/tools/libxc/xc_domain.c
+++ b/tools/libxc/xc_domain.c
@@ -981,6 +981,17 @@ int xc_domain_set_target(
     return do_domctl(xc_handle, &domctl);
 }
 
+int xc_dom_subscribe(int xc_handle, domid_t dom, evtchn_port_t port)
+{
+    DECLARE_DOMCTL;
+
+    domctl.cmd = XEN_DOMCTL_subscribe;
+    domctl.domain = dom;
+    domctl.u.subscribe.port = port;
+
+    return do_domctl(xc_handle, &domctl);
+}
+
 /*
  * Local variables:
  * mode: C

diff --git a/tools/libxc/xenctrl.h b/tools/libxc/xenctrl.h
index aaace1ff2a26e23c9d41336ec62149ea47a989f8..cbe6bf853691705d24d200ae4771fe9bc9e3b668 100644 (file)
--- a/tools/libxc/xenctrl.h
+++ b/tools/libxc/xenctrl.h
@@ -810,6 +810,12 @@ int xc_acm_op(int xc_handle, int cmd, void *arg, unsigned long arg_size);
 
 int xc_flask_op(int xc_handle, flask_op_t *op);
 
+/*
+ * Subscribe to state changes in a domain via evtchn.
+ * Returns -1 on failure, in which case errno will be set appropriately.
+ */
+int xc_dom_subscribe(int xc_handle, domid_t domid, evtchn_port_t port);
+
 /**************************
  * GRANT TABLE OPERATIONS *
  **************************/

diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c
index 89032bb91d964c8796235a166f639a96c4892370..8188fcf48ecd440b5921c89d9e06a123d68575e4 100644 (file)
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -61,6 +61,7 @@
 #include <asm/msr.h>
 #include <asm/shared.h>
 #include <asm/x86_emulate.h>
+#include <asm/traps.h>
 #include <asm/hvm/vpt.h>
 #include <public/arch-x86/cpuid.h>
 
@@ -2678,25 +2679,51 @@ asmlinkage void do_general_protection(struct cpu_user_regs *regs)
     panic("GENERAL PROTECTION FAULT\n[error_code=%04x]\n", regs->error_code);
 }
 
+static DEFINE_PER_CPU(struct softirq_trap, softirq_trap);
+
 static void nmi_mce_softirq(void)
 {
-    /* Only used to defer wakeup of dom0,vcpu0 to a safe (non-NMI) context. */
-    vcpu_kick(dom0->vcpu[0]);
+    int cpu = smp_processor_id();
+    struct softirq_trap *st = &per_cpu(softirq_trap, cpu);
+    cpumask_t affinity;
+
+    BUG_ON(st == NULL);
+    BUG_ON(st->vcpu == NULL);
+
+    /* Set the tmp value unconditionally, so that
+     * the check in the iret hypercall works. */
+    st->vcpu->cpu_affinity_tmp = st->vcpu->cpu_affinity;
+
+    if ((cpu != st->processor)
+       || (st->processor != st->vcpu->processor))
+    {
+        /* We are on a different physical cpu.
+         * Make sure to wakeup the vcpu on the
+         * specified processor.
+         */
+        cpus_clear(affinity);
+        cpu_set(st->processor, affinity);
+        vcpu_set_affinity(st->vcpu, &affinity);
+
+        /* Affinity is restored in the iret hypercall. */
+    }
+
+    /* Only used to defer wakeup of domain/vcpu to
+     * a safe (non-NMI/MCE) context.
+     */
+    vcpu_kick(st->vcpu);
 }
 
 static void nmi_dom0_report(unsigned int reason_idx)
 {
-    struct domain *d;
-    struct vcpu   *v;
+    struct domain *d = dom0;
 
-    if ( ((d = dom0) == NULL) || ((v = d->vcpu[0]) == NULL) )
+    if ( (d == NULL) || (d->vcpu[0] == NULL) )
         return;
 
     set_bit(reason_idx, nmi_reason(d));
 
-    /* Not safe to wake a vcpu here, or even to schedule a tasklet! */
-    if ( !test_and_set_bool(v->nmi_pending) )
-        raise_softirq(NMI_MCE_SOFTIRQ);
+    send_guest_trap(d, 0, TRAP_nmi);
 }
 
 asmlinkage void mem_parity_error(struct cpu_user_regs *regs)
@@ -3010,6 +3037,35 @@ long unregister_guest_nmi_callback(void)
     return 0;
 }
 
+int send_guest_trap(struct domain *d, uint16_t vcpuid, unsigned int trap_nr)
+{
+    struct vcpu *v;
+    struct softirq_trap *st;
+
+    BUG_ON(d == NULL);
+    BUG_ON(vcpuid >= MAX_VIRT_CPUS);
+    v = d->vcpu[vcpuid];
+
+    switch (trap_nr) {
+    case TRAP_nmi:
+        if ( !test_and_set_bool(v->nmi_pending) ) {
+               st = &per_cpu(softirq_trap, smp_processor_id());
+               st->domain = dom0;
+               st->vcpu = dom0->vcpu[0];
+               st->processor = st->vcpu->processor;
+
+               /* not safe to wake up a vcpu here */
+               raise_softirq(NMI_MCE_SOFTIRQ);
+               return 0;
+        }
+        break;
+    }
+
+    /* delivery failed */
+    return -EIO;
+}
+
+
 long do_set_trap_table(XEN_GUEST_HANDLE(const_trap_info_t) traps)
 {
     struct trap_info cur;

diff --git a/xen/arch/x86/x86_32/traps.c b/xen/arch/x86/x86_32/traps.c
index a09d15b523579183bcc438d55420843dc36709dc..ea75aa7bc2722f3eda2f1606202f991e17d25d00 100644 (file)
--- a/xen/arch/x86/x86_32/traps.c
+++ b/xen/arch/x86/x86_32/traps.c
@@ -255,6 +255,10 @@ unsigned long do_iret(void)
             goto exit_and_crash;
     }
 
+    /* Restore affinity.  */
+    if (!cpus_equal(v->cpu_affinity_tmp, v->cpu_affinity))
+        vcpu_set_affinity(v, &v->cpu_affinity_tmp);
+
     /* No longer in NMI context. */
     v->nmi_masked = 0;
 

diff --git a/xen/arch/x86/x86_64/compat/traps.c b/xen/arch/x86/x86_64/compat/traps.c
index 65c1e901473e9cbc2347b3d2fdf9f1a9b0b9d0b4..22098ea82588c1b9d2fe8efd7036965bd40400ea 100644 (file)
--- a/xen/arch/x86/x86_64/compat/traps.c
+++ b/xen/arch/x86/x86_64/compat/traps.c
@@ -121,6 +121,10 @@ unsigned int compat_iret(void)
     else
         regs->_esp += 16;
 
+    /* Restore affinity.  */
+    if (!cpus_equal(v->cpu_affinity_tmp, v->cpu_affinity))
+        vcpu_set_affinity(v, &v->cpu_affinity_tmp);
+
     /* No longer in NMI context. */
     v->nmi_masked = 0;
 

diff --git a/xen/arch/x86/x86_64/traps.c b/xen/arch/x86/x86_64/traps.c
index 14769786ec2a76579523c7949edfd60f3b6e267d..700d3a0e9492634afd3710d7c4db29d9fdad5d7e 100644 (file)
--- a/xen/arch/x86/x86_64/traps.c
+++ b/xen/arch/x86/x86_64/traps.c
@@ -288,6 +288,10 @@ unsigned long do_iret(void)
         regs->rcx = iret_saved.rcx;
     }
 
+    /* Restore affinity.  */
+    if (!cpus_equal(v->cpu_affinity_tmp, v->cpu_affinity))
+        vcpu_set_affinity(v, &v->cpu_affinity_tmp);
+
     /* No longer in NMI context. */
     v->nmi_masked = 0;
 

diff --git a/xen/common/domain.c b/xen/common/domain.c
index 43191cd641a38c24967d8f749800b345cfcc69e6..aa5dac5ebea4b2e8ef59b1bf1c0811920a04e326 100644 (file)
--- a/xen/common/domain.c
+++ b/xen/common/domain.c
@@ -97,7 +97,13 @@ static void __domain_finalise_shutdown(struct domain *d)
             return;
 
     d->is_shut_down = 1;
-    send_guest_global_virq(dom0, VIRQ_DOM_EXC);
+    if ( d->shutdown_code == SHUTDOWN_suspend
+         && d->suspend_evtchn > 0 )
+    {
+        evtchn_set_pending(dom0->vcpu[0], d->suspend_evtchn);
+    }
+    else
+        send_guest_global_virq(dom0, VIRQ_DOM_EXC);
 }
 
 static void vcpu_check_shutdown(struct vcpu *v)

diff --git a/xen/common/domctl.c b/xen/common/domctl.c
index 13db2152e7aca2269d2ee2576b024482b411e61f..0364116e945837d034ec81cd68e6675a6f0fd4b8 100644 (file)
--- a/xen/common/domctl.c
+++ b/xen/common/domctl.c
@@ -824,6 +824,21 @@ long do_domctl(XEN_GUEST_HANDLE(xen_domctl_t) u_domctl)
     }
     break;
 
+    case XEN_DOMCTL_subscribe:
+    {
+        struct domain *d;
+
+        ret = -ESRCH;
+        d = rcu_lock_domain_by_id(op->domain);
+        if ( d != NULL )
+        {
+            d->suspend_evtchn = op->u.subscribe.port;
+            rcu_unlock_domain(d);
+            ret = 0;
+        }
+    }
+    break;
+
     default:
         ret = arch_do_domctl(op, u_domctl);
         break;

diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h
index 8e400e2dfc85f8fe542c335091ce7eb7a4c7cc8f..d26ac75d5895230d0162035c72d72fc507af1bcf 100644 (file)
--- a/xen/include/public/domctl.h
+++ b/xen/include/public/domctl.h
@@ -600,6 +600,13 @@ typedef struct xen_domctl_cpuid xen_domctl_cpuid_t;
 DEFINE_XEN_GUEST_HANDLE(xen_domctl_cpuid_t);
 #endif
 
+#define XEN_DOMCTL_subscribe          29
+struct xen_domctl_subscribe {
+    uint32_t port; /* IN */
+};
+typedef struct xen_domctl_subscribe xen_domctl_subscribe_t;
+DEFINE_XEN_GUEST_HANDLE(xen_domctl_subscribe_t);
+
 struct xen_domctl {
     uint32_t cmd;
     uint32_t interface_version; /* XEN_DOMCTL_INTERFACE_VERSION */
@@ -638,6 +645,7 @@ struct xen_domctl {
         struct xen_domctl_ext_vcpucontext   ext_vcpucontext;
         struct xen_domctl_set_opt_feature   set_opt_feature;
         struct xen_domctl_set_target        set_target;
+        struct xen_domctl_subscribe         subscribe;
 #if defined(__i386__) || defined(__x86_64__)
         struct xen_domctl_cpuid             cpuid;
 #endif

diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h
index 1565e659788919459187aa559283ca007ccee334..7fa80d300cb80fa371e7e3866e8c5d311bc001f3 100644 (file)
--- a/xen/include/xen/sched.h
+++ b/xen/include/xen/sched.h
@@ -130,6 +130,8 @@ struct vcpu
 
     /* Bitmask of CPUs on which this VCPU may run. */
     cpumask_t        cpu_affinity;
+    /* Used to change affinity temporarily. */
+    cpumask_t        cpu_affinity_tmp;
 
     /* Bitmask of CPUs which are holding onto this VCPU's state. */
     cpumask_t        vcpu_dirty_cpumask;
@@ -209,6 +211,10 @@ struct domain
     bool_t           is_shut_down;     /* fully shut down? */
     int              shutdown_code;
 
+    /* If this is not 0, send suspend notification here instead of
+     * raising DOM_EXC */
+    int              suspend_evtchn;
+
     atomic_t         pause_count;
 
     unsigned long    vm_assist;