stubdom: add live migration support by having ioemu just notify the

hypervisor about memory changes.

The impact on disk performance is typically making it from 71.5MBps
down to 70.5Mbps during the live migration. The impact on network
performance is actually even hard to measure.

Signed-off-by: Samuel Thibault <samuel.thibault@eu.citrix.com>

diff --git a/tools/ioemu/target-i386-dm/exec-dm.c b/tools/ioemu/target-i386-dm/exec-dm.c
index 2ebdebff040cf5a7b75308f21588ab38b38a31c8..1df1ed96c2148608cea79433cb908e209b906d40 100644 (file)
--- a/tools/ioemu/target-i386-dm/exec-dm.c
+++ b/tools/ioemu/target-i386-dm/exec-dm.c
@@ -483,9 +483,11 @@ static void memcpy_words(void *dst, void *src, size_t n)
 }
 #endif
 
-void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, 
-                            int len, int is_write)
+void cpu_physical_memory_rw(target_phys_addr_t _addr, uint8_t *buf, 
+                            int _len, int is_write)
 {
+    target_phys_addr_t addr = _addr;
+    int len = _len;
     int l, io_index;
     uint8_t *ptr;
     uint32_t val;
@@ -520,6 +522,7 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
             } else if ((ptr = phys_ram_addr(addr)) != NULL) {
                 /* Writing to RAM */
                 memcpy_words(ptr, buf, l);
+#ifndef CONFIG_STUBDOM
                 if (logdirty_bitmap != NULL) {
                     /* Record that we have dirtied this frame */
                     unsigned long pfn = addr >> TARGET_PAGE_BITS;
@@ -531,6 +534,7 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
                             |= 1UL << pfn % HOST_LONG_BITS;
                     }
                 }
+#endif
 #ifdef __ia64__
                 sync_icache(ptr, l);
 #endif 
@@ -566,6 +570,13 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
         addr += l;
     }
 
+#ifdef CONFIG_STUBDOM
+    if (logdirty_bitmap != NULL)
+        xc_hvm_modified_memory(xc_handle, domid, _addr >> TARGET_PAGE_BITS,
+                (_addr + _len + TARGET_PAGE_SIZE - 1) >> TARGET_PAGE_BITS
+                    - _addr >> TARGET_PAGE_BITS);
+#endif
+
     mapcache_unlock();
 }
 #endif

diff --git a/tools/ioemu/xenstore.c b/tools/ioemu/xenstore.c
index 0162dfe7d0b96eb700b27506e9cfaff020c742e1..50faa25ef7f4126421d84fbd5297bf73d42a4b6c 100644 (file)
--- a/tools/ioemu/xenstore.c
+++ b/tools/ioemu/xenstore.c
@@ -402,6 +402,10 @@ void xenstore_process_logdirty_event(void)
             /* No key yet: wait for the next watch */
             return;
 
+#ifdef CONFIG_STUBDOM
+        /* We pass the writes to hypervisor */
+        seg = (void*)1;
+#else
         strncpy(key_terminated, key_ascii, 16);
         free(key_ascii);
         key = (key_t) strtoull(key_terminated, NULL, 16);
@@ -417,11 +421,6 @@ void xenstore_process_logdirty_event(void)
         fprintf(logfile, "%s: key=%16.16llx size=%lu\n", __FUNCTION__,
                 (unsigned long long)key, logdirty_bitmap_size);
 
-#ifdef CONFIG_STUBDOM
-        /* XXX we just can't use shm. */
-        fprintf(logfile, "Log dirty is not implemented in stub domains!\n");
-        return;
-#else
         shmid = shmget(key, 2 * logdirty_bitmap_size, S_IRUSR|S_IWUSR);
         if (shmid == -1) {
             fprintf(logfile, "Log-dirty: shmget failed: segment %16.16llx "

diff --git a/tools/libxc/xc_misc.c b/tools/libxc/xc_misc.c
index c68461a496fe48c38351fdad718b2356fed76734..0fe5272eabc3f1f5066b63ad0fd219dc2ff899e8 100644 (file)
--- a/tools/libxc/xc_misc.c
+++ b/tools/libxc/xc_misc.c
@@ -267,6 +267,34 @@ int xc_hvm_track_dirty_vram(
     return rc;
 }
 
+int xc_hvm_modified_memory(
+    int xc_handle, domid_t dom, uint64_t first_pfn, uint64_t nr)
+{
+    DECLARE_HYPERCALL;
+    struct xen_hvm_modified_memory arg;
+    int rc;
+
+    hypercall.op     = __HYPERVISOR_hvm_op;
+    hypercall.arg[0] = HVMOP_modified_memory;
+    hypercall.arg[1] = (unsigned long)&arg;
+
+    arg.domid     = dom;
+    arg.first_pfn = first_pfn;
+    arg.nr        = nr;
+
+    if ( (rc = lock_pages(&arg, sizeof(arg))) != 0 )
+    {
+        PERROR("Could not lock memory");
+        return rc;
+    }
+
+    rc = do_xen_hypercall(xc_handle, &hypercall);
+
+    unlock_pages(&arg, sizeof(arg));
+
+    return rc;
+}
+
 void *xc_map_foreign_pages(int xc_handle, uint32_t dom, int prot,
                            const xen_pfn_t *arr, int num)
 {

diff --git a/tools/libxc/xenctrl.h b/tools/libxc/xenctrl.h
index ed782b49f0990186c0528cb161192b219ebd01bc..aaace1ff2a26e23c9d41336ec62149ea47a989f8 100644 (file)
--- a/tools/libxc/xenctrl.h
+++ b/tools/libxc/xenctrl.h
@@ -929,6 +929,12 @@ int xc_hvm_track_dirty_vram(
     uint64_t first_pfn, uint64_t nr,
     unsigned long *bitmap);
 
+/*
+ * Notify that some pages got modified by the Device Model
+ */
+int xc_hvm_modified_memory(
+    int xc_handle, domid_t dom, uint64_t first_pfn, uint64_t nr);
+
 typedef enum {
   XC_ERROR_NONE = 0,
   XC_INTERNAL_ERROR = 1,

diff --git a/xen/arch/ia64/vmx/vmx_hypercall.c b/xen/arch/ia64/vmx/vmx_hypercall.c
index 97cc1bffb6a463ae21bb8df6c740007d2fc3ba92..1f4799956f831e68172a74c0c92ff18d088fe0dd 100644 (file)
--- a/xen/arch/ia64/vmx/vmx_hypercall.c
+++ b/xen/arch/ia64/vmx/vmx_hypercall.c
@@ -204,6 +204,53 @@ do_hvm_op(unsigned long op, XEN_GUEST_HANDLE(void) arg)
         rc = -ENOSYS;
         break;
 
+    case HVMOP_modified_memory:
+    {
+        struct xen_hvm_modified_memory a;
+        struct domain *d;
+        unsigned long pfn;
+
+        if ( copy_from_guest(&a, arg, 1) )
+            return -EFAULT;
+
+        if ( a.domid == DOMID_SELF )
+        {
+            d = rcu_lock_current_domain();
+        }
+        else
+        {
+            if ( (d = rcu_lock_domain_by_id(a.domid)) == NULL )
+                return -ESRCH;
+            if ( !IS_PRIV_FOR(current->domain, d) )
+            {
+                rc = -EPERM;
+                goto param_fail3;
+            }
+        }
+
+        rc = -EINVAL;
+        if ( !is_hvm_domain(d) )
+            goto param_fail3;
+
+        rc = -EINVAL;
+        if ( a.first_pfn > domain_get_maximum_gpfn(d)
+                || a.first_pfn + a.nr - 1 < a.first_pfn
+                || a.first_pfn + a.nr - 1 > domain_get_maximum_gpfn(d))
+            goto param_fail3;
+
+        rc = 0;
+        if ( !d->arch.shadow_bitmap )
+            goto param_fail3;
+
+        for (pfn = a.first_pfn; pfn < a.first_pfn + a.nr; pfn++)
+            if (pfn < d->arch.shadow_bitmap_size)
+                set_bit(pfn, d->arch.shadow_bitmap);
+
+    param_fail3:
+        rcu_unlock_domain(d);
+        break;
+    }
+
     default:
         gdprintk(XENLOG_INFO, "Bad HVM op %ld.\n", op);
         rc = -ENOSYS;

diff --git a/xen/arch/x86/hvm/hvm.c b/xen/arch/x86/hvm/hvm.c
index 0c4caa8e52b5d280b5c87c58580e21c187deec19..3a060d7a7595a6bb868b66abb07390da8a2c697a 100644 (file)
--- a/xen/arch/x86/hvm/hvm.c
+++ b/xen/arch/x86/hvm/hvm.c
@@ -2529,6 +2529,66 @@ long do_hvm_op(unsigned long op, XEN_GUEST_HANDLE(void) arg)
         break;
     }
 
+    case HVMOP_modified_memory:
+    {
+        struct xen_hvm_modified_memory a;
+        struct domain *d;
+        unsigned long pfn;
+
+        if ( copy_from_guest(&a, arg, 1) )
+            return -EFAULT;
+
+        if ( a.domid == DOMID_SELF )
+        {
+            d = rcu_lock_current_domain();
+        }
+        else
+        {
+            if ( (d = rcu_lock_domain_by_id(a.domid)) == NULL )
+                return -ESRCH;
+            if ( !IS_PRIV_FOR(current->domain, d) )
+            {
+                rc = -EPERM;
+                goto param_fail3;
+            }
+        }
+
+        rc = -EINVAL;
+        if ( !is_hvm_domain(d) )
+            goto param_fail3;
+
+        rc = xsm_hvm_param(d, op);
+        if ( rc )
+            goto param_fail3;
+
+        rc = -EINVAL;
+        if ( (a.first_pfn > domain_get_maximum_gpfn(d)) ||
+             ((a.first_pfn + a.nr - 1) < a.first_pfn) ||
+             ((a.first_pfn + a.nr - 1) > domain_get_maximum_gpfn(d)) )
+            goto param_fail3;
+
+        rc = 0;
+        if ( !paging_mode_log_dirty(d) )
+            goto param_fail3;
+
+        for ( pfn = a.first_pfn; pfn < a.first_pfn + a.nr; pfn++ )
+        {
+            p2m_type_t t;
+            mfn_t mfn = gfn_to_mfn(d, pfn, &t);
+            if ( mfn_x(mfn) != INVALID_MFN )
+            {
+                paging_mark_dirty(d, mfn_x(mfn));
+                /* These are most probably not page tables any more */
+                /* don't take a long time and don't die either */
+                sh_remove_shadows(d->vcpu[0], mfn, 1, 0);
+            }
+        }
+
+    param_fail3:
+        rcu_unlock_domain(d);
+        break;
+    }
+
     default:
     {
         gdprintk(XENLOG_WARNING, "Bad HVM op %ld.\n", op);

diff --git a/xen/include/public/hvm/hvm_op.h b/xen/include/public/hvm/hvm_op.h
index 19f0cb707f6b6f923b64b75854350780317e1989..a06d9416830d63b06e79e1180ad5ec175c3485c9 100644 (file)
--- a/xen/include/public/hvm/hvm_op.h
+++ b/xen/include/public/hvm/hvm_op.h
@@ -92,6 +92,19 @@ struct xen_hvm_track_dirty_vram {
 typedef struct xen_hvm_track_dirty_vram xen_hvm_track_dirty_vram_t;
 DEFINE_XEN_GUEST_HANDLE(xen_hvm_track_dirty_vram_t);
 
+/* Notify that some pages got modified by the Device Model. */
+#define HVMOP_modified_memory    7
+struct xen_hvm_modified_memory {
+    /* Domain to be updated. */
+    domid_t  domid;
+    /* First pfn. */
+    uint64_aligned_t first_pfn;
+    /* Number of pages. */
+    uint64_aligned_t nr;
+};
+typedef struct xen_hvm_modified_memory xen_hvm_modified_memory_t;
+DEFINE_XEN_GUEST_HANDLE(xen_hvm_modified_memory_t);
+
 #endif /* defined(__XEN__) || defined(__XEN_TOOLS__) */
 
 #endif /* __XEN_PUBLIC_HVM_HVM_OP_H__ */