Handle IOMMU device assignment for PV guests

Added more informative error codes for device assignment domctls.
Introduced python bindings for device assignment and enable xend to
also perform IOMMU device assignment when assigning PCI devices.

Signed-off-by: Espen Skoglund <espen.skoglund@netronome.com>

diff --git a/tools/python/xen/lowlevel/xc/xc.c b/tools/python/xen/lowlevel/xc/xc.c
index cd86022945e0ef4b5e70a4ee92c4adedff1eed63..90d541eca4c6c7983ff4982f67ca1bf5ba85274e 100644 (file)
--- a/tools/python/xen/lowlevel/xc/xc.c
+++ b/tools/python/xen/lowlevel/xc/xc.c
@@ -106,7 +106,7 @@ static PyObject *pyxc_domain_create(XcObject *self,
     static char *kwd_list[] = { "domid", "ssidref", "handle", "flags", "target", NULL };
 
     if ( !PyArg_ParseTupleAndKeywords(args, kwds, "|iiOii", kwd_list,
-                                     &dom, &ssidref, &pyhandle, &flags, &target))
+                                      &dom, &ssidref, &pyhandle, &flags, &target))
         return NULL;
     if ( pyhandle != NULL )
     {
@@ -434,44 +434,44 @@ static PyObject *pyxc_linux_build(XcObject *self,
     dom->vhpt_size_log2 = vhpt;
 
     if ( xc_dom_linux_build(self->xc_handle, dom, domid, mem_mb, image,
-                           ramdisk, flags, store_evtchn, &store_mfn,
-                           console_evtchn, &console_mfn) != 0 ) {
-       goto out;
+                            ramdisk, flags, store_evtchn, &store_mfn,
+                            console_evtchn, &console_mfn) != 0 ) {
+        goto out;
     }
 
     if ( !(elfnote_dict = PyDict_New()) )
-       goto out;
+        goto out;
     
     for ( i = 0; i < ARRAY_SIZE(dom->parms.elf_notes); i++ )
     {
-       switch ( dom->parms.elf_notes[i].type )
+        switch ( dom->parms.elf_notes[i].type )
         {
-       case XEN_ENT_NONE:
-           continue;
-       case XEN_ENT_LONG:
-           elfnote = Py_BuildValue("k", dom->parms.elf_notes[i].data.num);
-           break;
-       case XEN_ENT_STR:
-           elfnote = Py_BuildValue("s", dom->parms.elf_notes[i].data.str);
-           break;
-       }
-       PyDict_SetItemString(elfnote_dict,
-                            dom->parms.elf_notes[i].name,
-                            elfnote);
-       Py_DECREF(elfnote);
+        case XEN_ENT_NONE:
+            continue;
+        case XEN_ENT_LONG:
+            elfnote = Py_BuildValue("k", dom->parms.elf_notes[i].data.num);
+            break;
+        case XEN_ENT_STR:
+            elfnote = Py_BuildValue("s", dom->parms.elf_notes[i].data.str);
+            break;
+        }
+        PyDict_SetItemString(elfnote_dict,
+                             dom->parms.elf_notes[i].name,
+                             elfnote);
+        Py_DECREF(elfnote);
     }
 
     ret = Py_BuildValue("{s:i,s:i,s:N}",
-                       "store_mfn", store_mfn,
-                       "console_mfn", console_mfn,
-                       "notes", elfnote_dict);
+                        "store_mfn", store_mfn,
+                        "console_mfn", console_mfn,
+                        "notes", elfnote_dict);
 
     if ( dom->arch_hooks->native_protocol )
     {
-       PyObject *native_protocol =
-           Py_BuildValue("s", dom->arch_hooks->native_protocol);
-       PyDict_SetItemString(ret, "native_protocol", native_protocol);
-       Py_DECREF(native_protocol);
+        PyObject *native_protocol =
+            Py_BuildValue("s", dom->arch_hooks->native_protocol);
+        PyDict_SetItemString(ret, "native_protocol", native_protocol);
+        Py_DECREF(native_protocol);
     }
 
     xc_dom_release(dom);
@@ -556,7 +556,7 @@ static PyObject *pyxc_test_assign_device(XcObject *self,
 {
     uint32_t dom;
     char *pci_str;
-    uint32_t bdf = 0;
+    int32_t bdf = 0;
     int seg, bus, dev, func;
 
     static char *kwd_list[] = { "domid", "pci", NULL };
@@ -571,8 +571,75 @@ static PyObject *pyxc_test_assign_device(XcObject *self,
         bdf |= (func & 0x7) << 8;
 
         if ( xc_test_assign_device(self->xc_handle, dom, bdf) != 0 )
+        {
+            if (errno == ENOSYS)
+                bdf = -1;
+            break;
+        }
+        bdf = 0;
+    }
+
+    return Py_BuildValue("i", bdf);
+}
+
+static PyObject *pyxc_assign_device(XcObject *self,
+                                    PyObject *args,
+                                    PyObject *kwds)
+{
+    uint32_t dom;
+    char *pci_str;
+    int32_t bdf = 0;
+    int seg, bus, dev, func;
+
+    static char *kwd_list[] = { "domid", "pci", NULL };
+    if ( !PyArg_ParseTupleAndKeywords(args, kwds, "is", kwd_list,
+                                      &dom, &pci_str) )
+        return NULL;
+
+    while ( next_bdf(&pci_str, &seg, &bus, &dev, &func) )
+    {
+        bdf |= (bus & 0xff) << 16;
+        bdf |= (dev & 0x1f) << 11;
+        bdf |= (func & 0x7) << 8;
+
+        if ( xc_assign_device(self->xc_handle, dom, bdf) != 0 )
+        {
+            if (errno == ENOSYS)
+                bdf = -1;
             break;
+        }
+        bdf = 0;
+    }
 
+    return Py_BuildValue("i", bdf);
+}
+
+static PyObject *pyxc_deassign_device(XcObject *self,
+                                      PyObject *args,
+                                      PyObject *kwds)
+{
+    uint32_t dom;
+    char *pci_str;
+    int32_t bdf = 0;
+    int seg, bus, dev, func;
+
+    static char *kwd_list[] = { "domid", "pci", NULL };
+    if ( !PyArg_ParseTupleAndKeywords(args, kwds, "is", kwd_list,
+                                      &dom, &pci_str) )
+        return NULL;
+
+    while ( next_bdf(&pci_str, &seg, &bus, &dev, &func) )
+    {
+        bdf |= (bus & 0xff) << 16;
+        bdf |= (dev & 0x1f) << 11;
+        bdf |= (func & 0x7) << 8;
+
+        if ( xc_deassign_device(self->xc_handle, dom, bdf) != 0 )
+        {
+            if (errno == ENOSYS)
+                bdf = -1;
+            break;
+        }
         bdf = 0;
     }
 
@@ -729,8 +796,8 @@ static PyObject *pyxc_hvm_build(XcObject *self,
     int memsize, vcpus = 1, acpi = 0, apic = 1;
 
     static char *kwd_list[] = { "domid",
-                               "memsize", "image", "vcpus", "acpi",
-                               "apic", NULL };
+                                "memsize", "image", "vcpus", "acpi",
+                                "apic", NULL };
     if ( !PyArg_ParseTupleAndKeywords(args, kwds, "iis|iii", kwd_list,
                                       &dom, &memsize,
                                       &image, &vcpus, &acpi, &apic) )
@@ -782,8 +849,8 @@ static PyObject *pyxc_evtchn_alloc_unbound(XcObject *self,
 }
 
 static PyObject *pyxc_evtchn_reset(XcObject *self,
-                                  PyObject *args,
-                                  PyObject *kwds)
+                                   PyObject *args,
+                                   PyObject *kwds)
 {
     uint32_t dom;
 
@@ -947,11 +1014,11 @@ static PyObject *pyxc_physinfo(XcObject *self)
 
     for ( i = 0; i < info.nr_nodes; i++ )
     {
-       xc_availheap(self->xc_handle, 0, 0, i, &free_heap);
-       PyList_Append(node_to_memory_obj,
-           PyInt_FromLong(free_heap / 1024));
+        xc_availheap(self->xc_handle, 0, 0, i, &free_heap);
+        PyList_Append(node_to_memory_obj,
+                      PyInt_FromLong(free_heap / 1024));
     }
-       
+
     PyDict_SetItemString(ret_obj, "node_to_cpu", node_to_cpu_obj);
     PyDict_SetItemString(ret_obj, "node_to_memory", node_to_memory_obj);
  
@@ -1524,6 +1591,22 @@ static PyMethodDef pyxc_methods[] = {
        " dom     [int]:      Identifier of domain to build into.\n"
        " pci_str [str]:      PCI devices.\n"
        "Returns: [int] 0 on success, or device bdf that can't be assigned.\n" },
+
+     { "assign_device",
+       (PyCFunction)pyxc_assign_device,
+       METH_VARARGS | METH_KEYWORDS, "\n"
+       "Assign device to IOMMU domain.\n"
+       " dom     [int]:      Domain to assign device to.\n"
+       " pci_str [str]:      PCI devices.\n"
+       "Returns: [int] 0 on success, or device bdf that can't be assigned.\n" },
+
+     { "deassign_device",
+       (PyCFunction)pyxc_deassign_device,
+       METH_VARARGS | METH_KEYWORDS, "\n"
+       "Deassign device from IOMMU domain.\n"
+       " dom     [int]:      Domain to deassign device from.\n"
+       " pci_str [str]:      PCI devices.\n"
+       "Returns: [int] 0 on success, or device bdf that can't be deassigned.\n" },
   
     { "sched_id_get",
       (PyCFunction)pyxc_sched_id_get,

diff --git a/tools/python/xen/xend/server/pciif.py b/tools/python/xen/xend/server/pciif.py
index 3c62c4a29a287966121d05b5e4bf099b18aa92fc..3128adfbee52620f9438c4ccc1eb0ab1804d8029 100644 (file)
--- a/tools/python/xen/xend/server/pciif.py
+++ b/tools/python/xen/xend/server/pciif.py
@@ -248,6 +248,15 @@ class PciController(DevController):
         PCIQuirk(dev.vendor, dev.device, dev.subvendor, dev.subdevice, domain, 
                 bus, slot, func)
 
+        if not self.vm.info.is_hvm():
+            # Setup IOMMU device assignment
+            pci_str = "0x%x, 0x%x, 0x%x, 0x%x" % (domain, bus, slot, func)
+            bdf = xc.assign_device(fe_domid, pci_str)
+            if bdf > 0:
+                raise VmError("Failed to assign device to IOMMU (%x:%x.%x)"
+                              % (bus, slot, func))
+            log.debug("pci: assign device %x:%x.%x" % (bus, slot, func))
+
         for (start, size) in dev.ioports:
             log.debug('pci: enabling ioport 0x%x/0x%x'%(start,size))
             rc = xc.domain_ioport_permission(domid = fe_domid, first_port = start,
@@ -330,6 +339,14 @@ class PciController(DevController):
                     "bind your slot/device to the PCI backend using sysfs" \
                     )%(dev.name))
 
+        if not self.vm.info.is_hvm():
+            pci_str = "0x%x, 0x%x, 0x%x, 0x%x" % (domain, bus, slot, func)
+            bdf = xc.deassign_device(fe_domid, pci_str)
+            if bdf > 0:
+                raise VmError("Failed to deassign device from IOMMU (%x:%x.%x)"
+                              % (bus, slot, func))
+            log.debug("pci: deassign device %x:%x.%x" % (bus, slot, func))
+
         for (start, size) in dev.ioports:
             log.debug('pci: disabling ioport 0x%x/0x%x'%(start,size))
             rc = xc.domain_ioport_permission(domid = fe_domid, first_port = start,

diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c
index 3e1386b075b7215de51d7e238a4b46d1d3e2635b..c632b7015e34aac78e689070ac86fc01a7534fcb 100644 (file)
--- a/xen/arch/x86/domctl.c
+++ b/xen/arch/x86/domctl.c
@@ -530,10 +530,11 @@ long arch_do_domctl(
     {
         u8 bus, devfn;
 
-        ret = -EINVAL;
+        ret = -ENOSYS;
         if ( !iommu_enabled )
             break;
 
+        ret = -EINVAL;
         bus = (domctl->u.assign_device.machine_bdf >> 16) & 0xff;
         devfn = (domctl->u.assign_device.machine_bdf >> 8) & 0xff;
 
@@ -553,10 +554,11 @@ long arch_do_domctl(
         struct domain *d;
         u8 bus, devfn;
 
-        ret = -EINVAL;
+        ret = -ENOSYS;
         if ( !iommu_enabled )
             break;
 
+        ret = -EINVAL;
         if ( unlikely((d = get_domain_by_id(domctl->domain)) == NULL) )
         {
             gdprintk(XENLOG_ERR,
@@ -566,6 +568,12 @@ long arch_do_domctl(
         bus = (domctl->u.assign_device.machine_bdf >> 16) & 0xff;
         devfn = (domctl->u.assign_device.machine_bdf >> 8) & 0xff;
 
+        if ( !iommu_pv_enabled && !is_hvm_domain(d) )
+        {
+            ret = -ENOSYS;
+            break;
+        }
+
         if ( device_assigned(bus, devfn) )
         {
             gdprintk(XENLOG_ERR, "XEN_DOMCTL_assign_device: "
@@ -576,7 +584,7 @@ long arch_do_domctl(
 
         ret = assign_device(d, bus, devfn);
         gdprintk(XENLOG_INFO, "XEN_DOMCTL_assign_device: bdf = %x:%x:%x\n",
-            bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
+                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
         put_domain(d);
     }
     break;
@@ -586,10 +594,11 @@ long arch_do_domctl(
         struct domain *d;
         u8 bus, devfn;
 
-        ret = -EINVAL;
+        ret = -ENOSYS;
         if ( !iommu_enabled )
             break;
 
+        ret = -EINVAL;
         if ( unlikely((d = get_domain_by_id(domctl->domain)) == NULL) )
         {
             gdprintk(XENLOG_ERR,
@@ -599,9 +608,16 @@ long arch_do_domctl(
         bus = (domctl->u.assign_device.machine_bdf >> 16) & 0xff;
         devfn = (domctl->u.assign_device.machine_bdf >> 8) & 0xff;
 
+        if ( !iommu_pv_enabled && !is_hvm_domain(d) )
+        {
+            ret = -ENOSYS;
+            break;
+        }
+
         if ( !device_assigned(bus, devfn) )
             break;
 
+        ret = 0;
         deassign_device(d, bus, devfn);
         gdprintk(XENLOG_INFO, "XEN_DOMCTL_deassign_device: bdf = %x:%x:%x\n",
             bus, PCI_SLOT(devfn), PCI_FUNC(devfn));