libxl: Check whether a PCI device is assignable before assigning it do a domU

Implement a new libxl function libxl_device_pci_list_assignable. This is
used to implement the xl list-assignable-pci-devices command and part of
the implementation is used to make sure that PCI devices are not multiply
assigned to one or more domU's before doing the passthrough assignment.

The function libxl_device_pci_list changes to libxl_device_pci_list_assigned
due to a parameter change for consistency with pci_list_assignable.

Signed-off-by: Gianni Tedesco <gianni.tedesco@citrix.com>
Acked-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Committed-by: Ian Jackson <ian.jackson@eu.citrix.com>

diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 191d1e1bf356424e8be27e2d86878bb5fbe1caac..d9eeccf17bc5328f4dcc982fbf9d8a9496975643 100644 (file)
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -521,7 +521,8 @@ int libxl_device_vfb_hard_shutdown(libxl_ctx *ctx, uint32_t domid);
 int libxl_device_pci_add(libxl_ctx *ctx, uint32_t domid, libxl_device_pci *pcidev);
 int libxl_device_pci_remove(libxl_ctx *ctx, uint32_t domid, libxl_device_pci *pcidev);
 int libxl_device_pci_shutdown(libxl_ctx *ctx, uint32_t domid);
-libxl_device_pci *libxl_device_pci_list(libxl_ctx *ctx, uint32_t domid, int *num);
+int libxl_device_pci_list_assigned(libxl_ctx *ctx, libxl_device_pci **list, uint32_t domid, int *num);
+int libxl_device_pci_list_assignable(libxl_ctx *ctx, libxl_device_pci **list, int *num);
 int libxl_device_pci_init(libxl_device_pci *pcidev, unsigned int domain,
                           unsigned int bus, unsigned int dev,
                           unsigned int func, unsigned int vdevfn);

diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index a8adb92ec1342a1420aa3ed9a91590f111b79fce..36a2829f97e671e16b04acfe8f429350e41c8bdb 100644 (file)
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -91,6 +91,8 @@ typedef struct {
 #define XC_PCI_BDF             "0x%x, 0x%x, 0x%x, 0x%x"
 #define AUTO_PHP_SLOT          0x100
 #define SYSFS_PCI_DEV          "/sys/bus/pci/devices"
+#define SYSFS_PCIBACK_DRIVER   "/sys/bus/pci/drivers/pciback"
+
 #define PROC_PCI_NUM_RESOURCES 7
 #define PCI_BAR_IO             0x01
 

diff --git a/tools/libxl/libxl_pci.c b/tools/libxl/libxl_pci.c
index 3b2af3709e1cd94ca9a01e66c27e757fe48aa0e8..a4ae74bae0595d9c26917160ee5170fe2f9d5b1b 100644 (file)
--- a/tools/libxl/libxl_pci.c
+++ b/tools/libxl/libxl_pci.c
@@ -28,6 +28,7 @@
 #include <unistd.h> /* for write, unlink and close */
 #include <stdint.h>
 #include <inttypes.h>
+#include <dirent.h>
 #include <assert.h>
 
 #include "libxl.h"
@@ -258,23 +259,78 @@ retry_transaction2:
     return 0;
 }
 
-int libxl_device_pci_add(libxl_ctx *ctx, uint32_t domid, libxl_device_pci *pcidev)
+static int get_all_assigned_devices(libxl_ctx *ctx, libxl_device_pci **list, int *num)
 {
-    char *path;
-    char *state, *vdevfn;
-    int rc, hvm;
-    int stubdomid = 0;
+    libxl_device_pci *pcidevs = NULL;
+    char **domlist;
+    unsigned int nd = 0, i;
+
+    *list = NULL;
+    *num = 0;
+
+    domlist = libxl_xs_directory(ctx, XBT_NULL, "/local/domain", &nd);
+    for(i = 0; i < nd; i++) {
+        char *path, *num_devs;
+
+        path = libxl_sprintf(ctx, "/local/domain/0/backend/pci/%s/0/num_devs", domlist[i]);
+        num_devs = libxl_xs_read(ctx, XBT_NULL, path);
+        if ( num_devs ) {
+            int ndev = atoi(num_devs), j;
+            char *devpath, *bdf;
+
+            pcidevs = calloc(sizeof(*pcidevs), ndev);
+            for(j = (pcidevs) ? 0 : ndev; j < ndev; j++) {
+                devpath = libxl_sprintf(ctx, "/local/domain/0/backend/pci/%s/0/dev-%u",
+                                        domlist[i], j);
+                bdf = libxl_xs_read(ctx, XBT_NULL, devpath);
+                if ( bdf ) {
+                    unsigned dom, bus, dev, func;
+                    if ( sscanf(bdf, PCI_BDF, &dom, &bus, &dev, &func) != 4 )
+                        continue;
+
+                    libxl_device_pci_init(pcidevs + *num, dom, bus, dev, func, 0);
+                    (*num)++;
+                }
+            }
+        }
+    }
 
-    /* TODO: check if the device can be assigned */
+    if ( 0 == *num ) {
+        free(pcidevs);
+        pcidevs = NULL;
+    }else{
+        *list = pcidevs;
+    }
 
-    libxl_device_pci_reset(ctx, pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func);
+    return 0;
+}
 
-    stubdomid = libxl_get_stubdom_id(ctx, domid);
-    if (stubdomid != 0) {
-        libxl_device_pci pcidev_s = *pcidev;
-        libxl_device_pci_add(ctx, stubdomid, &pcidev_s);
+static int is_assigned(libxl_device_pci *assigned, int num_assigned,
+                       int dom, int bus, int dev, int func)
+{
+    int i;
+
+    for(i = 0; i < num_assigned; i++) {
+        if ( assigned[i].domain != dom )
+            continue;
+        if ( assigned[i].bus != bus )
+            continue;
+        if ( assigned[i].dev != dev )
+            continue;
+        if ( assigned[i].func != func )
+            continue;
+        return 1;
     }
 
+    return 0;
+}
+
+static int do_pci_add(libxl_ctx *ctx, uint32_t domid, libxl_device_pci *pcidev)
+{
+    char *path;
+    char *state, *vdevfn;
+    int rc, hvm;
+
     hvm = is_hvm(ctx, domid);
     if (hvm) {
         if (libxl_wait_for_device_model(ctx, domid, "running", NULL, NULL) < 0) {
@@ -370,6 +426,38 @@ out:
     return 0;
 }
 
+int libxl_device_pci_add(libxl_ctx *ctx, uint32_t domid, libxl_device_pci *pcidev)
+{
+    libxl_device_pci *assigned;
+    int num_assigned, rc;
+    int stubdomid = 0;
+
+    rc = get_all_assigned_devices(ctx, &assigned, &num_assigned);
+    if ( rc ) {
+        XL_LOG(ctx, XL_LOG_ERROR, "cannot determine if device is assigned, refusing to continue");
+        return ERROR_FAIL;
+    }
+    if ( is_assigned(assigned, num_assigned, pcidev->domain,
+                     pcidev->bus, pcidev->dev, pcidev->func) ) {
+        XL_LOG(ctx, XL_LOG_ERROR, "PCI device already attached to a domain");
+        free(assigned);
+        return ERROR_FAIL;
+    }
+    free(assigned);
+
+    libxl_device_pci_reset(ctx, pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func);
+
+    stubdomid = libxl_get_stubdom_id(ctx, domid);
+    if (stubdomid != 0) {
+        libxl_device_pci pcidev_s = *pcidev;
+        rc = do_pci_add(ctx, stubdomid, &pcidev_s);
+        if ( rc )
+            return rc;
+    }
+
+    return do_pci_add(ctx, domid, pcidev);
+}
+
 int libxl_device_pci_remove(libxl_ctx *ctx, uint32_t domid, libxl_device_pci *pcidev)
 {
     char *path;
@@ -466,7 +554,64 @@ out:
     return 0;
 }
 
-libxl_device_pci *libxl_device_pci_list(libxl_ctx *ctx, uint32_t domid, int *num)
+static libxl_device_pci *scan_sys_pcidir(libxl_device_pci *assigned,
+                                         int num_assigned, const char *path, int *num)
+{
+    libxl_device_pci *pcidevs = NULL, *new;
+    struct dirent *de;
+    DIR *dir;
+
+    dir = opendir(path);
+    if ( NULL == dir )
+        return pcidevs;
+
+    while( (de = readdir(dir)) ) {
+        unsigned dom, bus, dev, func;
+        if ( sscanf(de->d_name, PCI_BDF, &dom, &bus, &dev, &func) != 4 )
+            continue;
+
+        if ( is_assigned(assigned, num_assigned, dom, bus, dev, func) )
+            continue;
+
+        new = realloc(pcidevs, ((*num) + 1) * sizeof(*new));
+        if ( NULL == new )
+            continue;
+
+        pcidevs = new;
+        new = pcidevs + *num;
+
+        memset(new, 0, sizeof(*new));
+        libxl_device_pci_init(new, dom, bus, dev, func, 0);
+        (*num)++;
+    }
+
+    closedir(dir);
+    return pcidevs;
+}
+
+int libxl_device_pci_list_assignable(libxl_ctx *ctx, libxl_device_pci **list, int *num)
+{
+    libxl_device_pci *pcidevs = NULL;
+    libxl_device_pci *assigned;
+    int num_assigned, rc;
+
+    *num = 0;
+    *list = NULL;
+
+    rc = get_all_assigned_devices(ctx, &assigned, &num_assigned);
+    if ( rc )
+        return rc;
+
+    pcidevs = scan_sys_pcidir(assigned, num_assigned,
+                              SYSFS_PCIBACK_DRIVER, num);
+
+    free(assigned);
+    if ( *num )
+        *list = pcidevs;
+    return 0;
+}
+
+int libxl_device_pci_list_assigned(libxl_ctx *ctx, libxl_device_pci **list, uint32_t domid, int *num)
 {
     char *be_path, *num_devs, *xsdev, *xsvdevfn, *xsopts;
     int n, i;
@@ -477,7 +622,8 @@ libxl_device_pci *libxl_device_pci_list(libxl_ctx *ctx, uint32_t domid, int *num
     num_devs = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/num_devs", be_path));
     if (!num_devs) {
         *num = 0;
-        return NULL;
+        *list = NULL;
+        return ERROR_FAIL;
     }
     n = atoi(num_devs);
     pcidevs = calloc(n, sizeof(libxl_device_pci));
@@ -507,15 +653,19 @@ libxl_device_pci *libxl_device_pci_list(libxl_ctx *ctx, uint32_t domid, int *num
             } while ((p = strtok_r(NULL, ",=", &saveptr)) != NULL);
         }
     }
-    return pcidevs;
+    if ( *num )
+        *list = pcidevs;
+    return 0;
 }
 
 int libxl_device_pci_shutdown(libxl_ctx *ctx, uint32_t domid)
 {
     libxl_device_pci *pcidevs;
-    int num, i;
+    int num, i, rc;
 
-    pcidevs = libxl_device_pci_list(ctx, domid, &num);
+    rc = libxl_device_pci_list_assigned(ctx, &pcidevs, domid, &num);
+    if ( rc )
+        return rc;
     for (i = 0; i < num; i++) {
         if (libxl_device_pci_remove(ctx, domid, pcidevs + i) < 0)
             return ERROR_FAIL;

diff --git a/tools/libxl/xl.h b/tools/libxl/xl.h
index 839191b4bab16ba069328b2339f3a61ce9da1a00..468b18bec2bd555c3031d3154cc83256ebaedcbf 100644 (file)
--- a/tools/libxl/xl.h
+++ b/tools/libxl/xl.h
@@ -32,6 +32,7 @@ int main_cd_insert(int argc, char **argv);
 int main_console(int argc, char **argv);
 int main_vncviewer(int argc, char **argv);
 int main_pcilist(int argc, char **argv);
+int main_pcilist_assignable(int argc, char **argv);
 int main_pcidetach(int argc, char **argv);
 int main_pciattach(int argc, char **argv);
 int main_restore(int argc, char **argv);

diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 6a1ae0a612549dc62727ba7eb7da040b0acc70d1..044ba4bc4802d5dbf22f7f84e6088160963a3581 100644 (file)
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -1923,6 +1923,39 @@ int main_vncviewer(int argc, char **argv)
     exit(0);
 }
 
+void pcilist_assignable(void)
+{
+    libxl_device_pci *pcidevs;
+    int num, i;
+
+    if ( libxl_device_pci_list_assignable(&ctx, &pcidevs, &num) )
+        return;
+    for (i = 0; i < num; i++) {
+        printf("%04x:%02x:%02x:%01x\n",
+                pcidevs[i].domain, pcidevs[i].bus, pcidevs[i].dev, pcidevs[i].func);
+    }
+    free(pcidevs);
+}
+
+int main_pcilist_assignable(int argc, char **argv)
+{
+    int opt;
+
+    while ((opt = getopt(argc, argv, "h")) != -1) {
+        switch (opt) {
+        case 'h':
+            help("pci-list-assignable-devices");
+            exit(0);
+        default:
+            fprintf(stderr, "option not supported\n");
+            break;
+        }
+    }
+
+    pcilist_assignable();
+    exit(0);
+}
+
 void pcilist(char *dom)
 {
     libxl_device_pci *pcidevs;
@@ -1930,8 +1963,7 @@ void pcilist(char *dom)
 
     find_domain(dom);
 
-    pcidevs = libxl_device_pci_list(&ctx, domid, &num);
-    if (!num)
+    if (libxl_device_pci_list_assigned(&ctx, &pcidevs, domid, &num))
         return;
     printf("VFn  domain bus  slot func\n");
     for (i = 0; i < num; i++) {

diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c
index b7065bb60c08ce5f2e209784e2ad13fe6cd04340..b08798c1f869ef41fb3662a062ee5771a1e819a1 100644 (file)
--- a/tools/libxl/xl_cmdtable.c
+++ b/tools/libxl/xl_cmdtable.c
@@ -69,6 +69,11 @@ struct cmd_spec cmd_table[] = {
       "List pass-through pci devices for a domain",
       "<Domain>",
     },
+    { "pci-list-assignable-devices",
+      &main_pcilist_assignable,
+      "List all the assignable pci devices",
+      "",
+    },
     { "pause",
       &main_pause,
       "Pause execution of a domain",