Make PCI device release function generic

Release all pci devices before doing iommu domain teardown.  Also
moved pdev_flr() into generic pci code.

Signed-off-by: Espen Skoglund <espen.skoglund@netronome.com>

diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index d6a79195914829fe5a17c1c18fb9bb8ccfb16ccb..38775ab97267e1de6446c2508f247b17310bf5fc 100644 (file)
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -30,6 +30,7 @@
 #include <xen/percpu.h>
 #include <xen/compat.h>
 #include <xen/acpi.h>
+#include <xen/pci.h>
 #include <asm/regs.h>
 #include <asm/mc146818rtc.h>
 #include <asm/system.h>
@@ -473,6 +474,7 @@ void arch_domain_destroy(struct domain *d)
     if ( is_hvm_domain(d) )
         hvm_domain_destroy(d);
 
+    pci_release_devices(d);
     if ( !is_idle_domain(d) )
         iommu_domain_destroy(d);
 

diff --git a/xen/drivers/passthrough/amd/pci_amd_iommu.c b/xen/drivers/passthrough/amd/pci_amd_iommu.c
index 0cc63ae6f63803788a325aa656906dad597fc1b1..d908063b26d30b998b67ba155a8a3cd051b376fb 100644 (file)
--- a/xen/drivers/passthrough/amd/pci_amd_iommu.c
+++ b/xen/drivers/passthrough/amd/pci_amd_iommu.c
@@ -485,8 +485,6 @@ static void amd_iommu_disable_domain_device(
     }
 }
 
-extern void pdev_flr(u8 bus, u8 devfn);
-
 static int reassign_device( struct domain *source, struct domain *target,
                             u8 bus, u8 devfn)
 {
@@ -498,6 +496,7 @@ static int reassign_device( struct domain *source, struct domain *target,
     if ( !pdev )
        return -ENODEV;
 
+    pdev_flr(pdev);
     bdf = (bus << 8) | devfn;
     /* supported device? */
     iommu = (bdf < ivrs_bdf_entries) ?
@@ -545,26 +544,9 @@ static int amd_iommu_assign_device(struct domain *d, u8 bus, u8 devfn)
             ivrs_mappings[req_id].read_permission);
     }
 
-    pdev_flr(bus, devfn);
     return reassign_device(dom0, d, bus, devfn);
 }
 
-static void release_domain_devices(struct domain *d)
-{
-    struct pci_dev *pdev;
-    u8 bus, devfn;
-
-    while ( (pdev = pci_lock_domain_pdev(d, -1, -1)) )
-    {
-        pdev_flr(pdev->bus, pdev->devfn);
-       bus = pdev->bus; devfn = pdev->devfn;
-       spin_unlock(&pdev->lock);
-        amd_iov_info("release domain %d devices %x:%x.%x\n", d->domain_id,
-                    bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
-        reassign_device(d, dom0, bus, devfn);
-    }
-}
-
 static void deallocate_next_page_table(void *table, unsigned long index,
                                        int level)
 {
@@ -618,13 +600,11 @@ static void deallocate_iommu_page_tables(struct domain *d)
 static void amd_iommu_domain_destroy(struct domain *d)
 {
     deallocate_iommu_page_tables(d);
-    release_domain_devices(d);
 }
 
 static int amd_iommu_return_device(
     struct domain *s, struct domain *t, u8 bus, u8 devfn)
 {
-    pdev_flr(bus, devfn);
     return reassign_device(s, t, bus, devfn);
 }
 

diff --git a/xen/drivers/passthrough/pci.c b/xen/drivers/passthrough/pci.c
index 364d2c5f42968b9d318d208cb75872e80940944e..2faba151fbbad96a965e50aa21c2024dff153bea 100644 (file)
--- a/xen/drivers/passthrough/pci.c
+++ b/xen/drivers/passthrough/pci.c
@@ -17,9 +17,11 @@
 
 #include <xen/sched.h>
 #include <xen/pci.h>
+#include <xen/pci_regs.h>
 #include <xen/list.h>
 #include <xen/prefetch.h>
 #include <xen/iommu.h>
+#include <xen/delay.h>
 #include <xen/keyhandler.h>
 
 
@@ -145,6 +147,85 @@ int pci_remove_device(u8 bus, u8 devfn)
     return ret;
 }
 
+void pci_release_devices(struct domain *d)
+{
+    struct pci_dev *pdev;
+    u8 bus, devfn;
+
+    while ( (pdev = pci_lock_domain_pdev(d, -1, -1)) )
+    {
+        pci_cleanup_msi(pdev);
+        bus = pdev->bus; devfn = pdev->devfn;
+        spin_unlock(&pdev->lock);
+        deassign_device(d, bus, devfn);
+    }
+}
+
+#define PCI_D3hot              (3)
+#define PCI_CONFIG_DWORD_SIZE   (64)
+#define PCI_EXP_DEVCAP_FLR      (1 << 28)
+#define PCI_EXP_DEVCTL_FLR      (1 << 15)
+
+void pdev_flr(struct pci_dev *pdev)
+{
+    u8 pos;
+    u32 dev_cap, dev_status, pm_ctl;
+    int flr = 0;
+    u8 dev = PCI_SLOT(pdev->devfn);
+    u8 func = PCI_FUNC(pdev->devfn);
+
+    pos = pci_find_cap_offset(pdev->bus, dev, func, PCI_CAP_ID_EXP);
+    if ( pos != 0 )
+    {
+        dev_cap = pci_conf_read32(pdev->bus, dev, func, pos + PCI_EXP_DEVCAP);
+        if ( dev_cap & PCI_EXP_DEVCAP_FLR )
+        {
+            pci_conf_write32(pdev->bus, dev, func,
+                             pos + PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_FLR);
+            do {
+                dev_status = pci_conf_read32(pdev->bus, dev, func,
+                                             pos + PCI_EXP_DEVSTA);
+            } while ( dev_status & PCI_EXP_DEVSTA_TRPND );
+
+            flr = 1;
+        }
+    }
+
+    /* If this device doesn't support function level reset,
+     * program device from D0 t0 D3hot, and then return to D0
+     * to implement function level reset
+     */
+    if ( flr == 0 )
+    {
+        pos = pci_find_cap_offset(pdev->bus, dev, func, PCI_CAP_ID_PM);
+        if ( pos != 0 )
+        {
+            int i;
+            u32 config[PCI_CONFIG_DWORD_SIZE];
+            for ( i = 0; i < PCI_CONFIG_DWORD_SIZE; i++ )
+                config[i] = pci_conf_read32(pdev->bus, dev, func, i*4);
+
+            /* Enter D3hot without soft reset */
+            pm_ctl = pci_conf_read32(pdev->bus, dev, func, pos + PCI_PM_CTRL);
+            pm_ctl |= PCI_PM_CTRL_NO_SOFT_RESET;
+            pm_ctl &= ~PCI_PM_CTRL_STATE_MASK;
+            pm_ctl |= PCI_D3hot;
+            pci_conf_write32(pdev->bus, dev, func, pos + PCI_PM_CTRL, pm_ctl);
+            mdelay(10);
+
+            /* From D3hot to D0 */
+            pci_conf_write32(pdev->bus, dev, func, pos + PCI_PM_CTRL, 0);
+            mdelay(10);
+
+            /* Write saved configurations to device */
+            for ( i = 0; i < PCI_CONFIG_DWORD_SIZE; i++ )
+                pci_conf_write32(pdev->bus, dev, func, i*4, config[i]);
+
+            flr = 1;
+        }
+    }
+}
+
 static void dump_pci_devices(unsigned char ch)
 {
     struct pci_dev *pdev;
@@ -174,3 +255,14 @@ static int __init setup_dump_pcidevs(void)
     return 0;
 }
 __initcall(setup_dump_pcidevs);
+
+
+
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */

diff --git a/xen/drivers/passthrough/vtd/extern.h b/xen/drivers/passthrough/vtd/extern.h
index 300cf646bc246363a52935624bf632f852aabb8a..0220ae97b306167472431fabcedc9bd9318ef504 100644 (file)
--- a/xen/drivers/passthrough/vtd/extern.h
+++ b/xen/drivers/passthrough/vtd/extern.h
@@ -28,7 +28,6 @@ extern struct ir_ctrl *ir_ctrl;
 
 void print_iommu_regs(struct acpi_drhd_unit *drhd);
 void print_vtd_entries(struct iommu *iommu, int bus, int devfn, u64 gmfn);
-void pdev_flr(u8 bus, u8 devfn);
 
 int qinval_setup(struct iommu *iommu);
 int intremap_setup(struct iommu *iommu);

diff --git a/xen/drivers/passthrough/vtd/iommu.c b/xen/drivers/passthrough/vtd/iommu.c
index aab25809b790b19187932608899f80318bb0ecd9..95b4b1388fb90533e4b322215c340715fa8cc60b 100644 (file)
--- a/xen/drivers/passthrough/vtd/iommu.c
+++ b/xen/drivers/passthrough/vtd/iommu.c
@@ -1366,7 +1366,7 @@ static int reassign_device_ownership(
     if ( !(pdev = pci_lock_domain_pdev(source, bus, devfn)) )
         return -ENODEV;
 
-    pdev_flr(bus, devfn);
+    pdev_flr(pdev);
     drhd = acpi_find_matched_drhd_unit(bus, devfn);
     pdev_iommu = drhd->iommu;
     domain_context_unmap(bus, devfn);
@@ -1397,28 +1397,6 @@ static int reassign_device_ownership(
     return ret;
 }
 
-void return_devices_to_dom0(struct domain *d)
-{
-    struct pci_dev *pdev;
-
-    while ( (pdev = pci_lock_domain_pdev(d, -1, -1)) )
-    {
-        pci_cleanup_msi(pdev);
-        spin_unlock(&pdev->lock);
-        reassign_device_ownership(d, dom0, pdev->bus, pdev->devfn);
-    }
-
-#ifdef VTD_DEBUG
-    read_lock(&pcidevs_lock);
-    for_each_pdev ( dom0, pdev )
-        dprintk(XENLOG_INFO VTDPREFIX,
-                "return_devices_to_dom0:%x: bdf = %x:%x:%x\n",
-                dom0->domain_id, pdev->bus,
-                PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
-    read_unlock(&pcidevs_lock);
-#endif
-}
-
 void iommu_domain_teardown(struct domain *d)
 {
     struct hvm_iommu *hd = domain_hvm_iommu(d);
@@ -1426,7 +1404,6 @@ void iommu_domain_teardown(struct domain *d)
     if ( list_empty(&acpi_drhd_units) )
         return;
 
-    return_devices_to_dom0(d);
     iommu_free_pagetable(hd->pgd_maddr, agaw_to_level(hd->agaw));
     hd->pgd_maddr = 0;
     iommu_domid_release(d);

diff --git a/xen/drivers/passthrough/vtd/utils.c b/xen/drivers/passthrough/vtd/utils.c
index e8d1dc4eb27f6d65258784a304ee06cc297aa7ba..9f26c1a08842d227740eac5047df0118ca05428e 100644 (file)
--- a/xen/drivers/passthrough/vtd/utils.c
+++ b/xen/drivers/passthrough/vtd/utils.c
@@ -94,71 +94,6 @@ void disable_pmr(struct iommu *iommu)
             "Disabled protected memory registers\n");
 }
 
-#define PCI_D3hot   (3)
-#define PCI_CONFIG_DWORD_SIZE   (64)
-#define PCI_EXP_DEVCAP_FLR      (1 << 28)
-#define PCI_EXP_DEVCTL_FLR      (1 << 15)
-
-void pdev_flr(u8 bus, u8 devfn)
-{
-    u8 pos;
-    u32 dev_cap, dev_status, pm_ctl;
-    int flr = 0;
-    u8 dev = PCI_SLOT(devfn);
-    u8 func = PCI_FUNC(devfn);
-
-    pos = pci_find_cap_offset(bus, dev, func, PCI_CAP_ID_EXP);
-    if ( pos != 0 )
-    {
-        dev_cap = pci_conf_read32(bus, dev, func, pos + PCI_EXP_DEVCAP);
-        if ( dev_cap & PCI_EXP_DEVCAP_FLR )
-        {
-            pci_conf_write32(bus, dev, func,
-                             pos + PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_FLR);
-            do {
-                dev_status = pci_conf_read32(bus, dev, func,
-                                             pos + PCI_EXP_DEVSTA);
-            } while ( dev_status & PCI_EXP_DEVSTA_TRPND );
-
-            flr = 1;
-        }
-    }
-
-    /* If this device doesn't support function level reset,
-     * program device from D0 t0 D3hot, and then return to D0
-     * to implement function level reset
-     */
-    if ( flr == 0 )
-    {
-        pos = pci_find_cap_offset(bus, dev, func, PCI_CAP_ID_PM);
-        if ( pos != 0 )
-        {
-            int i;
-            u32 config[PCI_CONFIG_DWORD_SIZE];
-            for ( i = 0; i < PCI_CONFIG_DWORD_SIZE; i++ )
-                config[i] = pci_conf_read32(bus, dev, func, i*4);
-
-            /* Enter D3hot without soft reset */
-            pm_ctl = pci_conf_read32(bus, dev, func, pos + PCI_PM_CTRL);
-            pm_ctl |= PCI_PM_CTRL_NO_SOFT_RESET;
-            pm_ctl &= ~PCI_PM_CTRL_STATE_MASK;
-            pm_ctl |= PCI_D3hot;
-            pci_conf_write32(bus, dev, func, pos + PCI_PM_CTRL, pm_ctl);
-            mdelay(10);
-
-            /* From D3hot to D0 */
-            pci_conf_write32(bus, dev, func, pos + PCI_PM_CTRL, 0);
-            mdelay(10);
-
-            /* Write saved configurations to device */
-            for ( i = 0; i < PCI_CONFIG_DWORD_SIZE; i++ )
-                pci_conf_write32(bus, dev, func, i*4, config[i]);
-
-            flr = 1;
-        }
-    }
-}
-
 void print_iommu_regs(struct acpi_drhd_unit *drhd)
 {
     struct iommu *iommu = drhd->iommu;

diff --git a/xen/include/xen/pci.h b/xen/include/xen/pci.h
index f60c5855d7a62d4e1317eb78e2fafd21d8c08d04..22bcae986cbb35fe2c1887a18a05f3ba8ea5f8de 100644 (file)
--- a/xen/include/xen/pci.h
+++ b/xen/include/xen/pci.h
@@ -56,6 +56,8 @@ void free_pdev(struct pci_dev *pdev);
 struct pci_dev *pci_lock_pdev(int bus, int devfn);
 struct pci_dev *pci_lock_domain_pdev(struct domain *d, int bus, int devfn);
 
+void pdev_flr(struct pci_dev *pdev);
+void pci_release_devices(struct domain *d);
 int pci_add_device(u8 bus, u8 devfn);
 int pci_remove_device(u8 bus, u8 devfn);