x2APIC: improve enabling logic
authorKeir Fraser <keir.fraser@citrix.com>
Mon, 5 Jul 2010 07:31:29 +0000 (08:31 +0100)
committerKeir Fraser <keir.fraser@citrix.com>
Mon, 5 Jul 2010 07:31:29 +0000 (08:31 +0100)
This patch masks PIC and IOAPIC RTE's before x2APIC enabling, unmask
and restore them after x2APIC enabling. It also really enables
interrupt remapping before x2APIC enabling instead of just checking
interrupt remapping setting. This patch also handles all x2APIC
configuration including BIOS settings and command line
settings. Especially, it handles that BIOS hands over in x2APIC mode
(when there is apic id > 255). It checks if x2APIC is already enabled
by BIOS. If already enabled, it will disable interrupt remapping and
queued invalidation first, then enable them again.

Signed-off-by: Weidong Han <weidong.han@intel.com>
13 files changed:
xen/arch/x86/apic.c
xen/arch/x86/genapic/x2apic.c
xen/arch/x86/i8259.c
xen/arch/x86/io_apic.c
xen/arch/x86/setup.c
xen/drivers/passthrough/vtd/extern.h
xen/drivers/passthrough/vtd/intremap.c
xen/drivers/passthrough/vtd/iommu.c
xen/include/asm-x86/apic.h
xen/include/asm-x86/genapic.h
xen/include/asm-x86/io_apic.h
xen/include/asm-x86/irq.h
xen/include/xen/iommu.h

index 6a834b8292974aac4af76c1aee02e05d3f417534..844f46ff01330d177b90fffa4294d16fcd122bf0 100644 (file)
@@ -70,6 +70,9 @@ int apic_verbosity;
 int x2apic_enabled __read_mostly = 0;
 int directed_eoi_enabled __read_mostly = 0;
 
+/* x2APIC is enabled in BIOS */
+static int x2apic_preenabled;
+
 /*
  * The following vectors are part of the Linux architecture, there
  * is no hardware IRQ pin equivalent for them, they are triggered
@@ -489,6 +492,47 @@ static void apic_pm_activate(void)
     apic_pm_state.active = 1;
 }
 
+static void resume_x2apic(void)
+{
+    uint64_t msr_content;
+    struct IO_APIC_route_entry **ioapic_entries = NULL;
+
+    ASSERT(x2apic_enabled);
+
+    ioapic_entries = alloc_ioapic_entries();
+    if ( !ioapic_entries )
+    {
+        printk("Allocate ioapic_entries failed\n");
+        goto out;
+    }
+
+    if ( save_IO_APIC_setup(ioapic_entries) )
+    {
+        printk("Saving IO-APIC state failed\n");
+        goto out;
+    }
+
+    mask_8259A();
+    mask_IO_APIC_setup(ioapic_entries);
+
+    iommu_enable_IR();
+
+    rdmsrl(MSR_IA32_APICBASE, msr_content);
+    if ( !(msr_content & MSR_IA32_APICBASE_EXTD) )
+    {
+        msr_content |= MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD;
+        msr_content = (uint32_t)msr_content;
+        wrmsrl(MSR_IA32_APICBASE, msr_content);
+    }
+
+    restore_IO_APIC_setup(ioapic_entries);
+    unmask_8259A();
+
+out:
+    if ( ioapic_entries )
+        free_ioapic_entries(ioapic_entries);
+}
+
 void __devinit setup_local_APIC(void)
 {
     unsigned long oldvalue, value, ver, maxlvt;
@@ -727,7 +771,7 @@ int lapic_resume(void)
             msr_content | MSR_IA32_APICBASE_ENABLE | mp_lapic_addr);
     }
     else
-        enable_x2apic();
+        resume_x2apic();
 
     apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
     apic_write(APIC_ID, apic_pm_state.apic_id);
@@ -894,46 +938,154 @@ no_apic:
     return -1;
 }
 
-void enable_x2apic(void)
+void check_x2apic_preenabled(void)
 {
     uint64_t msr_content;
 
-    if ( smp_processor_id() == 0 )
+    if ( !x2apic_is_available() )
+        return;
+
+    rdmsrl(MSR_IA32_APICBASE, msr_content);
+    if ( msr_content & MSR_IA32_APICBASE_EXTD )
+    {
+        printk("x2APIC mode is already enabled by BIOS.\n");
+        x2apic_preenabled = 1;
+        x2apic_enabled = 1;
+    }
+}
+
+static void enable_bsp_x2apic(void)
+{
+    struct IO_APIC_route_entry **ioapic_entries = NULL;
+    const struct genapic *x2apic_genapic = NULL;
+
+    ASSERT(smp_processor_id() == 0);
+
+    if ( x2apic_preenabled )
     {
-        if ( !iommu_supports_eim() )
+        /*
+         * Interrupt remapping should be also enabled by BIOS when
+         * x2APIC is already enabled by BIOS, otherwise it's a BIOS
+         * bug
+         */
+        if ( !intremap_enabled() )
+            panic("Interrupt remapping is not enabled by BIOS while "
+                  "x2APIC is already enabled by BIOS!\n");
+    }
+
+    x2apic_genapic = apic_x2apic_probe();
+    if ( x2apic_genapic )
+        genapic = x2apic_genapic;
+    else
+    {
+        if ( x2apic_cmdline_disable() )
         {
-            printk("x2APIC would not be enabled without EIM.\n");
-            return;
+            if ( x2apic_preenabled )
+            {
+                /* Ignore x2apic=0, and set default x2apic mode */
+                genapic = &apic_x2apic_cluster;
+                printk("x2APIC: already enabled by BIOS, ignore x2apic=0.\n");
+            }
+            else
+            {
+                printk("Not enable x2APIC due to x2apic=0 is set.\n");
+                return;
+            }
         }
-
-        if ( apic_x2apic_phys.probe() )
-            genapic = &apic_x2apic_phys;
-        else if ( apic_x2apic_cluster.probe() )
-            genapic = &apic_x2apic_cluster;
         else
         {
-            printk("x2APIC would not be enabled due to x2apic=off.\n");
-            return;
+            if ( !iommu_enabled || !iommu_intremap || !iommu_qinval )
+                panic("Cannot enable x2APIC due to iommu or interrupt "
+                      "remapping or queued invalidation is disabled "
+                      "by command line!\n");
+            else
+            {
+                if ( x2apic_preenabled )
+                    panic("x2APIC: already enabled by BIOS, but "
+                          "iommu_supports_eim fails\n");
+                else
+                {
+                    printk("Not enable x2APIC due to "
+                           "iommu_supports_eim fails!\n");
+                    return;
+                }
+            }
         }
+    }
 
-        x2apic_enabled = 1;
-        printk("Switched to APIC driver %s.\n", genapic->name);
+    ioapic_entries = alloc_ioapic_entries();
+    if ( !ioapic_entries )
+    {
+        printk("Allocate ioapic_entries failed\n");
+        goto out;
     }
-    else
+
+    if ( save_IO_APIC_setup(ioapic_entries) )
     {
-        BUG_ON(!x2apic_enabled); /* APs only enable x2apic when BSP did so. */
+        printk("Saving IO-APIC state failed\n");
+        goto out;
     }
 
+    mask_8259A();
+    mask_IO_APIC_setup(ioapic_entries);
+
+    if ( iommu_enable_IR() )
+    {
+        printk("Would not enable x2APIC due to interrupt remapping "
+               "cannot be enabled.\n");
+        goto restore_out;
+    }
+
+    x2apic_enabled = 1;
+    printk("Switched to APIC driver %s.\n", genapic->name);
+
+    if ( !x2apic_preenabled )
+    {
+        uint64_t msr_content;
+        rdmsrl(MSR_IA32_APICBASE, msr_content);
+        if ( !(msr_content & MSR_IA32_APICBASE_EXTD) )
+        {
+            msr_content |= MSR_IA32_APICBASE_ENABLE |
+                           MSR_IA32_APICBASE_EXTD;
+            msr_content = (uint32_t)msr_content;
+            wrmsrl(MSR_IA32_APICBASE, msr_content);
+            printk("x2APIC mode enabled.\n");
+        }
+    }
+
+restore_out:
+    restore_IO_APIC_setup(ioapic_entries);
+    unmask_8259A();
+
+out:
+    if ( ioapic_entries )
+        free_ioapic_entries(ioapic_entries);
+}
+
+static void enable_ap_x2apic(void)
+{
+    uint64_t msr_content;
+
+    ASSERT(smp_processor_id() != 0);
+
+    /* APs only enable x2apic when BSP did so. */
+    BUG_ON(!x2apic_enabled);
+
     rdmsrl(MSR_IA32_APICBASE, msr_content);
     if ( !(msr_content & MSR_IA32_APICBASE_EXTD) )
     {
         msr_content |= MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD;
         msr_content = (uint32_t)msr_content;
         wrmsrl(MSR_IA32_APICBASE, msr_content);
-        printk("x2APIC mode enabled.\n");
     }
+}
+
+void enable_x2apic(void)
+{
+    if ( smp_processor_id() == 0 )
+        enable_bsp_x2apic();
     else
-        printk("x2APIC mode enabled by BIOS.\n");
+        enable_ap_x2apic();
 }
 
 void __init init_apic_mappings(void)
index 8c8d52ab013daa6df9e07b63cd7563c451952223..ae61567f59dd592d12850b0856127c547095393d 100644 (file)
@@ -33,6 +33,11 @@ boolean_param("x2apic", x2apic);
 static int  x2apic_phys; /* By default we use logical cluster mode. */
 boolean_param("x2apic_phys", x2apic_phys);
 
+int x2apic_cmdline_disable(void)
+{
+    return (x2apic == 0);
+}
+
 static int probe_x2apic_phys(void)
 {
     return x2apic && x2apic_phys && x2apic_is_available() &&
@@ -55,6 +60,20 @@ const struct genapic apic_x2apic_cluster = {
     GENAPIC_X2APIC_CLUSTER
 };
 
+const struct genapic *apic_x2apic_probe(void)
+{
+    if ( !x2apic || !x2apic_is_available() )
+        return NULL;
+
+    if ( !iommu_supports_eim() )
+        return NULL;
+
+    if ( x2apic_phys )
+        return &apic_x2apic_phys;
+    else
+        return &apic_x2apic_cluster;
+}
+
 void init_apic_ldr_x2apic_phys(void)
 {
     return;
index 2474f7864d4426e1d8396e2892bd3d5fd3d42782..bd9c956ef27c99c94a8833d9e53eb39ad07f889b 100644 (file)
@@ -175,6 +175,26 @@ int i8259A_irq_pending(unsigned int irq)
     return ret;
 }
 
+void mask_8259A(void)
+{
+    unsigned long flags;
+
+    spin_lock_irqsave(&i8259A_lock, flags);
+    outb(0xff, 0xA1);
+    outb(0xff, 0x21);
+    spin_unlock_irqrestore(&i8259A_lock, flags);
+}
+
+void unmask_8259A(void)
+{
+    unsigned long flags;
+
+    spin_lock_irqsave(&i8259A_lock, flags);
+    outb(cached_A1, 0xA1);
+    outb(cached_21, 0x21);
+    spin_unlock_irqrestore(&i8259A_lock, flags);
+}
+
 /*
  * This function assumes to be called rarely. Switching between
  * 8259A registers is slow.
index 2c5f03d0d2c8e64b98357099b7b2166e801dad31..55d3da78fb1c4ab12ad43e4d04340b10d8e37556 100644 (file)
@@ -136,6 +136,126 @@ static void __init replace_pin_at_irq(unsigned int irq,
     }
 }
 
+struct IO_APIC_route_entry **alloc_ioapic_entries(void)
+{
+    int apic;
+    struct IO_APIC_route_entry **ioapic_entries;
+
+    ioapic_entries = xmalloc_array(struct IO_APIC_route_entry *, nr_ioapics);
+    if (!ioapic_entries)
+        return 0;
+
+    for (apic = 0; apic < nr_ioapics; apic++) {
+        ioapic_entries[apic] =
+            xmalloc_array(struct IO_APIC_route_entry,
+                          nr_ioapic_registers[apic]);
+        if (!ioapic_entries[apic])
+            goto nomem;
+    }
+
+    return ioapic_entries;
+
+nomem:
+    while (--apic >= 0)
+        xfree(ioapic_entries[apic]);
+    xfree(ioapic_entries);
+
+    return 0;
+}
+
+/*
+ * Saves all the IO-APIC RTE's
+ */
+int save_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
+{
+    int apic, pin;
+
+    if (!ioapic_entries)
+        return -ENOMEM;
+
+    for (apic = 0; apic < nr_ioapics; apic++) {
+        if (!ioapic_entries[apic])
+            return -ENOMEM;
+
+        for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
+            *(((int *)&ioapic_entries[apic][pin])+0) =
+                __io_apic_read(apic, 0x10+pin*2);
+            *(((int *)&ioapic_entries[apic][pin])+1) =
+                __io_apic_read(apic, 0x11+pin*2);
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Mask all IO APIC entries.
+ */
+void mask_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
+{
+    int apic, pin;
+
+    if (!ioapic_entries)
+        return;
+
+    for (apic = 0; apic < nr_ioapics; apic++) {
+        if (!ioapic_entries[apic])
+            break;
+
+        for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
+            struct IO_APIC_route_entry entry;
+            unsigned long flags;
+
+            entry = ioapic_entries[apic][pin];
+            if (!entry.mask) {
+                entry.mask = 1;
+
+                spin_lock_irqsave(&ioapic_lock, flags);
+                __io_apic_write(apic, 0x11+2*pin, *(((int *)&entry)+1));
+                __io_apic_write(apic, 0x10+2*pin, *(((int *)&entry)+0));
+                spin_unlock_irqrestore(&ioapic_lock, flags);
+            }
+        }
+    }
+}
+
+/*
+ * Restore IO APIC entries which was saved in ioapic_entries.
+ */
+int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
+{
+    int apic, pin;
+    unsigned long flags;
+    struct IO_APIC_route_entry entry;
+
+    if (!ioapic_entries)
+        return -ENOMEM;
+
+    for (apic = 0; apic < nr_ioapics; apic++) {
+        if (!ioapic_entries[apic])
+            return -ENOMEM;
+
+        for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
+            entry = ioapic_entries[apic][pin];
+            spin_lock_irqsave(&ioapic_lock, flags);
+            __io_apic_write(apic, 0x11+2*pin, *(((int *)&entry)+1));
+            __io_apic_write(apic, 0x10+2*pin, *(((int *)&entry)+0));
+            spin_unlock_irqrestore(&ioapic_lock, flags);
+    }
+
+    return 0;
+}
+
+void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries)
+{
+    int apic;
+
+    for (apic = 0; apic < nr_ioapics; apic++)
+        xfree(ioapic_entries[apic]);
+
+    xfree(ioapic_entries);
+}
+
 static void __modify_IO_APIC_irq (unsigned int irq, unsigned long enable, unsigned long disable)
 {
     struct irq_pin_list *entry = irq_2_pin + irq;
index b54ba7b894b26ecee4558b740ab4c7c0253c696d..47cbe15850df97b35e72821517b2e2aa9a5ed770 100644 (file)
@@ -909,6 +909,9 @@ void __init __start_xen(unsigned long mbi_p)
 
     tboot_probe();
 
+    /* Check if x2APIC is already enabled in BIOS */
+    check_x2apic_preenabled();
+
     /* Unmap the first page of CPU0's stack. */
     memguard_guard_stack(cpu0_stack);
 
@@ -927,9 +930,6 @@ void __init __start_xen(unsigned long mbi_p)
 
     acpi_boot_init();
 
-    if ( x2apic_is_available() )
-        enable_x2apic();
-
     init_cpu_to_node();
 
     if ( smp_found_config )
@@ -942,6 +942,9 @@ void __init __start_xen(unsigned long mbi_p)
 
     init_apic_mappings();
 
+    if ( x2apic_is_available() )
+        enable_x2apic();
+
     init_IRQ();
 
     xsm_init(&initrdidx, mbi, initial_images_start);
index 131e97b4359b34f9e0eb5e15565c95d851682ede..e3880cdde3e5f188a22be6992d6e2b5ec18de74a 100644 (file)
@@ -44,6 +44,7 @@ int queue_invalidate_iec(struct iommu *iommu,
 int invalidate_sync(struct iommu *iommu);
 int iommu_flush_iec_global(struct iommu *iommu);
 int iommu_flush_iec_index(struct iommu *iommu, u8 im, u16 iidx);
+void clear_fault_bits(struct iommu *iommu);
 struct iommu * ioapic_to_iommu(unsigned int apic_id);
 struct acpi_drhd_unit * ioapic_to_drhd(unsigned int apic_id);
 struct acpi_drhd_unit * iommu_to_drhd(struct iommu *iommu);
index 6552e5297fd70545d0f136e0f5f2767bba82ff0a..828aeb043aef19cbe0b8953227e9030d010b995f 100644 (file)
@@ -134,6 +134,12 @@ int iommu_supports_eim(void)
     if ( !iommu_enabled || !iommu_qinval || !iommu_intremap )
         return 0;
 
+    if ( list_empty(&acpi_drhd_units) )
+    {
+        dprintk(XENLOG_WARNING VTDPREFIX, "VT-d is not supported\n");
+        return 0;
+    }
+
     /* We MUST have a DRHD unit for each IOAPIC. */
     for ( apic = 0; apic < nr_ioapics; apic++ )
         if ( !ioapic_to_drhd(IO_APIC_ID(apic)) )
@@ -144,9 +150,6 @@ int iommu_supports_eim(void)
             return 0;
     }
 
-    if ( list_empty(&acpi_drhd_units) )
-        return 0;
-
     for_each_drhd_unit ( drhd )
         if ( !ecap_queued_inval(drhd->iommu->ecap) ||
              !ecap_intr_remap(drhd->iommu->ecap) ||
@@ -804,3 +807,80 @@ void disable_intremap(struct iommu *iommu)
 out:
     spin_unlock_irqrestore(&iommu->register_lock, flags);
 }
+
+/*
+ * This function is used to enable Interrutp remapping when
+ * enable x2apic
+ */
+int iommu_enable_IR(void)
+{
+    struct acpi_drhd_unit *drhd;
+    struct iommu *iommu;
+
+    if ( !iommu_supports_eim() )
+        return -1;
+
+    for_each_drhd_unit ( drhd )
+    {
+        struct qi_ctrl *qi_ctrl = NULL;
+
+        iommu = drhd->iommu;
+        qi_ctrl = iommu_qi_ctrl(iommu);
+
+        /* Clear previous faults */
+        clear_fault_bits(iommu);
+
+        /*
+         * Disable interrupt remapping and queued invalidation if
+         * already enabled by BIOS
+         */
+        disable_intremap(iommu);
+        disable_qinval(iommu);
+    }
+
+    /* Enable queue invalidation */
+    for_each_drhd_unit ( drhd )
+    {
+        iommu = drhd->iommu;
+        if ( enable_qinval(iommu) != 0 )
+        {
+            dprintk(XENLOG_INFO VTDPREFIX,
+                    "Failed to enable Queued Invalidation!\n");
+            return -1;
+        }
+    }
+
+    /* Enable interrupt remapping */
+    for_each_drhd_unit ( drhd )
+    {
+        iommu = drhd->iommu;
+        if ( enable_intremap(iommu, 1) )
+        {
+            dprintk(XENLOG_INFO VTDPREFIX,
+                    "Failed to enable Interrupt Remapping!\n");
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Check if interrupt remapping is enabled or not
+ * return 1: enabled
+ * return 0: not enabled
+ */
+int intremap_enabled(void)
+{
+    struct acpi_drhd_unit *drhd;
+    u32 sts;
+
+    for_each_drhd_unit ( drhd )
+    {
+        sts = dmar_readl(drhd->iommu->reg, DMAR_GSTS_REG);
+        if ( !(sts & DMA_GSTS_IRES) )
+            return 0;
+    }
+
+    return 1;
+}
index c04d429939c85715bb03b0e7ffe4599928d22530..b8684141bbcc5e0b780975bab6f05e861035b6ee 100644 (file)
@@ -1790,7 +1790,7 @@ static void setup_dom0_devices(struct domain *d)
     spin_unlock(&pcidevs_lock);
 }
 
-static void clear_fault_bits(struct iommu *iommu)
+void clear_fault_bits(struct iommu *iommu)
 {
     u64 val;
     unsigned long flags;
index b02e7b38b9c7d5984a263ab1dfe87ca2d1e732cf..e867464332a1f260884c900b2194dd3efd70c8cb 100644 (file)
@@ -25,6 +25,8 @@ extern int apic_verbosity;
 extern int x2apic_enabled;
 extern int directed_eoi_enabled;
 
+extern void check_x2apic_preenabled(void);
+extern int x2apic_cmdline_disable(void);
 extern void enable_x2apic(void);
 
 static __inline int x2apic_is_available(void)
index ef3c35fd51cacdef2215774cbbdc0a464081b482..bcd03aed84308fbd37c7a2b3b47147951accff9f 100644 (file)
@@ -70,6 +70,7 @@ cpumask_t vector_allocation_domain_flat(int cpu);
        .send_IPI_mask = send_IPI_mask_flat, \
        .send_IPI_self = send_IPI_self_flat
 
+const struct genapic *apic_x2apic_probe(void);
 void init_apic_ldr_x2apic_phys(void);
 void init_apic_ldr_x2apic_cluster(void);
 void clustered_apic_check_x2apic(void);
index d546380435786d5b45697a41fa8b50ec599fbb54..7d577f39d6f5939340eab282b5c04cc49324566a 100644 (file)
@@ -199,6 +199,12 @@ extern int (*ioapic_renumber_irq)(int ioapic, int irq);
 extern void ioapic_suspend(void);
 extern void ioapic_resume(void);
 
+extern struct IO_APIC_route_entry **alloc_ioapic_entries(void);
+extern void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries);
+extern int save_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
+extern void mask_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
+extern int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
+
 #else  /* !CONFIG_X86_IO_APIC */
 static inline void init_ioapic_mappings(void) {}
 static inline void ioapic_suspend(void) {}
index 1f0487b2436e4c6e5bf3b685cfdcce74427e4b00..7f596ce5a90e907c8b22db3b9e3002e08e5c700f 100644 (file)
@@ -91,6 +91,8 @@ asmlinkage void do_IRQ(struct cpu_user_regs *regs);
 void disable_8259A_irq(unsigned int irq);
 void enable_8259A_irq(unsigned int irq);
 int i8259A_irq_pending(unsigned int irq);
+void mask_8259A(void);
+void unmask_8259A(void);
 void init_8259A(int aeoi);
 int i8259A_suspend(void);
 int i8259A_resume(void);
index 49f9a34eda82448c9b8d4d471fa60d74b6342e37..1b16f7f392885cedd0b121ae197eabd3ad1baa1c 100644 (file)
@@ -58,6 +58,8 @@ struct iommu {
 
 int iommu_setup(void);
 int iommu_supports_eim(void);
+int iommu_enable_IR(void);
+int intremap_enabled(void);
 
 int iommu_add_device(struct pci_dev *pdev);
 int iommu_remove_device(struct pci_dev *pdev);