int x2apic_enabled __read_mostly = 0;
int directed_eoi_enabled __read_mostly = 0;
+/* x2APIC is enabled in BIOS */
+static int x2apic_preenabled;
+
/*
* The following vectors are part of the Linux architecture, there
* is no hardware IRQ pin equivalent for them, they are triggered
apic_pm_state.active = 1;
}
+static void resume_x2apic(void)
+{
+ uint64_t msr_content;
+ struct IO_APIC_route_entry **ioapic_entries = NULL;
+
+ ASSERT(x2apic_enabled);
+
+ ioapic_entries = alloc_ioapic_entries();
+ if ( !ioapic_entries )
+ {
+ printk("Allocate ioapic_entries failed\n");
+ goto out;
+ }
+
+ if ( save_IO_APIC_setup(ioapic_entries) )
+ {
+ printk("Saving IO-APIC state failed\n");
+ goto out;
+ }
+
+ mask_8259A();
+ mask_IO_APIC_setup(ioapic_entries);
+
+ iommu_enable_IR();
+
+ rdmsrl(MSR_IA32_APICBASE, msr_content);
+ if ( !(msr_content & MSR_IA32_APICBASE_EXTD) )
+ {
+ msr_content |= MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD;
+ msr_content = (uint32_t)msr_content;
+ wrmsrl(MSR_IA32_APICBASE, msr_content);
+ }
+
+ restore_IO_APIC_setup(ioapic_entries);
+ unmask_8259A();
+
+out:
+ if ( ioapic_entries )
+ free_ioapic_entries(ioapic_entries);
+}
+
void __devinit setup_local_APIC(void)
{
unsigned long oldvalue, value, ver, maxlvt;
msr_content | MSR_IA32_APICBASE_ENABLE | mp_lapic_addr);
}
else
- enable_x2apic();
+ resume_x2apic();
apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
apic_write(APIC_ID, apic_pm_state.apic_id);
return -1;
}
-void enable_x2apic(void)
+void check_x2apic_preenabled(void)
{
uint64_t msr_content;
- if ( smp_processor_id() == 0 )
+ if ( !x2apic_is_available() )
+ return;
+
+ rdmsrl(MSR_IA32_APICBASE, msr_content);
+ if ( msr_content & MSR_IA32_APICBASE_EXTD )
+ {
+ printk("x2APIC mode is already enabled by BIOS.\n");
+ x2apic_preenabled = 1;
+ x2apic_enabled = 1;
+ }
+}
+
+static void enable_bsp_x2apic(void)
+{
+ struct IO_APIC_route_entry **ioapic_entries = NULL;
+ const struct genapic *x2apic_genapic = NULL;
+
+ ASSERT(smp_processor_id() == 0);
+
+ if ( x2apic_preenabled )
{
- if ( !iommu_supports_eim() )
+ /*
+ * Interrupt remapping should be also enabled by BIOS when
+ * x2APIC is already enabled by BIOS, otherwise it's a BIOS
+ * bug
+ */
+ if ( !intremap_enabled() )
+ panic("Interrupt remapping is not enabled by BIOS while "
+ "x2APIC is already enabled by BIOS!\n");
+ }
+
+ x2apic_genapic = apic_x2apic_probe();
+ if ( x2apic_genapic )
+ genapic = x2apic_genapic;
+ else
+ {
+ if ( x2apic_cmdline_disable() )
{
- printk("x2APIC would not be enabled without EIM.\n");
- return;
+ if ( x2apic_preenabled )
+ {
+ /* Ignore x2apic=0, and set default x2apic mode */
+ genapic = &apic_x2apic_cluster;
+ printk("x2APIC: already enabled by BIOS, ignore x2apic=0.\n");
+ }
+ else
+ {
+ printk("Not enable x2APIC due to x2apic=0 is set.\n");
+ return;
+ }
}
-
- if ( apic_x2apic_phys.probe() )
- genapic = &apic_x2apic_phys;
- else if ( apic_x2apic_cluster.probe() )
- genapic = &apic_x2apic_cluster;
else
{
- printk("x2APIC would not be enabled due to x2apic=off.\n");
- return;
+ if ( !iommu_enabled || !iommu_intremap || !iommu_qinval )
+ panic("Cannot enable x2APIC due to iommu or interrupt "
+ "remapping or queued invalidation is disabled "
+ "by command line!\n");
+ else
+ {
+ if ( x2apic_preenabled )
+ panic("x2APIC: already enabled by BIOS, but "
+ "iommu_supports_eim fails\n");
+ else
+ {
+ printk("Not enable x2APIC due to "
+ "iommu_supports_eim fails!\n");
+ return;
+ }
+ }
}
+ }
- x2apic_enabled = 1;
- printk("Switched to APIC driver %s.\n", genapic->name);
+ ioapic_entries = alloc_ioapic_entries();
+ if ( !ioapic_entries )
+ {
+ printk("Allocate ioapic_entries failed\n");
+ goto out;
}
- else
+
+ if ( save_IO_APIC_setup(ioapic_entries) )
{
- BUG_ON(!x2apic_enabled); /* APs only enable x2apic when BSP did so. */
+ printk("Saving IO-APIC state failed\n");
+ goto out;
}
+ mask_8259A();
+ mask_IO_APIC_setup(ioapic_entries);
+
+ if ( iommu_enable_IR() )
+ {
+ printk("Would not enable x2APIC due to interrupt remapping "
+ "cannot be enabled.\n");
+ goto restore_out;
+ }
+
+ x2apic_enabled = 1;
+ printk("Switched to APIC driver %s.\n", genapic->name);
+
+ if ( !x2apic_preenabled )
+ {
+ uint64_t msr_content;
+ rdmsrl(MSR_IA32_APICBASE, msr_content);
+ if ( !(msr_content & MSR_IA32_APICBASE_EXTD) )
+ {
+ msr_content |= MSR_IA32_APICBASE_ENABLE |
+ MSR_IA32_APICBASE_EXTD;
+ msr_content = (uint32_t)msr_content;
+ wrmsrl(MSR_IA32_APICBASE, msr_content);
+ printk("x2APIC mode enabled.\n");
+ }
+ }
+
+restore_out:
+ restore_IO_APIC_setup(ioapic_entries);
+ unmask_8259A();
+
+out:
+ if ( ioapic_entries )
+ free_ioapic_entries(ioapic_entries);
+}
+
+static void enable_ap_x2apic(void)
+{
+ uint64_t msr_content;
+
+ ASSERT(smp_processor_id() != 0);
+
+ /* APs only enable x2apic when BSP did so. */
+ BUG_ON(!x2apic_enabled);
+
rdmsrl(MSR_IA32_APICBASE, msr_content);
if ( !(msr_content & MSR_IA32_APICBASE_EXTD) )
{
msr_content |= MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD;
msr_content = (uint32_t)msr_content;
wrmsrl(MSR_IA32_APICBASE, msr_content);
- printk("x2APIC mode enabled.\n");
}
+}
+
+void enable_x2apic(void)
+{
+ if ( smp_processor_id() == 0 )
+ enable_bsp_x2apic();
else
- printk("x2APIC mode enabled by BIOS.\n");
+ enable_ap_x2apic();
}
void __init init_apic_mappings(void)
static int x2apic_phys; /* By default we use logical cluster mode. */
boolean_param("x2apic_phys", x2apic_phys);
+int x2apic_cmdline_disable(void)
+{
+ return (x2apic == 0);
+}
+
static int probe_x2apic_phys(void)
{
return x2apic && x2apic_phys && x2apic_is_available() &&
GENAPIC_X2APIC_CLUSTER
};
+const struct genapic *apic_x2apic_probe(void)
+{
+ if ( !x2apic || !x2apic_is_available() )
+ return NULL;
+
+ if ( !iommu_supports_eim() )
+ return NULL;
+
+ if ( x2apic_phys )
+ return &apic_x2apic_phys;
+ else
+ return &apic_x2apic_cluster;
+}
+
void init_apic_ldr_x2apic_phys(void)
{
return;
return ret;
}
+void mask_8259A(void)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&i8259A_lock, flags);
+ outb(0xff, 0xA1);
+ outb(0xff, 0x21);
+ spin_unlock_irqrestore(&i8259A_lock, flags);
+}
+
+void unmask_8259A(void)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&i8259A_lock, flags);
+ outb(cached_A1, 0xA1);
+ outb(cached_21, 0x21);
+ spin_unlock_irqrestore(&i8259A_lock, flags);
+}
+
/*
* This function assumes to be called rarely. Switching between
* 8259A registers is slow.
}
}
+struct IO_APIC_route_entry **alloc_ioapic_entries(void)
+{
+ int apic;
+ struct IO_APIC_route_entry **ioapic_entries;
+
+ ioapic_entries = xmalloc_array(struct IO_APIC_route_entry *, nr_ioapics);
+ if (!ioapic_entries)
+ return 0;
+
+ for (apic = 0; apic < nr_ioapics; apic++) {
+ ioapic_entries[apic] =
+ xmalloc_array(struct IO_APIC_route_entry,
+ nr_ioapic_registers[apic]);
+ if (!ioapic_entries[apic])
+ goto nomem;
+ }
+
+ return ioapic_entries;
+
+nomem:
+ while (--apic >= 0)
+ xfree(ioapic_entries[apic]);
+ xfree(ioapic_entries);
+
+ return 0;
+}
+
+/*
+ * Saves all the IO-APIC RTE's
+ */
+int save_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
+{
+ int apic, pin;
+
+ if (!ioapic_entries)
+ return -ENOMEM;
+
+ for (apic = 0; apic < nr_ioapics; apic++) {
+ if (!ioapic_entries[apic])
+ return -ENOMEM;
+
+ for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
+ *(((int *)&ioapic_entries[apic][pin])+0) =
+ __io_apic_read(apic, 0x10+pin*2);
+ *(((int *)&ioapic_entries[apic][pin])+1) =
+ __io_apic_read(apic, 0x11+pin*2);
+ }
+ }
+
+ return 0;
+}
+
+/*
+ * Mask all IO APIC entries.
+ */
+void mask_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
+{
+ int apic, pin;
+
+ if (!ioapic_entries)
+ return;
+
+ for (apic = 0; apic < nr_ioapics; apic++) {
+ if (!ioapic_entries[apic])
+ break;
+
+ for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
+ struct IO_APIC_route_entry entry;
+ unsigned long flags;
+
+ entry = ioapic_entries[apic][pin];
+ if (!entry.mask) {
+ entry.mask = 1;
+
+ spin_lock_irqsave(&ioapic_lock, flags);
+ __io_apic_write(apic, 0x11+2*pin, *(((int *)&entry)+1));
+ __io_apic_write(apic, 0x10+2*pin, *(((int *)&entry)+0));
+ spin_unlock_irqrestore(&ioapic_lock, flags);
+ }
+ }
+ }
+}
+
+/*
+ * Restore IO APIC entries which was saved in ioapic_entries.
+ */
+int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
+{
+ int apic, pin;
+ unsigned long flags;
+ struct IO_APIC_route_entry entry;
+
+ if (!ioapic_entries)
+ return -ENOMEM;
+
+ for (apic = 0; apic < nr_ioapics; apic++) {
+ if (!ioapic_entries[apic])
+ return -ENOMEM;
+
+ for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
+ entry = ioapic_entries[apic][pin];
+ spin_lock_irqsave(&ioapic_lock, flags);
+ __io_apic_write(apic, 0x11+2*pin, *(((int *)&entry)+1));
+ __io_apic_write(apic, 0x10+2*pin, *(((int *)&entry)+0));
+ spin_unlock_irqrestore(&ioapic_lock, flags);
+ }
+
+ return 0;
+}
+
+void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries)
+{
+ int apic;
+
+ for (apic = 0; apic < nr_ioapics; apic++)
+ xfree(ioapic_entries[apic]);
+
+ xfree(ioapic_entries);
+}
+
static void __modify_IO_APIC_irq (unsigned int irq, unsigned long enable, unsigned long disable)
{
struct irq_pin_list *entry = irq_2_pin + irq;
tboot_probe();
+ /* Check if x2APIC is already enabled in BIOS */
+ check_x2apic_preenabled();
+
/* Unmap the first page of CPU0's stack. */
memguard_guard_stack(cpu0_stack);
acpi_boot_init();
- if ( x2apic_is_available() )
- enable_x2apic();
-
init_cpu_to_node();
if ( smp_found_config )
init_apic_mappings();
+ if ( x2apic_is_available() )
+ enable_x2apic();
+
init_IRQ();
xsm_init(&initrdidx, mbi, initial_images_start);
int invalidate_sync(struct iommu *iommu);
int iommu_flush_iec_global(struct iommu *iommu);
int iommu_flush_iec_index(struct iommu *iommu, u8 im, u16 iidx);
+void clear_fault_bits(struct iommu *iommu);
struct iommu * ioapic_to_iommu(unsigned int apic_id);
struct acpi_drhd_unit * ioapic_to_drhd(unsigned int apic_id);
struct acpi_drhd_unit * iommu_to_drhd(struct iommu *iommu);
if ( !iommu_enabled || !iommu_qinval || !iommu_intremap )
return 0;
+ if ( list_empty(&acpi_drhd_units) )
+ {
+ dprintk(XENLOG_WARNING VTDPREFIX, "VT-d is not supported\n");
+ return 0;
+ }
+
/* We MUST have a DRHD unit for each IOAPIC. */
for ( apic = 0; apic < nr_ioapics; apic++ )
if ( !ioapic_to_drhd(IO_APIC_ID(apic)) )
return 0;
}
- if ( list_empty(&acpi_drhd_units) )
- return 0;
-
for_each_drhd_unit ( drhd )
if ( !ecap_queued_inval(drhd->iommu->ecap) ||
!ecap_intr_remap(drhd->iommu->ecap) ||
out:
spin_unlock_irqrestore(&iommu->register_lock, flags);
}
+
+/*
+ * This function is used to enable Interrutp remapping when
+ * enable x2apic
+ */
+int iommu_enable_IR(void)
+{
+ struct acpi_drhd_unit *drhd;
+ struct iommu *iommu;
+
+ if ( !iommu_supports_eim() )
+ return -1;
+
+ for_each_drhd_unit ( drhd )
+ {
+ struct qi_ctrl *qi_ctrl = NULL;
+
+ iommu = drhd->iommu;
+ qi_ctrl = iommu_qi_ctrl(iommu);
+
+ /* Clear previous faults */
+ clear_fault_bits(iommu);
+
+ /*
+ * Disable interrupt remapping and queued invalidation if
+ * already enabled by BIOS
+ */
+ disable_intremap(iommu);
+ disable_qinval(iommu);
+ }
+
+ /* Enable queue invalidation */
+ for_each_drhd_unit ( drhd )
+ {
+ iommu = drhd->iommu;
+ if ( enable_qinval(iommu) != 0 )
+ {
+ dprintk(XENLOG_INFO VTDPREFIX,
+ "Failed to enable Queued Invalidation!\n");
+ return -1;
+ }
+ }
+
+ /* Enable interrupt remapping */
+ for_each_drhd_unit ( drhd )
+ {
+ iommu = drhd->iommu;
+ if ( enable_intremap(iommu, 1) )
+ {
+ dprintk(XENLOG_INFO VTDPREFIX,
+ "Failed to enable Interrupt Remapping!\n");
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+/*
+ * Check if interrupt remapping is enabled or not
+ * return 1: enabled
+ * return 0: not enabled
+ */
+int intremap_enabled(void)
+{
+ struct acpi_drhd_unit *drhd;
+ u32 sts;
+
+ for_each_drhd_unit ( drhd )
+ {
+ sts = dmar_readl(drhd->iommu->reg, DMAR_GSTS_REG);
+ if ( !(sts & DMA_GSTS_IRES) )
+ return 0;
+ }
+
+ return 1;
+}
spin_unlock(&pcidevs_lock);
}
-static void clear_fault_bits(struct iommu *iommu)
+void clear_fault_bits(struct iommu *iommu)
{
u64 val;
unsigned long flags;
extern int x2apic_enabled;
extern int directed_eoi_enabled;
+extern void check_x2apic_preenabled(void);
+extern int x2apic_cmdline_disable(void);
extern void enable_x2apic(void);
static __inline int x2apic_is_available(void)
.send_IPI_mask = send_IPI_mask_flat, \
.send_IPI_self = send_IPI_self_flat
+const struct genapic *apic_x2apic_probe(void);
void init_apic_ldr_x2apic_phys(void);
void init_apic_ldr_x2apic_cluster(void);
void clustered_apic_check_x2apic(void);
extern void ioapic_suspend(void);
extern void ioapic_resume(void);
+extern struct IO_APIC_route_entry **alloc_ioapic_entries(void);
+extern void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries);
+extern int save_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
+extern void mask_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
+extern int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
+
#else /* !CONFIG_X86_IO_APIC */
static inline void init_ioapic_mappings(void) {}
static inline void ioapic_suspend(void) {}
void disable_8259A_irq(unsigned int irq);
void enable_8259A_irq(unsigned int irq);
int i8259A_irq_pending(unsigned int irq);
+void mask_8259A(void);
+void unmask_8259A(void);
void init_8259A(int aeoi);
int i8259A_suspend(void);
int i8259A_resume(void);
int iommu_setup(void);
int iommu_supports_eim(void);
+int iommu_enable_IR(void);
+int intremap_enabled(void);
int iommu_add_device(struct pci_dev *pdev);
int iommu_remove_device(struct pci_dev *pdev);