x86: Dynamically allocate percpu data area when a CPU comes online.

At the same time, the data area starts life zeroed.

Signed-off-by: Keir Fraser <keir.fraser@citrix.com>

diff --git a/xen/arch/x86/Makefile b/xen/arch/x86/Makefile
index ba5da39e4b1a1bf0f0bfe3f0976feb4732543943..9aeb1cebb7628c0253c593a3f00c53449ffad45f 100644 (file)
--- a/xen/arch/x86/Makefile
+++ b/xen/arch/x86/Makefile
@@ -36,6 +36,7 @@ obj-y += mpparse.o
 obj-y += nmi.o
 obj-y += numa.o
 obj-y += pci.o
+obj-y += percpu.o
 obj-y += physdev.o
 obj-y += setup.o
 obj-y += shutdown.o

diff --git a/xen/arch/x86/irq.c b/xen/arch/x86/irq.c
index c0b4f2289dac1c51fde6f41cce00047ed9a929d3..b18316e7da9c034d51b04988043145d488c747a0 100644 (file)
--- a/xen/arch/x86/irq.c
+++ b/xen/arch/x86/irq.c
@@ -50,9 +50,7 @@ static struct timer *__read_mostly irq_guest_eoi_timer;
 
 static DEFINE_SPINLOCK(vector_lock);
 
-DEFINE_PER_CPU(vector_irq_t, vector_irq) = {
-    [0 ... NR_VECTORS - 1] = -1
-};
+DEFINE_PER_CPU(vector_irq_t, vector_irq);
 
 DEFINE_PER_CPU(struct cpu_user_regs *, __irq_regs);
 
@@ -269,7 +267,10 @@ int init_irq_data(void)
 {
     struct irq_desc *desc;
     struct irq_cfg *cfg;
-    int irq;
+    int irq, vector;
+
+    for (vector = 0; vector < NR_VECTORS; ++vector)
+        this_cpu(vector_irq)[vector] = -1;
 
     irq_desc = xmalloc_array(struct irq_desc, nr_irqs);
     irq_cfg = xmalloc_array(struct irq_cfg, nr_irqs);

diff --git a/xen/arch/x86/nmi.c b/xen/arch/x86/nmi.c
index ee954adc6a844a1268fbc6032329a1d1047fa10b..703294329c30991cc78bf8d88a596c4e42701074 100644 (file)
--- a/xen/arch/x86/nmi.c
+++ b/xen/arch/x86/nmi.c
@@ -230,7 +230,7 @@ static inline void write_watchdog_counter(const char *descr)
 
     do_div(count, nmi_hz);
     if(descr)
-        Dprintk("setting %s to -0x%08Lx\n", descr, count);
+        Dprintk("setting %s to -0x%"PRIx64"\n", descr, count);
     wrmsrl(nmi_perfctr_msr, 0 - count);
 }
 

diff --git a/xen/arch/x86/percpu.c b/xen/arch/x86/percpu.c
new file mode 100644 (file)
index 0000000..ea82647
--- /dev/null
+++ b/xen/arch/x86/percpu.c
@@ -0,0 +1,69 @@
+#include <xen/config.h>
+#include <xen/percpu.h>
+#include <xen/cpu.h>
+#include <xen/init.h>
+#include <xen/mm.h>
+
+unsigned long __per_cpu_offset[NR_CPUS];
+#define INVALID_PERCPU_AREA (-(long)__per_cpu_start)
+#define PERCPU_ORDER (get_order_from_bytes(__per_cpu_data_end-__per_cpu_start))
+
+void __init percpu_init_areas(void)
+{
+    unsigned int cpu;
+    for ( cpu = 1; cpu < NR_CPUS; cpu++ )
+        __per_cpu_offset[cpu] = INVALID_PERCPU_AREA;
+}
+
+static int init_percpu_area(unsigned int cpu)
+{
+    char *p;
+    if ( __per_cpu_offset[cpu] != INVALID_PERCPU_AREA )
+        return 0;
+    if ( (p = alloc_xenheap_pages(PERCPU_ORDER, 0)) == NULL )
+        return -ENOMEM;
+    memset(p, 0, __per_cpu_data_end - __per_cpu_start);
+    __per_cpu_offset[cpu] = p - __per_cpu_start;
+    return 0;
+}
+
+static void free_percpu_area(unsigned int cpu)
+{
+    char *p = __per_cpu_start + __per_cpu_offset[cpu];
+    free_xenheap_pages(p, PERCPU_ORDER);
+    __per_cpu_offset[cpu] = INVALID_PERCPU_AREA;
+}
+
+static int cpu_percpu_callback(
+    struct notifier_block *nfb, unsigned long action, void *hcpu)
+{
+    unsigned int cpu = (unsigned long)hcpu;
+    int rc = 0;
+
+    switch ( action )
+    {
+    case CPU_UP_PREPARE:
+        rc = init_percpu_area(cpu);
+        break;
+    case CPU_UP_CANCELED:
+    case CPU_DEAD:
+        free_percpu_area(cpu);
+        break;
+    default:
+        break;
+    }
+
+    return !rc ? NOTIFY_DONE : notifier_from_errno(rc);
+}
+
+static struct notifier_block cpu_percpu_nfb = {
+    .notifier_call = cpu_percpu_callback,
+    .priority = 100 /* highest priority */
+};
+
+static int __init percpu_presmp_init(void)
+{
+    register_cpu_notifier(&cpu_percpu_nfb);
+    return 0;
+}
+presmp_initcall(percpu_presmp_init);

diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 6140820a24f6b6a4b2593f6b94cf0410a74c8179..89994066424d45a3a905e4ad509d7e3982cf8b51 100644 (file)
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -107,12 +107,6 @@ unsigned long __initdata xenheap_initial_phys_start;
 unsigned long __read_mostly xenheap_phys_end;
 #endif
 
-DEFINE_PER_CPU_READ_MOSTLY(struct desc_struct *, gdt_table) = boot_cpu_gdt_table;
-#ifdef CONFIG_COMPAT
-DEFINE_PER_CPU_READ_MOSTLY(struct desc_struct *, compat_gdt_table)
-    = boot_cpu_compat_gdt_table;
-#endif
-
 DEFINE_PER_CPU(struct tss_struct, init_tss);
 
 char __attribute__ ((__section__(".bss.stack_aligned"))) cpu0_stack[STACK_SIZE];
@@ -192,46 +186,6 @@ static void free_xen_data(char *s, char *e)
 }
 
 extern char __init_begin[], __init_end[], __bss_start[];
-extern char __per_cpu_start[], __per_cpu_data_end[];
-
-static void __init percpu_init_areas(void)
-{
-    unsigned int i, data_size = __per_cpu_data_end - __per_cpu_start;
-
-    BUG_ON((unsigned long)__per_cpu_start & ~PAGE_MASK);
-    BUG_ON((unsigned long)__per_cpu_data_end & ~PAGE_MASK);
-    BUG_ON(data_size > PERCPU_SIZE);
-
-    /* Initialise per-cpu data area for all possible secondary CPUs. */
-    for ( i = 1; i < NR_CPUS; i++ )
-        memcpy(__per_cpu_start + (i << PERCPU_SHIFT),
-               __per_cpu_start,
-               data_size);
-}
-
-static void __init percpu_free_unused_areas(void)
-{
-    unsigned int i, data_size = __per_cpu_data_end - __per_cpu_start;
-    unsigned int first_unused;
-
-    /* Find first 'impossible' secondary CPU. */
-    for ( i = 1; i < NR_CPUS; i++ )
-        if ( !cpu_possible(i) )
-            break;
-    first_unused = i;
-
-    /* Check that there are no holes in cpu_possible_map. */
-    for ( ; i < NR_CPUS; i++ )
-        BUG_ON(cpu_possible(i));
-
-    /* Free all unused per-cpu data areas. */
-    free_xen_data(&__per_cpu_start[first_unused << PERCPU_SHIFT], __bss_start);
-
-    if ( data_size != PERCPU_SIZE )
-        for ( i = 0; i < first_unused; i++ )
-            free_xen_data(&__per_cpu_start[(i << PERCPU_SHIFT) + data_size],
-                          &__per_cpu_start[(i+1) << PERCPU_SHIFT]);
-}
 
 static void __init init_idle_domain(void)
 {
@@ -1013,8 +967,6 @@ void __init __start_xen(unsigned long mbi_p)
 
     init_apic_mappings();
 
-    percpu_free_unused_areas();
-
     init_IRQ();
 
     xsm_init(&initrdidx, mbi, initial_images_start);
@@ -1200,7 +1152,7 @@ int xen_in_range(unsigned long mfn)
     paddr_t start, end;
     int i;
 
-    enum { region_s3, region_text, region_percpu, region_bss, nr_regions };
+    enum { region_s3, region_text, region_bss, nr_regions };
     static struct {
         paddr_t s, e;
     } xen_regions[nr_regions];
@@ -1214,10 +1166,6 @@ int xen_in_range(unsigned long mfn)
         /* hypervisor code + data */
         xen_regions[region_text].s =__pa(&_stext);
         xen_regions[region_text].e = __pa(&__init_begin);
-        /* per-cpu data */
-        xen_regions[region_percpu].s = __pa(__per_cpu_start);
-        xen_regions[region_percpu].e = xen_regions[region_percpu].s +
-            (((paddr_t)last_cpu(cpu_possible_map) + 1) << PERCPU_SHIFT);
         /* bss */
         xen_regions[region_bss].s = __pa(&__bss_start);
         xen_regions[region_bss].e = __pa(&_end);
@@ -1226,25 +1174,8 @@ int xen_in_range(unsigned long mfn)
     start = (paddr_t)mfn << PAGE_SHIFT;
     end = start + PAGE_SIZE;
     for ( i = 0; i < nr_regions; i++ )
-    {
-        if ( (start >= xen_regions[i].e) || (end <= xen_regions[i].s) )
-            continue;
-
-        if ( i == region_percpu )
-        {
-            /*
-             * Check if the given page falls into an unused (and therefore
-             * freed) section of the per-cpu data space. Each CPU's data
-             * area is page-aligned, so the following arithmetic is safe.
-             */
-            unsigned int off = ((start - __pa(__per_cpu_start))
-                                & (PERCPU_SIZE - 1));
-            unsigned int data_sz = __per_cpu_data_end - __per_cpu_start;
-            return off < data_sz;
-        }
-
-        return 1;
-    }
+        if ( (start < xen_regions[i].e) && (end > xen_regions[i].s) )
+            return 1;
 
     return 0;
 }

diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c
index 8db5d0a152e4963ed543551181074a734f708350..53925beae70fca777dc468ef7e9dbd11db681ef7 100644 (file)
--- a/xen/arch/x86/smpboot.c
+++ b/xen/arch/x86/smpboot.c
@@ -487,7 +487,7 @@ static int wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip)
 
     for ( i = 0; i < num_starts; i++ )
     {
-        Dprintk("Sending STARTUP #%d.\n",j);
+        Dprintk("Sending STARTUP #%d.\n", i+1);
         apic_read_around(APIC_SPIV);
         apic_write(APIC_ESR, 0);
         apic_read(APIC_ESR);

diff --git a/xen/arch/x86/tboot.c b/xen/arch/x86/tboot.c
index 9cbed0b09f6ac8bfbb74f89a9387b4f6b1ad4701..63aba56820838c90fd341ab6d3fb88405b3922db 100644 (file)
--- a/xen/arch/x86/tboot.c
+++ b/xen/arch/x86/tboot.c
@@ -357,7 +357,7 @@ void tboot_shutdown(uint32_t shutdown_type)
         /*
          * Xen regions for tboot to MAC
          */
-        g_tboot_shared->num_mac_regions = 4;
+        g_tboot_shared->num_mac_regions = 3;
         /* S3 resume code (and other real mode trampoline code) */
         g_tboot_shared->mac_regions[0].start = bootsym_phys(trampoline_start);
         g_tboot_shared->mac_regions[0].size = bootsym_phys(trampoline_end) -
@@ -366,13 +366,9 @@ void tboot_shutdown(uint32_t shutdown_type)
         g_tboot_shared->mac_regions[1].start = (uint64_t)__pa(&_stext);
         g_tboot_shared->mac_regions[1].size = __pa(&__init_begin) -
                                               __pa(&_stext);
-        /* per-cpu data */
-        g_tboot_shared->mac_regions[2].start = (uint64_t)__pa(&__per_cpu_start);
-        g_tboot_shared->mac_regions[2].size =
-            (((uint64_t)last_cpu(cpu_possible_map) + 1) << PERCPU_SHIFT);
         /* bss */
-        g_tboot_shared->mac_regions[3].start = (uint64_t)__pa(&__bss_start);
-        g_tboot_shared->mac_regions[3].size = __pa(&_end) - __pa(&__bss_start);
+        g_tboot_shared->mac_regions[2].start = (uint64_t)__pa(&__bss_start);
+        g_tboot_shared->mac_regions[2].size = __pa(&_end) - __pa(&__bss_start);
 
         /*
          * MAC domains and other Xen memory

diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c
index f6ef710154d2044856ee943ccb298605f5d2574b..9d965a786b0fa8e5ad8951984be96d0df6bee9f2 100644 (file)
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -83,6 +83,11 @@ string_param("nmi", opt_nmi);
 
 DEFINE_PER_CPU_READ_MOSTLY(u32, ler_msr);
 
+DEFINE_PER_CPU_READ_MOSTLY(struct desc_struct *, gdt_table);
+#ifdef CONFIG_COMPAT
+DEFINE_PER_CPU_READ_MOSTLY(struct desc_struct *, compat_gdt_table);
+#endif
+
 /* Master table, used by CPU0. */
 idt_entry_t idt_table[IDT_ENTRIES];
 
@@ -3290,6 +3295,11 @@ void __init trap_init(void)
     /* CPU0 uses the master IDT. */
     idt_tables[0] = idt_table;
 
+    this_cpu(gdt_table) = boot_cpu_gdt_table;
+#ifdef CONFIG_COMPAT
+    this_cpu(compat_gdt_table) = boot_cpu_compat_gdt_table;
+#endif
+
     percpu_traps_init();
 
     cpu_init();

diff --git a/xen/arch/x86/x86_32/supervisor_mode_kernel.S b/xen/arch/x86/x86_32/supervisor_mode_kernel.S
index 2719aaf25a88c651e9342b711eff3ce85b2f4341..a9560b7ee692e81928e4c6e42fbca453e102bf71 100644 (file)
--- a/xen/arch/x86/x86_32/supervisor_mode_kernel.S
+++ b/xen/arch/x86/x86_32/supervisor_mode_kernel.S
@@ -102,7 +102,7 @@ ENTRY(fixup_ring0_guest_stack)
 
         movl  $PER_CPU_GDT_ENTRY*8,%ecx
         lsll  %ecx,%ecx
-        shll  $PERCPU_SHIFT,%ecx
+        movl  __per_cpu_offset(,%ecx,4),%ecx
         addl  $per_cpu__init_tss,%ecx
 
         # Load Xen stack from TSS.

diff --git a/xen/arch/x86/xen.lds.S b/xen/arch/x86/xen.lds.S
index 31ac018e71069af82b5ab289d9fb75a6c0a5b77c..2bc66042ded054e2ed1bdf39a3cbb68ac7f40c3e 100644 (file)
--- a/xen/arch/x86/xen.lds.S
+++ b/xen/arch/x86/xen.lds.S
@@ -59,7 +59,7 @@ SECTIONS
        CONSTRUCTORS
   } :text
 
-  . = ALIGN(128);
+  . = ALIGN(SMP_CACHE_BYTES);
   .data.read_mostly : {
        *(.data.read_mostly)
   } :text
@@ -71,7 +71,7 @@ SECTIONS
   __lock_profile_end = .;
 #endif
 
-  . = ALIGN(4096);             /* Init code and data */
+  . = ALIGN(PAGE_SIZE);             /* Init code and data */
   __init_begin = .;
   .init.text : {
        _sinittext = .;
@@ -99,33 +99,22 @@ SECTIONS
        *(.xsm_initcall.init)
        __xsm_initcall_end = .;
   } :text
-  . = ALIGN(PAGE_SIZE);
+  . = ALIGN(STACK_SIZE);
   __init_end = .;
 
-  __per_cpu_shift = PERCPU_SHIFT; /* kdump assist */
-  .data.percpu : {
-       __per_cpu_start = .;
-       *(.data.percpu)
-       . = ALIGN(SMP_CACHE_BYTES);
-       *(.data.percpu.read_mostly)
-       . = ALIGN(PAGE_SIZE);
-       __per_cpu_data_end = .;
-  } :text
-  . = __per_cpu_start + (NR_CPUS << PERCPU_SHIFT);
-  . = ALIGN(PAGE_SIZE);
-
-  /*
-   * Do not insert anything here - the unused portion of .data.percpu
-   * will be freed/unmapped up to __bss_start (defined below).
-   */
-
   .bss : {                     /* BSS */
-       . = ALIGN(STACK_SIZE);
        __bss_start = .;
        *(.bss.stack_aligned)
        . = ALIGN(PAGE_SIZE);
        *(.bss.page_aligned)
        *(.bss)
+       . = ALIGN(SMP_CACHE_BYTES);
+       __per_cpu_start = .;
+       *(.bss.percpu)
+       . = ALIGN(SMP_CACHE_BYTES);
+       *(.bss.percpu.read_mostly)
+       . = ALIGN(SMP_CACHE_BYTES);
+       __per_cpu_data_end = .;
   } :text
   _end = . ;
 

diff --git a/xen/common/rcupdate.c b/xen/common/rcupdate.c
index 827b8bd5ca046a3c70478bf48d215bf0509aa57b..c8f44bb804a51e46fc539c06c8b1b8c9f1bd68f4 100644 (file)
--- a/xen/common/rcupdate.c
+++ b/xen/common/rcupdate.c
@@ -53,7 +53,7 @@ struct rcu_ctrlblk rcu_ctrlblk = {
     .cpumask = CPU_MASK_NONE,
 };
 
-DEFINE_PER_CPU(struct rcu_data, rcu_data) = { 0L };
+DEFINE_PER_CPU(struct rcu_data, rcu_data);
 
 static int blimit = 10;
 static int qhimark = 10000;

diff --git a/xen/common/sched_credit.c b/xen/common/sched_credit.c
index 6dff698a640c055fe5c22fa36cc4f584d3ee69aa..0aeae90cb7b53dd06f361b9600e655b681047f3d 100644 (file)
--- a/xen/common/sched_credit.c
+++ b/xen/common/sched_credit.c
@@ -231,7 +231,7 @@ static void burn_credits(struct csched_vcpu *svc, s_time_t now)
 static int opt_tickle_one_idle __read_mostly = 1;
 boolean_param("tickle_one_idle_cpu", opt_tickle_one_idle);
 
-DEFINE_PER_CPU(unsigned int, last_tickle_cpu) = 0;
+DEFINE_PER_CPU(unsigned int, last_tickle_cpu);
 
 static inline void
 __runq_tickle(unsigned int cpu, struct csched_vcpu *new)

diff --git a/xen/drivers/cpufreq/utility.c b/xen/drivers/cpufreq/utility.c
index 612f7e0919251e4e9c8b406c48e5bdfe754080e6..3085173ae0f3db1223889df4f9995b94dffd67c8 100644 (file)
--- a/xen/drivers/cpufreq/utility.c
+++ b/xen/drivers/cpufreq/utility.c
@@ -36,7 +36,7 @@ struct cpufreq_driver   *cpufreq_driver;
 struct processor_pminfo *__read_mostly processor_pminfo[NR_CPUS];
 struct cpufreq_policy   *__read_mostly cpufreq_cpu_policy[NR_CPUS];
 
-DEFINE_PER_CPU(spinlock_t, cpufreq_statistic_lock) = SPIN_LOCK_UNLOCKED;
+DEFINE_PER_CPU(spinlock_t, cpufreq_statistic_lock);
 
 /*********************************************************************
  *                    Px STATISTIC INFO                              *
@@ -95,6 +95,8 @@ int cpufreq_statistic_init(unsigned int cpuid)
     spinlock_t *cpufreq_statistic_lock = 
                           &per_cpu(cpufreq_statistic_lock, cpuid);
 
+    spin_lock_init(cpufreq_statistic_lock);
+
     if ( !pmpt )
         return -EINVAL;
 

diff --git a/xen/include/asm-x86/percpu.h b/xen/include/asm-x86/percpu.h
index d8860203ae12e4fc35045094d16ccf004964be57..fb1d4140ecbb55e93ba5ad41aa5d05cbf3c69155 100644 (file)
--- a/xen/include/asm-x86/percpu.h
+++ b/xen/include/asm-x86/percpu.h
@@ -1,17 +1,20 @@
 #ifndef __X86_PERCPU_H__
 #define __X86_PERCPU_H__
 
-#define PERCPU_SHIFT 13
-#define PERCPU_SIZE  (1UL << PERCPU_SHIFT)
+#ifndef __ASSEMBLY__
+extern char __per_cpu_start[], __per_cpu_data_end[];
+extern unsigned long __per_cpu_offset[NR_CPUS];
+void percpu_init_areas(void);
+#endif
 
 /* Separate out the type, so (int[3], foo) works. */
 #define __DEFINE_PER_CPU(type, name, suffix)                    \
-    __attribute__((__section__(".data.percpu" #suffix)))        \
+    __attribute__((__section__(".bss.percpu" #suffix)))         \
     __typeof__(type) per_cpu_##name
 
 /* var is in discarded region: offset to particular copy we want */
 #define per_cpu(var, cpu)  \
-    (*RELOC_HIDE(&per_cpu__##var, ((unsigned int)(cpu))<<PERCPU_SHIFT))
+    (*RELOC_HIDE(&per_cpu__##var, __per_cpu_offset[cpu]))
 #define __get_cpu_var(var) \
     (per_cpu(var, smp_processor_id()))
 

diff --git a/xen/include/asm-x86/x86_32/asm_defns.h b/xen/include/asm-x86/x86_32/asm_defns.h
index bbdd55efce52e386ba4b124a4c41f0a67646ed9e..10bf8fe25ab5a0be436482956d0fa05615e091c2 100644 (file)
--- a/xen/include/asm-x86/x86_32/asm_defns.h
+++ b/xen/include/asm-x86/x86_32/asm_defns.h
@@ -82,7 +82,7 @@
 #define PERFC_INCR(_name,_idx,_cur)                     \
         pushl _cur;                                     \
         movl VCPU_processor(_cur),_cur;                 \
-        shll $PERCPU_SHIFT,_cur;                        \
+        movl __per_cpu_offset(,_cur,4),_cur;            \
         incl per_cpu__perfcounters+_name*4(_cur,_idx,4);\
         popl _cur
 #else

diff --git a/xen/include/asm-x86/x86_64/asm_defns.h b/xen/include/asm-x86/x86_64/asm_defns.h
index 328c5263aa5628479607ea68afa680e32be07366..df1343f6073821922c43b398932edb250e1afc4d 100644 (file)
--- a/xen/include/asm-x86/x86_64/asm_defns.h
+++ b/xen/include/asm-x86/x86_64/asm_defns.h
@@ -68,7 +68,7 @@
         movslq VCPU_processor(_cur),_cur;       \
         pushq %rdx;                             \
         leaq per_cpu__perfcounters(%rip),%rdx;  \
-        shlq $PERCPU_SHIFT,_cur;                \
+        movq __per_cpu_offset(,_cur,8),_cur;    \
         addq %rdx,_cur;                         \
         popq %rdx;                              \
         incl _name*4(_cur,_idx,4);              \

diff --git a/xen/xsm/flask/avc.c b/xen/xsm/flask/avc.c
index 06a3c7dc92cdbfe457b41b99ae8d5aca708c25e5..57b1657d619f01ab8d972b4df6e272f895598433 100644 (file)
--- a/xen/xsm/flask/avc.c
+++ b/xen/xsm/flask/avc.c
@@ -118,7 +118,7 @@ struct avc_callback_node {
 unsigned int avc_cache_threshold = AVC_DEF_CACHE_THRESHOLD;
 
 #ifdef FLASK_AVC_STATS
-DEFINE_PER_CPU(struct avc_cache_stats, avc_cache_stats) = { 0 };
+DEFINE_PER_CPU(struct avc_cache_stats, avc_cache_stats);
 #endif
 
 static struct avc_cache avc_cache;