#define L3_PROT (_PAGE_PRESENT)
#elif defined(__x86_64__)
/* Allow ring-3 access in long mode as guest cannot use ring 1. */
-#define L1_PROT (_PAGE_PRESENT|_PAGE_RW|_PAGE_ACCESSED|_PAGE_USER)
-#define L2_PROT (_PAGE_PRESENT|_PAGE_RW|_PAGE_ACCESSED|_PAGE_DIRTY|_PAGE_USER)
-#define L3_PROT (_PAGE_PRESENT|_PAGE_RW|_PAGE_ACCESSED|_PAGE_DIRTY|_PAGE_USER)
-#define L4_PROT (_PAGE_PRESENT|_PAGE_RW|_PAGE_ACCESSED|_PAGE_DIRTY|_PAGE_USER)
+#define BASE_PROT (_PAGE_PRESENT|_PAGE_RW|_PAGE_ACCESSED|_PAGE_USER)
+#define L1_PROT (BASE_PROT|_PAGE_GUEST_KERNEL)
+#define L2_PROT (BASE_PROT|_PAGE_DIRTY)
+#define L3_PROT (BASE_PROT|_PAGE_DIRTY)
+#define L4_PROT (BASE_PROT|_PAGE_DIRTY)
#endif
#define round_pgup(_p) (((_p)+(PAGE_SIZE-1))&PAGE_MASK)
* TLB flushes are timestamped using a global virtual 'clock' which ticks
* on any TLB flush on any processor.
*
- * Copyright (c) 2003-2004, K A Fraser
+ * Copyright (c) 2003-2006, K A Fraser
*/
#include <xen/config.h>
#include <xen/sched.h>
#include <xen/softirq.h>
#include <asm/flushtlb.h>
+#include <asm/page.h>
/* Debug builds: Wrap frequently to stress-test the wrap logic. */
#ifdef NDEBUG
u32 tlbflush_clock = 1U;
DEFINE_PER_CPU(u32, tlbflush_time);
-void write_cr3(unsigned long cr3)
+/*
+ * pre_flush(): Increment the virtual TLB-flush clock. Returns new clock value.
+ *
+ * This must happen *before* we flush the TLB. If we do it after, we race other
+ * CPUs invalidating PTEs. For example, a page invalidated after the flush
+ * might get the old timestamp, but this CPU can speculatively fetch the
+ * mapping into its TLB after the flush but before inc'ing the clock.
+ */
+static u32 pre_flush(void)
{
u32 t, t1, t2;
- unsigned long flags;
-
- /* This non-reentrant function is sometimes called in interrupt context. */
- local_irq_save(flags);
-
- /*
- * STEP 1. Increment the virtual clock *before* flushing the TLB.
- * If we do it after, we race other CPUs invalidating PTEs.
- * (e.g., a page invalidated after the flush might get the old
- * timestamp, but this CPU can speculatively fetch the mapping
- * into its TLB after the flush but before inc'ing the clock).
- */
t = tlbflush_clock;
do {
if ( unlikely(t2 == 0) )
raise_softirq(NEW_TLBFLUSH_CLOCK_PERIOD_SOFTIRQ);
- /*
- * STEP 2. Update %CR3, thereby flushing the TLB.
- */
-
skip_clocktick:
+ return t2;
+}
+
+/*
+ * post_flush(): Update this CPU's timestamp with specified clock value.
+ *
+ * Note that this happens *after* flushing the TLB, as otherwise we can race a
+ * NEED_FLUSH() test on another CPU. (e.g., other CPU sees the updated CPU
+ * stamp and so does not force a synchronous TLB flush, but the flush in this
+ * function hasn't yet occurred and so the TLB might be stale). The ordering
+ * would only actually matter if this function were interruptible, and
+ * something that abuses the stale mapping could exist in an interrupt
+ * handler. In fact neither of these is the case, so really we are being ultra
+ * paranoid.
+ */
+static void post_flush(u32 t)
+{
+ this_cpu(tlbflush_time) = t;
+}
+
+void write_cr3(unsigned long cr3)
+{
+ unsigned long flags;
+ u32 t;
+
+ /* This non-reentrant function is sometimes called in interrupt context. */
+ local_irq_save(flags);
+
+ t = pre_flush();
+
+#ifdef USER_MAPPINGS_ARE_GLOBAL
+ __pge_off();
+ __asm__ __volatile__ ( "mov %0, %%cr3" : : "r" (cr3) : "memory" );
+ __pge_on();
+#else
__asm__ __volatile__ ( "mov %0, %%cr3" : : "r" (cr3) : "memory" );
+#endif
+
+ post_flush(t);
+
+ local_irq_restore(flags);
+}
+
+void local_flush_tlb(void)
+{
+ unsigned long flags;
+ u32 t;
+
+ /* This non-reentrant function is sometimes called in interrupt context. */
+ local_irq_save(flags);
+
+ t = pre_flush();
+
+#ifdef USER_MAPPINGS_ARE_GLOBAL
+ __pge_off();
+ __pge_on();
+#else
+ __asm__ __volatile__ ( "mov %0, %%cr3" : : "r" (read_cr3()) : "memory" );
+#endif
- /*
- * STEP 3. Update this CPU's timestamp. Note that this happens *after*
- * flushing the TLB, as otherwise we can race a NEED_FLUSH() test
- * on another CPU. (e.g., other CPU sees the updated CPU stamp and
- * so does not force a synchronous TLB flush, but the flush in this
- * function hasn't yet occurred and so the TLB might be stale).
- * The ordering would only actually matter if this function were
- * interruptible, and something that abuses the stale mapping could
- * exist in an interrupt handler. In fact neither of these is the
- * case, so really we are being ultra paranoid.
- */
-
- this_cpu(tlbflush_time) = t2;
+ post_flush(t);
local_irq_restore(flags);
}
#endif /* 4 level */
#ifdef __x86_64__
+
+#ifdef USER_MAPPINGS_ARE_GLOBAL
+#define adjust_guest_l1e(pl1e) \
+ do { \
+ if ( likely(l1e_get_flags((pl1e)) & _PAGE_PRESENT) ) \
+ { \
+ /* _PAGE_GUEST_KERNEL page cannot have the Global bit set. */ \
+ if ( (l1e_get_flags((pl1e)) & (_PAGE_GUEST_KERNEL|_PAGE_GLOBAL)) \
+ == (_PAGE_GUEST_KERNEL|_PAGE_GLOBAL) ) \
+ MEM_LOG("Global bit is set to kernel page %lx", \
+ l1e_get_pfn((pl1e))); \
+ if ( !(l1e_get_flags((pl1e)) & _PAGE_USER) ) \
+ l1e_add_flags((pl1e), (_PAGE_GUEST_KERNEL|_PAGE_USER)); \
+ if ( !(l1e_get_flags((pl1e)) & _PAGE_GUEST_KERNEL) ) \
+ l1e_add_flags((pl1e), (_PAGE_GLOBAL|_PAGE_USER)); \
+ } \
+ } while ( 0 )
+#else
#define adjust_guest_l1e(pl1e) \
- do { \
+ do { \
if ( likely(l1e_get_flags((pl1e)) & _PAGE_PRESENT) ) \
l1e_add_flags((pl1e), _PAGE_USER); \
} while ( 0 )
+#endif
#define adjust_guest_l2e(pl2e) \
do { \
if ( likely(l4e_get_flags((pl4e)) & _PAGE_PRESENT) ) \
l4e_add_flags((pl4e), _PAGE_USER); \
} while ( 0 )
-#else
+
+#else /* !defined(__x86_64__) */
+
#define adjust_guest_l1e(_p) ((void)0)
#define adjust_guest_l2e(_p) ((void)0)
#define adjust_guest_l3e(_p) ((void)0)
+
#endif
void put_page_from_l1e(l1_pgentry_t l1e, struct domain *d)
projects / xen.git / commitdiff