From: awilliam@xenbuild.aw <awilliam@xenbuild.aw>
Date: Fri, 16 Jun 2006 16:18:54 +0000 (-0600)
Subject: merge with xen-unstable.hg
X-Git-Tag: archive/raspbian/4.8.0-1+rpi1~1^2~15954
X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=9beb92215a3396b4ffd3e6e0deb963ff832c6169;p=xen.git

merge with xen-unstable.hg
---

9beb92215a3396b4ffd3e6e0deb963ff832c6169
diff --cc xen/arch/ia64/xen/mm.c
index 384216c9ad,0000000000..96c64dc5a3
mode 100644,000000..100644
--- a/xen/arch/ia64/xen/mm.c
+++ b/xen/arch/ia64/xen/mm.c
@@@ -1,1447 -1,0 +1,1448 @@@
 +/*
 + *  Copyright (C) 2005 Intel Co
 + *	Kun Tian (Kevin Tian) <kevin.tian@intel.com>
 + *
 + * 05/04/29 Kun Tian (Kevin Tian) <kevin.tian@intel.com> Add VTI domain support
 + *
 + * Copyright (c) 2006 Isaku Yamahata <yamahata at valinux co jp>
 + *                    VA Linux Systems Japan K.K.
 + *                    dom0 vp model support
 + */
 +
 +#include <xen/config.h>
 +#include <xen/sched.h>
 +#include <xen/domain.h>
 +#include <asm/xentypes.h>
 +#include <asm/mm.h>
 +#include <asm/pgalloc.h>
 +#include <asm/vhpt.h>
 +#include <asm/vcpu.h>
 +#include <linux/efi.h>
 +
 +#ifndef CONFIG_XEN_IA64_DOM0_VP
 +#define CONFIG_DOMAIN0_CONTIGUOUS
 +#else
 +static void domain_page_flush(struct domain* d, unsigned long mpaddr,
 +                              unsigned long old_mfn, unsigned long new_mfn);
 +#endif
 +
 +static struct domain *dom_xen, *dom_io;
 +
 +// followings are stolen from arch_init_memory() @ xen/arch/x86/mm.c
 +void
 +alloc_dom_xen_and_dom_io(void)
 +{
 +    /*
 +     * Initialise our DOMID_XEN domain.
 +     * Any Xen-heap pages that we will allow to be mapped will have
 +     * their domain field set to dom_xen.
 +     */
 +    dom_xen = alloc_domain(DOMID_XEN);
 +    BUG_ON(dom_xen == NULL);
 +
 +    /*
 +     * Initialise our DOMID_IO domain.
 +     * This domain owns I/O pages that are within the range of the page_info
 +     * array. Mappings occur at the priv of the caller.
 +     */
 +    dom_io = alloc_domain(DOMID_IO);
 +    BUG_ON(dom_io == NULL);
 +}
 +
 +// heavily depends on the struct page_info layout.
 +// if (page_get_owner(page) == d &&
 +//     test_and_clear_bit(_PGC_allocated, &page->count_info)) {
 +//     put_page(page);
 +// }
 +static void
 +try_to_clear_PGC_allocate(struct domain* d, struct page_info* page)
 +{
 +    u32 _d, _nd;
 +    u64 x, nx, y;
 +
 +    _d = pickle_domptr(d);
 +    y = *((u64*)&page->count_info);
 +    do {
 +        x = y;
 +        _nd = x >> 32;
 +        nx = x - 1;
 +        __clear_bit(_PGC_allocated, &nx);
 +
 +        if (unlikely(!(x & PGC_allocated)) || unlikely(_nd != _d)) {
 +            struct domain* nd = unpickle_domptr(_nd);
 +            if (nd == NULL) {
 +                DPRINTK("gnttab_transfer: Bad page %p: ed=%p(%u) 0x%x, "
 +                        "sd=%p 0x%x,"
 +                        " caf=%016lx, taf=%" PRtype_info "\n",
 +                        (void *) page_to_mfn(page),
 +                        d, d->domain_id, _d,
 +                        nd, _nd,
 +                        x,
 +                        page->u.inuse.type_info);
 +            }
 +            break;
 +        }
 +
 +        BUG_ON((nx & PGC_count_mask) < 1);
 +        y = cmpxchg((u64*)&page->count_info, x, nx);
 +    } while (unlikely(y != x));
 +}
 +
 +static void
 +relinquish_pte(struct domain* d, pte_t* pte)
 +{
 +    unsigned long mfn = pte_pfn(*pte);
 +    struct page_info* page;
 +
 +    // vmx domain use bit[58:56] to distinguish io region from memory.
 +    // see vmx_build_physmap_table() in vmx_init.c
 +    if (!pte_mem(*pte))
 +        return;
 +
 +    // domain might map IO space or acpi table pages. check it.
 +    if (!mfn_valid(mfn))
 +        return;
 +    page = mfn_to_page(mfn);
 +    // struct page_info corresponding to mfn may exist or not depending
 +    // on CONFIG_VIRTUAL_FRAME_TABLE.
 +    // This check is too easy.
 +    // The right way is to check whether this page is of io area or acpi pages
 +    if (page_get_owner(page) == NULL) {
 +        BUG_ON(page->count_info != 0);
 +        return;
 +    }
 +
 +#ifdef CONFIG_XEN_IA64_DOM0_VP
 +    if (page_get_owner(page) == d) {
 +        BUG_ON(get_gpfn_from_mfn(mfn) == INVALID_M2P_ENTRY);
 +        set_gpfn_from_mfn(mfn, INVALID_M2P_ENTRY);
 +    }
 +#endif
 +    try_to_clear_PGC_allocate(d, page);
 +    put_page(page);
 +}
 +
 +static void
 +relinquish_pmd(struct domain* d, pmd_t* pmd, unsigned long offset)
 +{
 +    unsigned long i;
 +    pte_t* pte = pte_offset_map(pmd, offset);
 +
 +    for (i = 0; i < PTRS_PER_PTE; i++, pte++) {
 +        if (!pte_present(*pte))
 +            continue;
 +
 +        relinquish_pte(d, pte);
 +    }
 +    pte_free_kernel(pte_offset_map(pmd, offset));
 +}
 +
 +static void
 +relinquish_pud(struct domain* d, pud_t *pud, unsigned long offset)
 +{
 +    unsigned long i;
 +    pmd_t *pmd = pmd_offset(pud, offset);
 +
 +    for (i = 0; i < PTRS_PER_PMD; i++, pmd++) {
 +        if (!pmd_present(*pmd))
 +            continue;
 +
 +        relinquish_pmd(d, pmd, offset + (i << PMD_SHIFT));
 +    }
 +    pmd_free(pmd_offset(pud, offset));
 +}
 +
 +static void
 +relinquish_pgd(struct domain* d, pgd_t *pgd, unsigned long offset)
 +{
 +    unsigned long i;
 +    pud_t *pud = pud_offset(pgd, offset);
 +
 +    for (i = 0; i < PTRS_PER_PUD; i++, pud++) {
 +        if (!pud_present(*pud))
 +            continue;
 +
 +        relinquish_pud(d, pud, offset + (i << PUD_SHIFT));
 +    }
 +    pud_free(pud_offset(pgd, offset));
 +}
 +
 +void
 +relinquish_mm(struct domain* d)
 +{
 +    struct mm_struct* mm = &d->arch.mm;
 +    unsigned long i;
 +    pgd_t* pgd;
 +
 +    if (mm->pgd == NULL)
 +        return;
 +
 +    pgd = pgd_offset(mm, 0);
 +    for (i = 0; i < PTRS_PER_PGD; i++, pgd++) {
 +        if (!pgd_present(*pgd))
 +            continue;
 +
 +        relinquish_pgd(d, pgd, i << PGDIR_SHIFT);
 +    }
 +    pgd_free(mm->pgd);
 +    mm->pgd = NULL;
 +}
 +
 +// stolen from share_xen_page_with_guest() in xen/arch/x86/mm.c
 +void
 +share_xen_page_with_guest(struct page_info *page,
 +                          struct domain *d, int readonly)
 +{
 +    if ( page_get_owner(page) == d )
 +        return;
 +
 +#if 1
 +    if (readonly) {
 +        printk("%s:%d readonly is not supported yet\n", __func__, __LINE__);
 +    }
 +#endif
 +
 +    // alloc_xenheap_pages() doesn't initialize page owner.
 +    //BUG_ON(page_get_owner(page) != NULL);
 +
 +    spin_lock(&d->page_alloc_lock);
 +
 +#ifndef __ia64__
 +    /* The incremented type count pins as writable or read-only. */
 +    page->u.inuse.type_info  = (readonly ? PGT_none : PGT_writable_page);
 +    page->u.inuse.type_info |= PGT_validated | 1;
 +#endif
 +
 +    page_set_owner(page, d);
 +    wmb(); /* install valid domain ptr before updating refcnt. */
 +    ASSERT(page->count_info == 0);
 +    page->count_info |= PGC_allocated | 1;
 +
 +    if ( unlikely(d->xenheap_pages++ == 0) )
 +        get_knownalive_domain(d);
 +    list_add_tail(&page->list, &d->xenpage_list);
 +
 +    // grant_table_destroy() releases these pages.
 +    // but it doesn't clear their m2p entry. So there might remain stale
 +    // entries. such a stale entry is cleared here.
 +    set_gpfn_from_mfn(page_to_mfn(page), INVALID_M2P_ENTRY);
 +
 +    spin_unlock(&d->page_alloc_lock);
 +}
 +
 +void
 +share_xen_page_with_privileged_guests(struct page_info *page, int readonly)
 +{
 +    share_xen_page_with_guest(page, dom_xen, readonly);
 +}
 +
 +unsigned long
 +gmfn_to_mfn_foreign(struct domain *d, unsigned long gpfn)
 +{
 +	unsigned long pte;
 +
 +#ifndef CONFIG_XEN_IA64_DOM0_VP
 +	if (d == dom0)
 +		return(gpfn);
 +#endif
 +	pte = lookup_domain_mpa(d,gpfn << PAGE_SHIFT);
 +	if (!pte) {
 +		panic("gmfn_to_mfn_foreign: bad gpfn. spinning...\n");
 +	}
 +	return ((pte & _PFN_MASK) >> PAGE_SHIFT);
 +}
 +
 +// given a domain virtual address, pte and pagesize, extract the metaphysical
 +// address, convert the pte for a physical address for (possibly different)
 +// Xen PAGE_SIZE and return modified pte.  (NOTE: TLB insert should use
 +// PAGE_SIZE!)
 +u64 translate_domain_pte(u64 pteval, u64 address, u64 itir__, u64* logps)
 +{
 +	struct domain *d = current->domain;
 +	ia64_itir_t itir = {.itir = itir__};
 +	u64 mask, mpaddr, pteval2;
 +	u64 arflags;
 +	u64 arflags2;
 +
 +	pteval &= ((1UL << 53) - 1);// ignore [63:53] bits
 +
 +	// FIXME address had better be pre-validated on insert
 +	mask = ~itir_mask(itir.itir);
 +	mpaddr = (((pteval & ~_PAGE_ED) & _PAGE_PPN_MASK) & ~mask) |
 +	         (address & mask);
 +#ifdef CONFIG_XEN_IA64_DOM0_VP
 +	if (itir.ps > PAGE_SHIFT) {
 +		itir.ps = PAGE_SHIFT;
 +	}
 +#endif
 +	*logps = itir.ps;
 +#ifndef CONFIG_XEN_IA64_DOM0_VP
 +	if (d == dom0) {
 +		if (mpaddr < dom0_start || mpaddr >= dom0_start + dom0_size) {
 +			/*
 +			printk("translate_domain_pte: out-of-bounds dom0 mpaddr 0x%lx! itc=%lx...\n",
 +				mpaddr, ia64_get_itc());
 +			*/
 +		}
 +	}
 +	else if ((mpaddr >> PAGE_SHIFT) > d->max_pages) {
 +		/* Address beyond the limit.  However the grant table is
 +		   also beyond the limit.  Display a message if not in the
 +		   grant table.  */
 +		if (mpaddr >= IA64_GRANT_TABLE_PADDR
 +		    && mpaddr < (IA64_GRANT_TABLE_PADDR 
 +				 + (ORDER_GRANT_FRAMES << PAGE_SHIFT)))
 +			printf("translate_domain_pte: bad mpa=0x%lx (> 0x%lx),"
 +			       "vadr=0x%lx,pteval=0x%lx,itir=0x%lx\n",
 +			       mpaddr, (unsigned long)d->max_pages<<PAGE_SHIFT,
 +			       address, pteval, itir.itir);
 +	}
 +#endif
 +	pteval2 = lookup_domain_mpa(d,mpaddr);
 +	arflags  = pteval  & _PAGE_AR_MASK;
 +	arflags2 = pteval2 & _PAGE_AR_MASK;
 +	if (arflags != _PAGE_AR_R && arflags2 == _PAGE_AR_R) {
 +#if 0
 +		DPRINTK("%s:%d "
 +		        "pteval 0x%lx arflag 0x%lx address 0x%lx itir 0x%lx "
 +		        "pteval2 0x%lx arflags2 0x%lx mpaddr 0x%lx\n",
 +		        __func__, __LINE__,
 +		        pteval, arflags, address, itir__,
 +		        pteval2, arflags2, mpaddr);
 +#endif
 +		pteval = (pteval & ~_PAGE_AR_MASK) | _PAGE_AR_R;
 +}
 +
 +	pteval2 &= _PAGE_PPN_MASK; // ignore non-addr bits
 +	pteval2 |= (pteval & _PAGE_ED);
 +	pteval2 |= _PAGE_PL_2; // force PL0->2 (PL3 is unaffected)
 +	pteval2 = (pteval & ~_PAGE_PPN_MASK) | pteval2;
 +	return pteval2;
 +}
 +
 +// given a current domain metaphysical address, return the physical address
 +unsigned long translate_domain_mpaddr(unsigned long mpaddr)
 +{
 +	unsigned long pteval;
 +
 +#ifndef CONFIG_XEN_IA64_DOM0_VP
 +	if (current->domain == dom0) {
 +		if (mpaddr < dom0_start || mpaddr >= dom0_start + dom0_size) {
 +			printk("translate_domain_mpaddr: out-of-bounds dom0 mpaddr 0x%lx! continuing...\n",
 +				mpaddr);
 +		}
 +	}
 +#endif
 +	pteval = lookup_domain_mpa(current->domain,mpaddr);
 +	return ((pteval & _PAGE_PPN_MASK) | (mpaddr & ~PAGE_MASK));
 +}
 +
 +//XXX !xxx_present() should be used instread of !xxx_none()?
 +// pud, pmd, pte page is zero cleared when they are allocated.
 +// Their area must be visible before population so that
 +// cmpxchg must have release semantics.
 +static pte_t*
 +lookup_alloc_domain_pte(struct domain* d, unsigned long mpaddr)
 +{
 +    struct mm_struct *mm = &d->arch.mm;
 +    pgd_t *pgd;
 +    pud_t *pud;
 +    pmd_t *pmd;
 +
 +    BUG_ON(mm->pgd == NULL);
 +
 +    pgd = pgd_offset(mm, mpaddr);
 + again_pgd:
 +    if (unlikely(pgd_none(*pgd))) {
 +        pud_t *old_pud = NULL;
 +        pud = pud_alloc_one(mm, mpaddr);
 +        if (unlikely(!pgd_cmpxchg_rel(mm, pgd, old_pud, pud))) {
 +            pud_free(pud);
 +            goto again_pgd;
 +        }
 +    }
 +
 +    pud = pud_offset(pgd, mpaddr);
 + again_pud:
 +    if (unlikely(pud_none(*pud))) {
 +        pmd_t* old_pmd = NULL;
 +        pmd = pmd_alloc_one(mm, mpaddr);
 +        if (unlikely(!pud_cmpxchg_rel(mm, pud, old_pmd, pmd))) {
 +            pmd_free(pmd);
 +            goto again_pud;
 +        }
 +    }
 +
 +    pmd = pmd_offset(pud, mpaddr);
 + again_pmd:
 +    if (unlikely(pmd_none(*pmd))) {
 +        pte_t* old_pte = NULL;
 +        pte_t* pte = pte_alloc_one_kernel(mm, mpaddr);
 +        if (unlikely(!pmd_cmpxchg_kernel_rel(mm, pmd, old_pte, pte))) {
 +            pte_free_kernel(pte);
 +            goto again_pmd;
 +        }
 +    }
 +
 +    return pte_offset_map(pmd, mpaddr);
 +}
 +
 +//XXX xxx_none() should be used instread of !xxx_present()?
 +static pte_t*
 +lookup_noalloc_domain_pte(struct domain* d, unsigned long mpaddr)
 +{
 +    struct mm_struct *mm = &d->arch.mm;
 +    pgd_t *pgd;
 +    pud_t *pud;
 +    pmd_t *pmd;
 +
 +    BUG_ON(mm->pgd == NULL);
 +    pgd = pgd_offset(mm, mpaddr);
 +    if (unlikely(!pgd_present(*pgd)))
 +        return NULL;
 +
 +    pud = pud_offset(pgd, mpaddr);
 +    if (unlikely(!pud_present(*pud)))
 +        return NULL;
 +
 +    pmd = pmd_offset(pud, mpaddr);
 +    if (unlikely(!pmd_present(*pmd)))
 +        return NULL;
 +
 +    return pte_offset_map(pmd, mpaddr);
 +}
 +
 +#ifdef CONFIG_XEN_IA64_DOM0_VP
 +static pte_t*
 +lookup_noalloc_domain_pte_none(struct domain* d, unsigned long mpaddr)
 +{
 +    struct mm_struct *mm = &d->arch.mm;
 +    pgd_t *pgd;
 +    pud_t *pud;
 +    pmd_t *pmd;
 +
 +    BUG_ON(mm->pgd == NULL);
 +    pgd = pgd_offset(mm, mpaddr);
 +    if (unlikely(pgd_none(*pgd)))
 +        return NULL;
 +
 +    pud = pud_offset(pgd, mpaddr);
 +    if (unlikely(pud_none(*pud)))
 +        return NULL;
 +
 +    pmd = pmd_offset(pud, mpaddr);
 +    if (unlikely(pmd_none(*pmd)))
 +        return NULL;
 +
 +    return pte_offset_map(pmd, mpaddr);
 +}
 +
 +unsigned long
 +____lookup_domain_mpa(struct domain *d, unsigned long mpaddr)
 +{
 +    pte_t *pte;
 +
 +    pte = lookup_noalloc_domain_pte(d, mpaddr);
 +    if (pte == NULL)
 +        return INVALID_MFN;
 +
 +    if (pte_present(*pte))
 +        return (pte->pte & _PFN_MASK);
 +    else if (VMX_DOMAIN(d->vcpu[0]))
 +        return GPFN_INV_MASK;
 +    return INVALID_MFN;
 +}
 +
 +unsigned long
 +__lookup_domain_mpa(struct domain *d, unsigned long mpaddr)
 +{
 +    unsigned long machine = ____lookup_domain_mpa(d, mpaddr);
 +    if (machine != INVALID_MFN)
 +        return machine;
 +
 +    printk("%s: d 0x%p id %d current 0x%p id %d\n",
 +           __func__, d, d->domain_id, current, current->vcpu_id);
 +    printk("%s: bad mpa 0x%lx (max_pages 0x%lx)\n",
 +           __func__, mpaddr, (unsigned long)d->max_pages << PAGE_SHIFT);
 +    return INVALID_MFN;
 +}
 +#endif
 +
 +unsigned long lookup_domain_mpa(struct domain *d, unsigned long mpaddr)
 +{
 +    pte_t *pte;
 +
 +#ifdef CONFIG_DOMAIN0_CONTIGUOUS
 +    if (d == dom0) {
 +        pte_t pteval;
 +        if (mpaddr < dom0_start || mpaddr >= dom0_start + dom0_size) {
 +            //printk("lookup_domain_mpa: bad dom0 mpaddr 0x%lx!\n",mpaddr);
 +            //printk("lookup_domain_mpa: start=0x%lx,end=0x%lx!\n",dom0_start,dom0_start+dom0_size);
 +        }
 +        pteval = pfn_pte(mpaddr >> PAGE_SHIFT,
 +            __pgprot(__DIRTY_BITS | _PAGE_PL_2 | _PAGE_AR_RWX));
 +        return pte_val(pteval);
 +    }
 +#endif
 +    pte = lookup_noalloc_domain_pte(d, mpaddr);
 +    if (pte != NULL) {
 +        if (pte_present(*pte)) {
 +//printk("lookup_domain_page: found mapping for %lx, pte=%lx\n",mpaddr,pte_val(*pte));
 +            return pte_val(*pte);
 +        } else if (VMX_DOMAIN(d->vcpu[0]))
 +            return GPFN_INV_MASK;
 +    }
 +
 +    printk("%s: d 0x%p id %d current 0x%p id %d\n",
 +           __func__, d, d->domain_id, current, current->vcpu_id);
 +    if ((mpaddr >> PAGE_SHIFT) < d->max_pages)
 +        printk("%s: non-allocated mpa 0x%lx (< 0x%lx)\n", __func__,
 +               mpaddr, (unsigned long)d->max_pages << PAGE_SHIFT);
 +    else
 +        printk("%s: bad mpa 0x%lx (=> 0x%lx)\n", __func__,
 +               mpaddr, (unsigned long)d->max_pages << PAGE_SHIFT);
 +
 +    //XXX This is a work around until the emulation memory access to a region
 +    //    where memory or device are attached is implemented.
 +    return pte_val(pfn_pte(0, __pgprot(__DIRTY_BITS | _PAGE_PL_2 | _PAGE_AR_RWX)));
 +}
 +
 +// FIXME: ONLY USE FOR DOMAIN PAGE_SIZE == PAGE_SIZE
 +#if 1
 +void *domain_mpa_to_imva(struct domain *d, unsigned long mpaddr)
 +{
 +    unsigned long pte = lookup_domain_mpa(d,mpaddr);
 +    unsigned long imva;
 +
 +    pte &= _PAGE_PPN_MASK;
 +    imva = (unsigned long) __va(pte);
 +    imva |= mpaddr & ~PAGE_MASK;
 +    return (void*)imva;
 +}
 +#else
 +void *domain_mpa_to_imva(struct domain *d, unsigned long mpaddr)
 +{
 +    unsigned long imva = __gpa_to_mpa(d, mpaddr);
 +
 +    return (void *)__va(imva);
 +}
 +#endif
 +
 +/* Allocate a new page for domain and map it to the specified metaphysical
 +   address.  */
 +struct page_info *
 +__assign_new_domain_page(struct domain *d, unsigned long mpaddr, pte_t* pte)
 +{
 +    struct page_info *p = NULL;
 +    unsigned long maddr;
 +    int ret;
 +
 +    BUG_ON(!pte_none(*pte));
 +
 +#ifdef CONFIG_DOMAIN0_CONTIGUOUS
 +    if (d == dom0) {
 +#if 0
 +        if (mpaddr < dom0_start || mpaddr >= dom0_start + dom0_size) {
 +            /* FIXME: is it true ?
 +               dom0 memory is not contiguous!  */
 +            panic("assign_new_domain_page: bad domain0 "
 +                  "mpaddr=%lx, start=%lx, end=%lx!\n",
 +                  mpaddr, dom0_start, dom0_start+dom0_size);
 +        }
 +#endif
 +        p = mfn_to_page((mpaddr >> PAGE_SHIFT));
 +        return p;
 +    }
 +#endif
 +
 +    p = alloc_domheap_page(d);
 +    if (unlikely(!p)) {
 +        printf("assign_new_domain_page: Can't alloc!!!! Aaaargh!\n");
 +        return(p);
 +    }
 +
 +    // zero out pages for security reasons
 +    clear_page(page_to_virt(p));
 +    maddr = page_to_maddr (p);
 +    if (unlikely(maddr > __get_cpu_var(vhpt_paddr)
 +                 && maddr < __get_cpu_var(vhpt_pend))) {
 +        /* FIXME: how can this happen ?
 +           vhpt is allocated by alloc_domheap_page.  */
 +        printf("assign_new_domain_page: reassigned vhpt page %lx!!\n",
 +               maddr);
 +    }
 +
 +    ret = get_page(p, d);
 +    BUG_ON(ret == 0);
 +    set_gpfn_from_mfn(page_to_mfn(p), mpaddr >> PAGE_SHIFT);
 +    // clear_page() and set_gpfn_from_mfn() become visible before set_pte_rel()
 +    // because set_pte_rel() has release semantics
 +    set_pte_rel(pte,
 +                pfn_pte(maddr >> PAGE_SHIFT,
 +                        __pgprot(__DIRTY_BITS | _PAGE_PL_2 | _PAGE_AR_RWX)));
 +
 +    smp_mb();
 +    return p;
 +}
 +
 +struct page_info *
 +assign_new_domain_page(struct domain *d, unsigned long mpaddr)
 +{
 +#ifdef CONFIG_DOMAIN0_CONTIGUOUS
 +    pte_t dummy_pte = __pte(0);
 +    return __assign_new_domain_page(d, mpaddr, &dummy_pte);
 +#else
 +    struct page_info *p = NULL;
 +    pte_t *pte;
 +
 +    pte = lookup_alloc_domain_pte(d, mpaddr);
 +    if (pte_none(*pte))
 +        p = __assign_new_domain_page(d, mpaddr, pte);
 +
 +    return p;
 +#endif
 +}
 +
 +void
 +assign_new_domain0_page(struct domain *d, unsigned long mpaddr)
 +{
 +#ifndef CONFIG_DOMAIN0_CONTIGUOUS
 +    pte_t *pte;
 +
 +    BUG_ON(d != dom0);
 +    pte = lookup_alloc_domain_pte(d, mpaddr);
 +    if (pte_none(*pte)) {
 +        struct page_info *p = __assign_new_domain_page(d, mpaddr, pte);
 +        if (p == NULL) {
 +            panic("%s: can't allocate page for dom0", __func__);
 +        }
 +    }
 +#endif
 +}
 +
 +/* map a physical address to the specified metaphysical addr */
 +// flags: currently only ASSIGN_readonly
 +void
 +__assign_domain_page(struct domain *d,
 +                     unsigned long mpaddr, unsigned long physaddr,
 +                     unsigned long flags)
 +{
 +    pte_t *pte;
 +    unsigned long arflags = (flags & ASSIGN_readonly)? _PAGE_AR_R: _PAGE_AR_RWX;
 +
 +    pte = lookup_alloc_domain_pte(d, mpaddr);
 +    if (pte_none(*pte)) {
 +        set_pte_rel(pte,
 +                    pfn_pte(physaddr >> PAGE_SHIFT,
 +                            __pgprot(__DIRTY_BITS | _PAGE_PL_2 | arflags)));
 +        smp_mb();
 +    }
 +}
 +
 +/* get_page() and map a physical address to the specified metaphysical addr */
 +void
 +assign_domain_page(struct domain *d,
 +                   unsigned long mpaddr, unsigned long physaddr)
 +{
 +    struct page_info* page = mfn_to_page(physaddr >> PAGE_SHIFT);
 +    int ret;
 +
 +    BUG_ON((physaddr & GPFN_IO_MASK) != GPFN_MEM);
 +    ret = get_page(page, d);
 +    BUG_ON(ret == 0);
 +    set_gpfn_from_mfn(physaddr >> PAGE_SHIFT, mpaddr >> PAGE_SHIFT);
 +    // because __assign_domain_page() uses set_pte_rel() which has
 +    // release semantics, smp_mb() isn't needed.
 +    __assign_domain_page(d, mpaddr, physaddr, ASSIGN_writable);
 +}
 +
 +#ifdef CONFIG_XEN_IA64_DOM0_VP
 +static void
 +assign_domain_same_page(struct domain *d,
 +                        unsigned long mpaddr, unsigned long size,
 +                        unsigned long flags)
 +{
 +    //XXX optimization
 +    unsigned long end = PAGE_ALIGN(mpaddr + size);
 +    for (mpaddr &= PAGE_MASK; mpaddr < end; mpaddr += PAGE_SIZE) {
 +        __assign_domain_page(d, mpaddr, mpaddr, flags);
 +    }
 +}
 +
 +int
 +efi_mmio(unsigned long physaddr, unsigned long size)
 +{
 +    void *efi_map_start, *efi_map_end;
 +    u64 efi_desc_size;
 +    void* p;
 +
 +    efi_map_start = __va(ia64_boot_param->efi_memmap);
 +    efi_map_end   = efi_map_start + ia64_boot_param->efi_memmap_size;
 +    efi_desc_size = ia64_boot_param->efi_memdesc_size;
 +
 +    for (p = efi_map_start; p < efi_map_end; p += efi_desc_size) {
 +        efi_memory_desc_t* md = (efi_memory_desc_t *)p;
 +        unsigned long start = md->phys_addr;
 +        unsigned long end = md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT);
 +
 +        if (start <= physaddr && physaddr < end) {
 +            if ((physaddr + size) > end) {
 +                DPRINTK("%s:%d physaddr 0x%lx size = 0x%lx\n",
 +                        __func__, __LINE__, physaddr, size);
 +                return 0;
 +            }
 +
 +            // for io space
 +            if (md->type == EFI_MEMORY_MAPPED_IO ||
 +                md->type == EFI_MEMORY_MAPPED_IO_PORT_SPACE) {
 +                return 1;
 +            }
 +
 +            // for runtime
 +            // see efi_enter_virtual_mode(void)
 +            // in linux/arch/ia64/kernel/efi.c
 +            if ((md->attribute & EFI_MEMORY_RUNTIME) &&
 +                !(md->attribute & EFI_MEMORY_WB)) {
 +                return 1;
 +            }
 +
 +            DPRINTK("%s:%d physaddr 0x%lx size = 0x%lx\n",
 +                    __func__, __LINE__, physaddr, size);
 +            return 0;
 +        }
 +
 +        if (physaddr < start) {
 +            break;
 +        }
 +    }
 +
 +    return 1;
 +}
 +
 +unsigned long
 +assign_domain_mmio_page(struct domain *d,
 +                        unsigned long mpaddr, unsigned long size)
 +{
 +    if (size == 0) {
 +        DPRINTK("%s: domain %p mpaddr 0x%lx size = 0x%lx\n",
 +                __func__, d, mpaddr, size);
 +    }
 +    if (!efi_mmio(mpaddr, size)) {
 +        DPRINTK("%s:%d domain %p mpaddr 0x%lx size = 0x%lx\n",
 +                __func__, __LINE__, d, mpaddr, size);
 +        return -EINVAL;
 +    }
 +    assign_domain_same_page(d, mpaddr, size, ASSIGN_writable);
 +    return mpaddr;
 +}
 +
 +unsigned long
 +assign_domain_mach_page(struct domain *d,
 +                        unsigned long mpaddr, unsigned long size,
 +                        unsigned long flags)
 +{
 +    assign_domain_same_page(d, mpaddr, size, flags);
 +    return mpaddr;
 +}
 +
 +// caller must get_page(mfn_to_page(mfn)) before call.
 +// caller must call set_gpfn_from_mfn() before call if necessary.
 +// because set_gpfn_from_mfn() result must be visible before pte xchg
 +// caller must use memory barrier. NOTE: xchg has acquire semantics.
 +// flags: currently only ASSIGN_readonly
 +static void
 +assign_domain_page_replace(struct domain *d, unsigned long mpaddr,
 +                           unsigned long mfn, unsigned long flags)
 +{
 +    struct mm_struct *mm = &d->arch.mm;
 +    pte_t* pte;
 +    pte_t old_pte;
 +    pte_t npte;
 +    unsigned long arflags = (flags & ASSIGN_readonly)? _PAGE_AR_R: _PAGE_AR_RWX;
 +    pte = lookup_alloc_domain_pte(d, mpaddr);
 +
 +    // update pte
 +    npte = pfn_pte(mfn, __pgprot(__DIRTY_BITS | _PAGE_PL_2 | arflags));
 +    old_pte = ptep_xchg(mm, mpaddr, pte, npte);
 +    if (pte_mem(old_pte)) {
 +        unsigned long old_mfn = pte_pfn(old_pte);
 +
 +        // mfn = old_mfn case can happen when domain maps a granted page
 +        // twice with the same pseudo physial address.
 +        // It's non sense, but allowed.
 +        // __gnttab_map_grant_ref()
 +        //   => create_host_mapping()
 +        //      => assign_domain_page_replace()
 +        if (mfn != old_mfn) {
 +            struct page_info* old_page = mfn_to_page(old_mfn);
 +
 +            if (page_get_owner(old_page) == d) {
 +                BUG_ON(get_gpfn_from_mfn(old_mfn) != (mpaddr >> PAGE_SHIFT));
 +                set_gpfn_from_mfn(old_mfn, INVALID_M2P_ENTRY);
 +            }
 +
 +            domain_page_flush(d, mpaddr, old_mfn, mfn);
 +
 +            try_to_clear_PGC_allocate(d, old_page);
 +            put_page(old_page);
 +        }
 +    }
 +}
 +
 +// caller must get_page(new_page) before
- // Only steal_page_for_grant_transfer() calls this function.
++// Only steal_page() calls this function.
 +static int
 +assign_domain_page_cmpxchg_rel(struct domain* d, unsigned long mpaddr,
 +                               struct page_info* old_page,
 +                               struct page_info* new_page,
 +                               unsigned long flags)
 +{
 +    struct mm_struct *mm = &d->arch.mm;
 +    pte_t* pte;
 +    unsigned long old_mfn;
 +    unsigned long old_arflags;
 +    pte_t old_pte;
 +    unsigned long new_mfn;
 +    unsigned long new_arflags;
 +    pte_t new_pte;
 +    pte_t ret_pte;
 +
 +    pte = lookup_alloc_domain_pte(d, mpaddr);
 +
 + again:
 +    old_arflags = pte_val(*pte) & ~_PAGE_PPN_MASK;//XXX
 +    old_mfn = page_to_mfn(old_page);
 +    old_pte = pfn_pte(old_mfn, __pgprot(old_arflags));
 +
 +    new_arflags = (flags & ASSIGN_readonly)? _PAGE_AR_R: _PAGE_AR_RWX;
 +    new_mfn = page_to_mfn(new_page);
 +    new_pte = pfn_pte(new_mfn,
 +                      __pgprot(__DIRTY_BITS | _PAGE_PL_2 | new_arflags));
 +
 +    // update pte
 +    ret_pte = ptep_cmpxchg_rel(mm, mpaddr, pte, old_pte, new_pte);
 +    if (unlikely(pte_val(old_pte) != pte_val(ret_pte))) {
 +        if (pte_pfn(old_pte) == pte_pfn(ret_pte)) {
 +            goto again;
 +        }
 +
 +        DPRINTK("%s: old_pte 0x%lx old_arflags 0x%lx old_mfn 0x%lx "
 +                "ret_pte 0x%lx ret_mfn 0x%lx\n",
 +                __func__,
 +                pte_val(old_pte), old_arflags, old_mfn,
 +                pte_val(ret_pte), pte_pfn(ret_pte));
 +        return -EINVAL;
 +    }
 +
 +    BUG_ON(!pte_mem(old_pte));
 +    BUG_ON(page_get_owner(old_page) != d);
 +    BUG_ON(get_gpfn_from_mfn(old_mfn) != (mpaddr >> PAGE_SHIFT));
 +    BUG_ON(old_mfn == new_mfn);
 +
 +    set_gpfn_from_mfn(old_mfn, INVALID_M2P_ENTRY);
 +
 +    domain_page_flush(d, mpaddr, old_mfn, new_mfn);
 +    put_page(old_page);
 +    return 0;
 +}
 +
 +static void
 +zap_domain_page_one(struct domain *d, unsigned long mpaddr)
 +{
 +    struct mm_struct *mm = &d->arch.mm;
 +    pte_t *pte;
 +    pte_t old_pte;
 +    unsigned long mfn;
 +    struct page_info *page;
 +
 +    pte = lookup_noalloc_domain_pte_none(d, mpaddr);
 +    if (pte == NULL)
 +        return;
 +    if (pte_none(*pte))
 +        return;
 +
 +    // update pte
 +    old_pte = ptep_get_and_clear(mm, mpaddr, pte);
 +    mfn = pte_pfn(old_pte);
 +    page = mfn_to_page(mfn);
 +    BUG_ON((page->count_info & PGC_count_mask) == 0);
 +
 +    if (page_get_owner(page) == d) {
 +        BUG_ON(get_gpfn_from_mfn(mfn) != (mpaddr >> PAGE_SHIFT));
 +        set_gpfn_from_mfn(mfn, INVALID_M2P_ENTRY);
 +    }
 +
 +    domain_page_flush(d, mpaddr, mfn, INVALID_MFN);
 +
 +    try_to_clear_PGC_allocate(d, page);
 +    put_page(page);
 +}
 +
 +unsigned long
 +dom0vp_zap_physmap(struct domain *d, unsigned long gpfn,
 +                   unsigned int extent_order)
 +{
 +    if (extent_order != 0) {
 +        //XXX
 +        return -ENOSYS;
 +    }
 +
 +    zap_domain_page_one(d, gpfn << PAGE_SHIFT);
 +    return 0;
 +}
 +
 +unsigned long
 +dom0vp_add_physmap(struct domain* d, unsigned long gpfn, unsigned long mfn,
 +                   unsigned long flags, domid_t domid)
 +{
 +    int error = 0;
 +    struct domain* rd;
 +
 +    rd = find_domain_by_id(domid);
 +    if (unlikely(rd == NULL)) {
 +        switch (domid) {
 +        case DOMID_XEN:
 +            rd = dom_xen;
 +            break;
 +        case DOMID_IO:
 +            rd = dom_io;
 +            break;
 +        default:
 +            DPRINTK("d 0x%p domid %d "
 +                    "pgfn 0x%lx mfn 0x%lx flags 0x%lx domid %d\n",
 +                    d, d->domain_id, gpfn, mfn, flags, domid);
 +            return -ESRCH;
 +        }
 +        BUG_ON(rd == NULL);
 +        get_knownalive_domain(rd);
 +    }
 +
 +    if (unlikely(rd == d)) {
 +        error = -EINVAL;
 +        goto out1;
 +    }
 +    BUG_ON(!mfn_valid(mfn));
 +    if (unlikely(get_page(mfn_to_page(mfn), rd) == 0)) {
 +        error = -EINVAL;
 +        goto out1;
 +    }
 +    BUG_ON(page_get_owner(mfn_to_page(mfn)) == d &&
 +           get_gpfn_from_mfn(mfn) != INVALID_M2P_ENTRY);
 +    assign_domain_page_replace(d, gpfn << PAGE_SHIFT, mfn, flags);
 +    //don't update p2m table because this page belongs to rd, not d.
 +out1:
 +    put_domain(rd);
 +    return error;
 +}
 +
 +// grant table host mapping
 +// mpaddr: host_addr: pseudo physical address
 +// mfn: frame: machine page frame
 +// flags: GNTMAP_readonly | GNTMAP_application_map | GNTMAP_contains_pte
 +int
 +create_grant_host_mapping(unsigned long gpaddr,
 +              unsigned long mfn, unsigned int flags)
 +{
 +    struct domain* d = current->domain;
 +    struct page_info* page;
 +    int ret;
 +
 +    if (flags & (GNTMAP_device_map |
 +                 GNTMAP_application_map | GNTMAP_contains_pte)) {
 +        DPRINTK("%s: flags 0x%x\n", __func__, flags);
 +        return GNTST_general_error;
 +    }
 +
 +    BUG_ON(!mfn_valid(mfn));
 +    page = mfn_to_page(mfn);
 +    ret = get_page(page, page_get_owner(page));
 +    BUG_ON(ret == 0);
 +    BUG_ON(page_get_owner(mfn_to_page(mfn)) == d &&
 +           get_gpfn_from_mfn(mfn) != INVALID_M2P_ENTRY);
 +    assign_domain_page_replace(d, gpaddr, mfn, (flags & GNTMAP_readonly)?
 +                                              ASSIGN_readonly: ASSIGN_writable);
 +    return GNTST_okay;
 +}
 +
 +// grant table host unmapping
 +int
 +destroy_grant_host_mapping(unsigned long gpaddr,
 +               unsigned long mfn, unsigned int flags)
 +{
 +    struct domain* d = current->domain;
 +    pte_t* pte;
 +    pte_t old_pte;
 +    unsigned long old_mfn = INVALID_MFN;
 +    struct page_info* old_page;
 +
 +    if (flags & (GNTMAP_application_map | GNTMAP_contains_pte)) {
 +        DPRINTK("%s: flags 0x%x\n", __func__, flags);
 +        return GNTST_general_error;
 +    }
 +
 +    pte = lookup_noalloc_domain_pte(d, gpaddr);
 +    if (pte == NULL || !pte_present(*pte) || pte_pfn(*pte) != mfn)
 +        return GNTST_general_error;
 +
 +    // update pte
 +    old_pte = ptep_get_and_clear(&d->arch.mm, gpaddr, pte);
 +    if (pte_present(old_pte)) {
 +        old_mfn = pte_pfn(old_pte);
 +    } else {
 +        return GNTST_general_error;
 +    }
 +    domain_page_flush(d, gpaddr, old_mfn, INVALID_MFN);
 +
 +    old_page = mfn_to_page(old_mfn);
 +    BUG_ON(page_get_owner(old_page) == d);//try_to_clear_PGC_allocate(d, page) is not needed.
 +    put_page(old_page);
 +
 +    return GNTST_okay;
 +}
 +
 +// heavily depends on the struct page layout.
 +int
- steal_page_for_grant_transfer(struct domain *d, struct page_info *page)
++steal_page(struct domain *d, struct page_info *page, unsigned int memflags)
 +{
 +#if 0 /* if big endian */
- # error "implement big endian version of steal_page_for_grant_transfer()"
++# error "implement big endian version of steal_page()"
 +#endif
 +    u32 _d, _nd;
 +    u64 x, nx, y;
 +    unsigned long gpfn;
 +    struct page_info *new;
 +    unsigned long new_mfn;
 +    int ret;
 +    new = alloc_domheap_page(d);
 +    if (new == NULL) {
 +        DPRINTK("alloc_domheap_page() failed\n");
 +        return -1;
 +    }
 +    // zero out pages for security reasons
 +    clear_page(page_to_virt(new));
 +    // assign_domain_page_cmpxchg_rel() has release semantics
 +    // so smp_mb() isn't needed.
 +
 +    ret = get_page(new, d);
 +    BUG_ON(ret == 0);
 +
 +    gpfn = get_gpfn_from_mfn(page_to_mfn(page));
 +    if (gpfn == INVALID_M2P_ENTRY) {
 +        free_domheap_page(new);
 +        return -1;
 +    }
 +    new_mfn = page_to_mfn(new);
 +    set_gpfn_from_mfn(new_mfn, gpfn);
 +    // smp_mb() isn't needed because assign_domain_pge_cmpxchg_rel()
 +    // has release semantics.
 +
 +    ret = assign_domain_page_cmpxchg_rel(d, gpfn << PAGE_SHIFT, page, new,
 +                                         ASSIGN_writable);
 +    if (ret < 0) {
 +        DPRINTK("assign_domain_page_cmpxchg_rel failed %d\n", ret);
 +        set_gpfn_from_mfn(new_mfn, INVALID_M2P_ENTRY);
 +        free_domheap_page(new);
 +        return -1;
 +    }
 +
 +    spin_lock(&d->page_alloc_lock);
 +
 +    /*
 +     * The tricky bit: atomically release ownership while there is just one
 +     * benign reference to the page (PGC_allocated). If that reference
 +     * disappears then the deallocation routine will safely spin.
 +     */
 +    _d  = pickle_domptr(d);
 +    y = *((u64*)&page->count_info);
 +    do {
 +        x = y;
 +        nx = x & 0xffffffff;
 +        // page->count_info: untouched
 +        // page->u.inused._domain = 0;
 +        _nd = x >> 32;
 +
 +        if (unlikely((x & (PGC_count_mask | PGC_allocated)) !=
 +                     (1 | PGC_allocated)) ||
 +            unlikely(_nd != _d)) {
 +            struct domain* nd = unpickle_domptr(_nd);
 +            if (nd == NULL) {
 +                DPRINTK("gnttab_transfer: Bad page %p: ed=%p(%u) 0x%x, "
 +                        "sd=%p 0x%x,"
 +                        " caf=%016lx, taf=%" PRtype_info "\n",
 +                        (void *) page_to_mfn(page),
 +                        d, d->domain_id, _d,
 +                        nd, _nd,
 +                        x,
 +                        page->u.inuse.type_info);
 +            } else {
 +                DPRINTK("gnttab_transfer: Bad page %p: ed=%p(%u) 0x%x, "
 +                        "sd=%p(%u) 0x%x,"
 +                        " caf=%016lx, taf=%" PRtype_info "\n",
 +                        (void *) page_to_mfn(page),
 +                        d, d->domain_id, _d,
 +                        nd, nd->domain_id, _nd,
 +                        x,
 +                        page->u.inuse.type_info);
 +            }
 +            spin_unlock(&d->page_alloc_lock);
 +            return -1;
 +        }
 +
 +        y = cmpxchg((u64*)&page->count_info, x, nx);
 +    } while (unlikely(y != x));
 +
 +    /*
 +     * Unlink from 'd'. At least one reference remains (now anonymous), so
 +     * noone else is spinning to try to delete this page from 'd'.
 +     */
-     d->tot_pages--;
++    if ( !(memflags & MEMF_no_refcount) )
++        d->tot_pages--;
 +    list_del(&page->list);
 +
 +    spin_unlock(&d->page_alloc_lock);
 +    return 0;
 +}
 +
 +void
 +guest_physmap_add_page(struct domain *d, unsigned long gpfn,
 +                       unsigned long mfn)
 +{
 +    int ret;
 +
 +    BUG_ON(!mfn_valid(mfn));
 +    ret = get_page(mfn_to_page(mfn), d);
 +    BUG_ON(ret == 0);
 +    BUG_ON(page_get_owner(mfn_to_page(mfn)) == d &&
 +           get_gpfn_from_mfn(mfn) != INVALID_M2P_ENTRY);
 +    set_gpfn_from_mfn(mfn, gpfn);
 +    smp_mb();
 +    assign_domain_page_replace(d, gpfn << PAGE_SHIFT, mfn, ASSIGN_writable);
 +
 +    //BUG_ON(mfn != ((lookup_domain_mpa(d, gpfn << PAGE_SHIFT) & _PFN_MASK) >> PAGE_SHIFT));
 +}
 +
 +void
 +guest_physmap_remove_page(struct domain *d, unsigned long gpfn,
 +                          unsigned long mfn)
 +{
 +    BUG_ON(mfn == 0);//XXX
 +    zap_domain_page_one(d, gpfn << PAGE_SHIFT);
 +}
 +
 +//XXX sledgehammer.
 +//    flush finer range.
 +void
 +domain_page_flush(struct domain* d, unsigned long mpaddr,
 +                  unsigned long old_mfn, unsigned long new_mfn)
 +{
 +    domain_flush_vtlb_all();
 +}
 +
 +int
 +domain_page_mapped(struct domain* d, unsigned long mpaddr)
 +{
 +    pte_t * pte;
 +
 +    pte = lookup_noalloc_domain_pte(d, mpaddr);
 +    if(pte != NULL && !pte_none(*pte))
 +       return 1;
 +    return 0;
 +}
 +#endif
 +
 +/* Flush cache of domain d.  */
 +void domain_cache_flush (struct domain *d, int sync_only)
 +{
 +    struct mm_struct *mm = &d->arch.mm;
 +    pgd_t *pgd = mm->pgd;
 +    unsigned long maddr;
 +    int i,j,k, l;
 +    int nbr_page = 0;
 +    void (*flush_func)(unsigned long start, unsigned long end);
 +    extern void flush_dcache_range (unsigned long, unsigned long);
 +
 +    if (sync_only)
 +        flush_func = &flush_icache_range;
 +    else
 +        flush_func = &flush_dcache_range;
 +
 +#ifdef CONFIG_DOMAIN0_CONTIGUOUS
 +    if (d == dom0) {
 +        /* This is not fully correct (because of hole), but it should
 +           be enough for now.  */
 +        (*flush_func)(__va_ul (dom0_start),
 +                  __va_ul (dom0_start + dom0_size));
 +        return;
 +    }
 +#endif
 +    for (i = 0; i < PTRS_PER_PGD; pgd++, i++) {
 +        pud_t *pud;
 +        if (!pgd_present(*pgd))
 +            continue;
 +        pud = pud_offset(pgd, 0);
 +        for (j = 0; j < PTRS_PER_PUD; pud++, j++) {
 +            pmd_t *pmd;
 +            if (!pud_present(*pud))
 +                continue;
 +            pmd = pmd_offset(pud, 0);
 +            for (k = 0; k < PTRS_PER_PMD; pmd++, k++) {
 +                pte_t *pte;
 +                if (!pmd_present(*pmd))
 +                    continue;
 +                pte = pte_offset_map(pmd, 0);
 +                for (l = 0; l < PTRS_PER_PTE; pte++, l++) {
 +                    if (!pte_present(*pte))
 +                        continue;
 +                    /* Convert PTE to maddr.  */
 +                    maddr = __va_ul (pte_val(*pte)
 +                             & _PAGE_PPN_MASK);
 +                    (*flush_func)(maddr, maddr+ PAGE_SIZE);
 +                    nbr_page++;
 +                }
 +            }
 +        }
 +    }
 +    //printf ("domain_cache_flush: %d %d pages\n", d->domain_id, nbr_page);
 +}
 +
 +#ifdef VERBOSE
 +#define MEM_LOG(_f, _a...)                           \
 +  printk("DOM%u: (file=mm.c, line=%d) " _f "\n", \
 +         current->domain->domain_id , __LINE__ , ## _a )
 +#else
 +#define MEM_LOG(_f, _a...) ((void)0)
 +#endif
 +
 +static void free_page_type(struct page_info *page, u32 type)
 +{
 +}
 +
 +static int alloc_page_type(struct page_info *page, u32 type)
 +{
 +	return 1;
 +}
 +
 +unsigned long __get_free_pages(unsigned int mask, unsigned int order)
 +{
 +	void *p = alloc_xenheap_pages(order);
 +
 +	memset(p,0,PAGE_SIZE<<order);
 +	return (unsigned long)p;
 +}
 +
 +void __free_pages(struct page_info *page, unsigned int order)
 +{
 +	if (order) BUG();
 +	free_xenheap_page(page);
 +}
 +
 +void *pgtable_quicklist_alloc(void)
 +{
 +    void *p;
 +    p = alloc_xenheap_pages(0);
 +    if (p)
 +        clear_page(p);
 +    return p;
 +}
 +
 +void pgtable_quicklist_free(void *pgtable_entry)
 +{
 +	free_xenheap_page(pgtable_entry);
 +}
 +
 +void cleanup_writable_pagetable(struct domain *d)
 +{
 +  return;
 +}
 +
 +void put_page_type(struct page_info *page)
 +{
 +    u32 nx, x, y = page->u.inuse.type_info;
 +
 + again:
 +    do {
 +        x  = y;
 +        nx = x - 1;
 +
 +        ASSERT((x & PGT_count_mask) != 0);
 +
 +        /*
 +         * The page should always be validated while a reference is held. The
 +         * exception is during domain destruction, when we forcibly invalidate
 +         * page-table pages if we detect a referential loop.
 +         * See domain.c:relinquish_list().
 +         */
 +        ASSERT((x & PGT_validated) ||
 +               test_bit(_DOMF_dying, &page_get_owner(page)->domain_flags));
 +
 +        if ( unlikely((nx & PGT_count_mask) == 0) )
 +        {
 +            /* Record TLB information for flush later. Races are harmless. */
 +            page->tlbflush_timestamp = tlbflush_current_time();
 +
 +            if ( unlikely((nx & PGT_type_mask) <= PGT_l4_page_table) &&
 +                 likely(nx & PGT_validated) )
 +            {
 +                /*
 +                 * Page-table pages must be unvalidated when count is zero. The
 +                 * 'free' is safe because the refcnt is non-zero and validated
 +                 * bit is clear => other ops will spin or fail.
 +                 */
 +                if ( unlikely((y = cmpxchg(&page->u.inuse.type_info, x,
 +                                           x & ~PGT_validated)) != x) )
 +                    goto again;
 +                /* We cleared the 'valid bit' so we do the clean up. */
 +                free_page_type(page, x);
 +                /* Carry on, but with the 'valid bit' now clear. */
 +                x  &= ~PGT_validated;
 +                nx &= ~PGT_validated;
 +            }
 +        }
 +        else if ( unlikely(((nx & (PGT_pinned | PGT_count_mask)) ==
 +                            (PGT_pinned | 1)) &&
 +                           ((nx & PGT_type_mask) != PGT_writable_page)) )
 +        {
 +            /* Page is now only pinned. Make the back pointer mutable again. */
 +            nx |= PGT_va_mutable;
 +        }
 +    }
 +    while ( unlikely((y = cmpxchg_rel(&page->u.inuse.type_info, x, nx)) != x) );
 +}
 +
 +
 +int get_page_type(struct page_info *page, u32 type)
 +{
 +    u32 nx, x, y = page->u.inuse.type_info;
 +
 + again:
 +    do {
 +        x  = y;
 +        nx = x + 1;
 +        if ( unlikely((nx & PGT_count_mask) == 0) )
 +        {
 +            MEM_LOG("Type count overflow on pfn %lx", page_to_mfn(page));
 +            return 0;
 +        }
 +        else if ( unlikely((x & PGT_count_mask) == 0) )
 +        {
 +            if ( (x & (PGT_type_mask|PGT_va_mask)) != type )
 +            {
 +                if ( (x & PGT_type_mask) != (type & PGT_type_mask) )
 +                {
 +                    /*
 +                     * On type change we check to flush stale TLB
 +                     * entries. This may be unnecessary (e.g., page
 +                     * was GDT/LDT) but those circumstances should be
 +                     * very rare.
 +                     */
 +                    cpumask_t mask =
 +                        page_get_owner(page)->domain_dirty_cpumask;
 +                    tlbflush_filter(mask, page->tlbflush_timestamp);
 +
 +                    if ( unlikely(!cpus_empty(mask)) )
 +                    {
 +                        perfc_incrc(need_flush_tlb_flush);
 +                        flush_tlb_mask(mask);
 +                    }
 +                }
 +
 +                /* We lose existing type, back pointer, and validity. */
 +                nx &= ~(PGT_type_mask | PGT_va_mask | PGT_validated);
 +                nx |= type;
 +
 +                /* No special validation needed for writable pages. */
 +                /* Page tables and GDT/LDT need to be scanned for validity. */
 +                if ( type == PGT_writable_page )
 +                    nx |= PGT_validated;
 +            }
 +        }
 +        else
 +        {
 +            if ( unlikely((x & (PGT_type_mask|PGT_va_mask)) != type) )
 +            {
 +                if ( unlikely((x & PGT_type_mask) != (type & PGT_type_mask) ) )
 +                {
 +                    if ( current->domain == page_get_owner(page) )
 +                    {
 +                        /*
 +                         * This ensures functions like set_gdt() see up-to-date
 +                         * type info without needing to clean up writable p.t.
 +                         * state on the fast path.
 +                         */
 +                        LOCK_BIGLOCK(current->domain);
 +                        cleanup_writable_pagetable(current->domain);
 +                        y = page->u.inuse.type_info;
 +                        UNLOCK_BIGLOCK(current->domain);
 +                        /* Can we make progress now? */
 +                        if ( ((y & PGT_type_mask) == (type & PGT_type_mask)) ||
 +                             ((y & PGT_count_mask) == 0) )
 +                            goto again;
 +                    }
 +                    if ( ((x & PGT_type_mask) != PGT_l2_page_table) ||
 +                         ((type & PGT_type_mask) != PGT_l1_page_table) )
 +                        MEM_LOG("Bad type (saw %08x != exp %08x) "
 +                                "for mfn %016lx (pfn %016lx)",
 +                                x, type, page_to_mfn(page),
 +                                get_gpfn_from_mfn(page_to_mfn(page)));
 +                    return 0;
 +                }
 +                else if ( (x & PGT_va_mask) == PGT_va_mutable )
 +                {
 +                    /* The va backpointer is mutable, hence we update it. */
 +                    nx &= ~PGT_va_mask;
 +                    nx |= type; /* we know the actual type is correct */
 +                }
 +                else if ( ((type & PGT_va_mask) != PGT_va_mutable) &&
 +                          ((type & PGT_va_mask) != (x & PGT_va_mask)) )
 +                {
 +#ifdef CONFIG_X86_PAE
 +                    /* We use backptr as extra typing. Cannot be unknown. */
 +                    if ( (type & PGT_type_mask) == PGT_l2_page_table )
 +                        return 0;
 +#endif
 +                    /* This table is possibly mapped at multiple locations. */
 +                    nx &= ~PGT_va_mask;
 +                    nx |= PGT_va_unknown;
 +                }
 +            }
 +            if ( unlikely(!(x & PGT_validated)) )
 +            {
 +                /* Someone else is updating validation of this page. Wait... */
 +                while ( (y = page->u.inuse.type_info) == x )
 +                    cpu_relax();
 +                goto again;
 +            }
 +        }
 +    }
 +    while ( unlikely((y = cmpxchg_acq(&page->u.inuse.type_info, x, nx)) != x) );
 +
 +    if ( unlikely(!(nx & PGT_validated)) )
 +    {
 +        /* Try to validate page type; drop the new reference on failure. */
 +        if ( unlikely(!alloc_page_type(page, type)) )
 +        {
 +            MEM_LOG("Error while validating mfn %lx (pfn %lx) for type %08x"
 +                    ": caf=%08x taf=%" PRtype_info,
 +                    page_to_mfn(page), get_gpfn_from_mfn(page_to_mfn(page)),
 +                    type, page->count_info, page->u.inuse.type_info);
 +            /* Noone else can get a reference. We hold the only ref. */
 +            page->u.inuse.type_info = 0;
 +            return 0;
 +        }
 +
 +        /* Noone else is updating simultaneously. */
 +        __set_bit(_PGT_validated, &page->u.inuse.type_info);
 +    }
 +
 +    return 1;
 +}
 +
 +/*
 + * Local variables:
 + * mode: C
 + * c-set-style: "BSD"
 + * c-basic-offset: 4
 + * tab-width: 4
 + * indent-tabs-mode: nil
 + * End:
 + */