//XXX heavily depends on the struct page layout.
//XXX SMP
int
-steal_page_for_grant_transfer(struct domain *d, struct page_info *page)
+steal_page(struct domain *d, struct page_info *page, unsigned int memflags)
{
#if 0 /* if big endian */
-# error "implement big endian version of steal_page_for_grant_transfer()"
+# error "implement big endian version of steal_page()"
#endif
u32 _d, _nd;
u64 x, nx, y;
* Unlink from 'd'. At least one reference remains (now anonymous), so
* noone else is spinning to try to delete this page from 'd'.
*/
- d->tot_pages--;
+ if ( !(memflags & MEMF_no_refcount) )
+ d->tot_pages--;
list_del(&page->list);
spin_unlock(&d->page_alloc_lock);
* Allocate from DMA pool: on i386 this ensures that our low-memory 1:1
* mapping covers the allocation.
*/
- if ( (page = alloc_domheap_pages(d, order, ALLOC_DOM_DMA)) == NULL )
+ if ( (page = alloc_domheap_pages(d, order, MEMF_dma)) == NULL )
panic("Not enough RAM for domain 0 allocation.\n");
alloc_spfn = page_to_mfn(page);
alloc_epfn = alloc_spfn + d->tot_pages;
return destroy_grant_va_mapping(addr, frame);
}
-int steal_page_for_grant_transfer(
- struct domain *d, struct page_info *page)
+int steal_page(
+ struct domain *d, struct page_info *page, unsigned int memflags)
{
u32 _d, _nd, x, y;
* Unlink from 'd'. At least one reference remains (now anonymous), so
* noone else is spinning to try to delete this page from 'd'.
*/
- d->tot_pages--;
+ if ( !(memflags & MEMF_no_refcount) )
+ d->tot_pages--;
list_del(&page->list);
spin_unlock(&d->page_alloc_lock);
#elif CONFIG_PAGING_LEVELS >= 3
if ( d->arch.ops->guest_paging_levels == PAGING_L2 &&
psh_type == PGT_l4_shadow ) /* allocated for PAE PDP page */
- page = alloc_domheap_pages(NULL, 0, ALLOC_DOM_DMA);
+ page = alloc_domheap_pages(NULL, 0, MEMF_dma);
else if ( d->arch.ops->guest_paging_levels == PAGING_L3 &&
(psh_type == PGT_l3_shadow || psh_type == PGT_l4_shadow) )
- page = alloc_domheap_pages(NULL, 0, ALLOC_DOM_DMA); /* allocated for PAE PDP page */
+ page = alloc_domheap_pages(NULL, 0, MEMF_dma); /* allocated for PAE PDP page */
else
page = alloc_domheap_page(NULL);
#endif
struct page_info *page;
l3_pgentry_t *root;
- if ( !(page = alloc_domheap_pages(NULL, 0, ALLOC_DOM_DMA)) )
+ if ( !(page = alloc_domheap_pages(NULL, 0, MEMF_dma)) )
return 0;
root = map_domain_page(page_to_mfn(page));
ASSERT(!pagetable_get_paddr(v->arch.monitor_table)); /* we should only get called once */
- m3mfn_info = alloc_domheap_pages(NULL, 0, ALLOC_DOM_DMA);
+ m3mfn_info = alloc_domheap_pages(NULL, 0, MEMF_dma);
ASSERT( m3mfn_info );
m3mfn = page_to_mfn(m3mfn_info);
goto copyback;
}
- if ( steal_page_for_grant_transfer(d, page) < 0 )
+ if ( steal_page(d, page, 0) < 0 )
{
gop.status = GNTST_bad_page;
goto copyback;
XEN_GUEST_HANDLE(xen_pfn_t) extent_list,
unsigned int nr_extents,
unsigned int extent_order,
- unsigned int flags,
+ unsigned int memflags,
int *preempted)
{
struct page_info *page;
}
if ( unlikely((page = alloc_domheap_pages(
- d, extent_order, flags)) == NULL) )
+ d, extent_order, memflags)) == NULL) )
{
DPRINTK("Could not allocate order=%d extent: "
- "id=%d flags=%x (%ld of %d)\n",
- extent_order, d->domain_id, flags, i, nr_extents);
+ "id=%d memflags=%x (%ld of %d)\n",
+ extent_order, d->domain_id, memflags, i, nr_extents);
return i;
}
XEN_GUEST_HANDLE(xen_pfn_t) extent_list,
unsigned int nr_extents,
unsigned int extent_order,
- unsigned int flags,
+ unsigned int memflags,
int *preempted)
{
struct page_info *page;
goto out;
if ( unlikely((page = alloc_domheap_pages(
- d, extent_order, flags)) == NULL) )
+ d, extent_order, memflags)) == NULL) )
{
DPRINTK("Could not allocate order=%d extent: "
- "id=%d flags=%x (%ld of %d)\n",
- extent_order, d->domain_id, flags, i, nr_extents);
+ "id=%d memflags=%x (%ld of %d)\n",
+ extent_order, d->domain_id, memflags, i, nr_extents);
goto out;
}
XEN_GUEST_HANDLE(xen_pfn_t) extent_list,
unsigned int nr_extents,
unsigned int extent_order,
- unsigned int flags,
int *preempted)
{
unsigned long i, j;
return 0;
}
+static long
+memory_exchange(XEN_GUEST_HANDLE(xen_memory_exchange_t) arg)
+{
+ struct xen_memory_exchange exch;
+ LIST_HEAD(in_chunk_list);
+ LIST_HEAD(out_chunk_list);
+ unsigned long in_chunk_order, out_chunk_order;
+ unsigned long gpfn, gmfn, mfn;
+ unsigned long i, j, k;
+ unsigned int memflags = 0;
+ long rc = 0;
+ struct domain *d;
+ struct page_info *page;
+
+ if ( copy_from_guest(&exch, arg, 1) )
+ return -EFAULT;
+
+ /* Various sanity checks. */
+ if ( (exch.nr_exchanged > exch.in.nr_extents) ||
+ /* Input and output domain identifiers match? */
+ (exch.in.domid != exch.out.domid) ||
+ /* Sizes of input and output lists do not overflow a long? */
+ ((~0UL >> exch.in.extent_order) < exch.in.nr_extents) ||
+ ((~0UL >> exch.out.extent_order) < exch.out.nr_extents) ||
+ /* Sizes of input and output lists match? */
+ ((exch.in.nr_extents << exch.in.extent_order) !=
+ (exch.out.nr_extents << exch.out.extent_order)) )
+ {
+ rc = -EINVAL;
+ goto fail_early;
+ }
+
+ /* Only privileged guests can allocate multi-page contiguous extents. */
+ if ( ((exch.in.extent_order != 0) || (exch.out.extent_order != 0)) &&
+ !multipage_allocation_permitted(current->domain) )
+ {
+ rc = -EPERM;
+ goto fail_early;
+ }
+
+ if ( (exch.out.address_bits != 0) &&
+ (exch.out.address_bits <
+ (get_order_from_pages(max_page) + PAGE_SHIFT)) )
+ {
+ if ( exch.out.address_bits < 31 )
+ {
+ rc = -ENOMEM;
+ goto fail_early;
+ }
+ memflags = MEMF_dma;
+ }
+
+ guest_handle_add_offset(exch.in.extent_start, exch.nr_exchanged);
+ exch.in.nr_extents -= exch.nr_exchanged;
+
+ if ( exch.in.extent_order <= exch.out.extent_order )
+ {
+ in_chunk_order = exch.out.extent_order - exch.in.extent_order;
+ out_chunk_order = 0;
+ guest_handle_add_offset(
+ exch.out.extent_start, exch.nr_exchanged >> in_chunk_order);
+ exch.out.nr_extents -= exch.nr_exchanged >> in_chunk_order;
+ }
+ else
+ {
+ in_chunk_order = 0;
+ out_chunk_order = exch.in.extent_order - exch.out.extent_order;
+ guest_handle_add_offset(
+ exch.out.extent_start, exch.nr_exchanged << out_chunk_order);
+ exch.out.nr_extents -= exch.nr_exchanged << out_chunk_order;
+ }
+
+ /*
+ * Only support exchange on calling domain right now. Otherwise there are
+ * tricky corner cases to consider (e.g., DOMF_dying domain).
+ */
+ if ( unlikely(exch.in.domid != DOMID_SELF) )
+ {
+ rc = IS_PRIV(current->domain) ? -EINVAL : -EPERM;
+ goto fail_early;
+ }
+ d = current->domain;
+
+ for ( i = 0; i < (exch.in.nr_extents >> in_chunk_order); i++ )
+ {
+ if ( hypercall_preempt_check() )
+ {
+ exch.nr_exchanged += i << in_chunk_order;
+ if ( copy_field_to_guest(arg, &exch, nr_exchanged) )
+ return -EFAULT;
+ return hypercall_create_continuation(
+ __HYPERVISOR_memory_op, "lh", XENMEM_exchange, arg);
+ }
+
+ /* Steal a chunk's worth of input pages from the domain. */
+ for ( j = 0; j < (1UL << in_chunk_order); j++ )
+ {
+ if ( unlikely(__copy_from_guest_offset(
+ &gmfn, exch.in.extent_start, (i<<in_chunk_order)+j, 1)) )
+ {
+ rc = -EFAULT;
+ goto fail;
+ }
+
+ for ( k = 0; k < (1UL << exch.in.extent_order); k++ )
+ {
+ mfn = gmfn_to_mfn(d, gmfn + k);
+ if ( unlikely(!mfn_valid(mfn)) )
+ {
+ rc = -EINVAL;
+ goto fail;
+ }
+
+ page = mfn_to_page(mfn);
+
+ if ( unlikely(steal_page(d, page, MEMF_no_refcount)) )
+ {
+ rc = -EINVAL;
+ goto fail;
+ }
+
+ list_add(&page->list, &in_chunk_list);
+ }
+ }
+
+ /* Allocate a chunk's worth of anonymous output pages. */
+ for ( j = 0; j < (1UL << out_chunk_order); j++ )
+ {
+ page = alloc_domheap_pages(
+ NULL, exch.out.extent_order, memflags);
+ if ( unlikely(page == NULL) )
+ {
+ rc = -ENOMEM;
+ goto fail;
+ }
+
+ list_add(&page->list, &out_chunk_list);
+ }
+
+ /*
+ * Success! Beyond this point we cannot fail for this chunk.
+ */
+
+ /* Destroy final reference to each input page. */
+ while ( !list_empty(&in_chunk_list) )
+ {
+ page = list_entry(in_chunk_list.next, struct page_info, list);
+ list_del(&page->list);
+ if ( !test_and_clear_bit(_PGC_allocated, &page->count_info) )
+ BUG();
+ mfn = page_to_mfn(page);
+ guest_physmap_remove_page(d, mfn_to_gmfn(d, mfn), mfn);
+ put_page(page);
+ }
+
+ /* Assign each output page to the domain. */
+ j = 0;
+ while ( !list_empty(&out_chunk_list) )
+ {
+ page = list_entry(out_chunk_list.next, struct page_info, list);
+ list_del(&page->list);
+ if ( assign_pages(d, page, exch.out.extent_order,
+ MEMF_no_refcount) )
+ BUG();
+
+ /* Note that we ignore errors accessing the output extent list. */
+ (void)__copy_from_guest_offset(
+ &gpfn, exch.out.extent_start, (i<<out_chunk_order)+j, 1);
+
+ mfn = page_to_mfn(page);
+ if ( unlikely(shadow_mode_translate(d)) )
+ {
+ for ( k = 0; k < (1UL << exch.out.extent_order); k++ )
+ guest_physmap_add_page(d, gpfn + k, mfn + k);
+ }
+ else
+ {
+ for ( k = 0; k < (1UL << exch.out.extent_order); k++ )
+ set_gpfn_from_mfn(mfn + k, gpfn + k);
+ (void)__copy_to_guest_offset(
+ exch.out.extent_start, (i<<out_chunk_order)+j, &mfn, 1);
+ }
+
+ j++;
+ }
+ BUG_ON(j != (1UL << out_chunk_order));
+ }
+
+ exch.nr_exchanged += exch.in.nr_extents;
+ if ( copy_field_to_guest(arg, &exch, nr_exchanged) )
+ rc = -EFAULT;
+ return rc;
+
+ /*
+ * Failed a chunk! Free any partial chunk work. Tell caller how many
+ * chunks succeeded.
+ */
+ fail:
+ /* Reassign any input pages we managed to steal. */
+ while ( !list_empty(&in_chunk_list) )
+ {
+ page = list_entry(in_chunk_list.next, struct page_info, list);
+ list_del(&page->list);
+ if ( assign_pages(d, page, 0, MEMF_no_refcount) )
+ BUG();
+ }
+
+ /* Free any output pages we managed to allocate. */
+ while ( !list_empty(&out_chunk_list) )
+ {
+ page = list_entry(out_chunk_list.next, struct page_info, list);
+ list_del(&page->list);
+ free_domheap_pages(page, exch.out.extent_order);
+ }
+
+ exch.nr_exchanged += i << in_chunk_order;
+
+ fail_early:
+ if ( copy_field_to_guest(arg, &exch, nr_exchanged) )
+ rc = -EFAULT;
+ return rc;
+}
+
long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE(void) arg)
{
struct domain *d;
- int rc, op, flags = 0, preempted = 0;
+ int rc, op, preempted = 0;
+ unsigned int memflags = 0;
unsigned long start_extent, progress;
struct xen_memory_reservation reservation;
domid_t domid;
case XENMEM_increase_reservation:
case XENMEM_decrease_reservation:
case XENMEM_populate_physmap:
+ start_extent = cmd >> START_EXTENT_SHIFT;
+
if ( copy_from_guest(&reservation, arg, 1) )
- return -EFAULT;
+ return start_extent;
/* Is size too large for us to encode a continuation? */
if ( reservation.nr_extents > (ULONG_MAX >> START_EXTENT_SHIFT) )
- return -EINVAL;
+ return start_extent;
- start_extent = cmd >> START_EXTENT_SHIFT;
if ( unlikely(start_extent > reservation.nr_extents) )
- return -EINVAL;
+ return start_extent;
if ( !guest_handle_is_null(reservation.extent_start) )
guest_handle_add_offset(reservation.extent_start, start_extent);
(get_order_from_pages(max_page) + PAGE_SHIFT)) )
{
if ( reservation.address_bits < 31 )
- return -ENOMEM;
- flags = ALLOC_DOM_DMA;
+ return start_extent;
+ memflags = MEMF_dma;
}
if ( likely(reservation.domid == DOMID_SELF) )
d = current->domain;
- else if ( !IS_PRIV(current->domain) )
- return -EPERM;
- else if ( (d = find_domain_by_id(reservation.domid)) == NULL )
- return -ESRCH;
+ else if ( !IS_PRIV(current->domain) ||
+ ((d = find_domain_by_id(reservation.domid)) == NULL) )
+ return start_extent;
switch ( op )
{
reservation.extent_start,
reservation.nr_extents,
reservation.extent_order,
- flags,
+ memflags,
&preempted);
break;
case XENMEM_decrease_reservation:
reservation.extent_start,
reservation.nr_extents,
reservation.extent_order,
- flags,
&preempted);
break;
case XENMEM_populate_physmap:
reservation.extent_start,
reservation.nr_extents,
reservation.extent_order,
- flags,
+ memflags,
&preempted);
break;
}
break;
+ case XENMEM_exchange:
+ rc = memory_exchange(guest_handle_cast(arg, xen_memory_exchange_t));
+ break;
+
case XENMEM_maximum_ram_page:
rc = max_page;
break;
}
+int assign_pages(
+ struct domain *d,
+ struct page_info *pg,
+ unsigned int order,
+ unsigned int memflags)
+{
+ unsigned long i;
+
+ spin_lock(&d->page_alloc_lock);
+
+ if ( unlikely(test_bit(_DOMF_dying, &d->domain_flags)) )
+ {
+ DPRINTK("Cannot assign page to domain%d -- dying.\n", d->domain_id);
+ goto fail;
+ }
+
+ if ( !(memflags & MEMF_no_refcount) )
+ {
+ if ( unlikely((d->tot_pages + (1 << order)) > d->max_pages) )
+ {
+ DPRINTK("Over-allocation for domain %u: %u > %u\n",
+ d->domain_id, d->tot_pages + (1 << order), d->max_pages);
+ goto fail;
+ }
+
+ if ( unlikely(d->tot_pages == 0) )
+ get_knownalive_domain(d);
+
+ d->tot_pages += 1 << order;
+ }
+
+ for ( i = 0; i < (1 << order); i++ )
+ {
+ ASSERT(page_get_owner(&pg[i]) == NULL);
+ ASSERT((pg[i].count_info & ~(PGC_allocated | 1)) == 0);
+ page_set_owner(&pg[i], d);
+ wmb(); /* Domain pointer must be visible before updating refcnt. */
+ pg[i].count_info = PGC_allocated | 1;
+ list_add_tail(&pg[i].list, &d->page_list);
+ }
+
+ spin_unlock(&d->page_alloc_lock);
+ return 0;
+
+ fail:
+ spin_unlock(&d->page_alloc_lock);
+ return -1;
+}
+
+
struct page_info *alloc_domheap_pages(
- struct domain *d, unsigned int order, unsigned int flags)
+ struct domain *d, unsigned int order, unsigned int memflags)
{
struct page_info *pg = NULL;
cpumask_t mask;
- int i;
+ unsigned long i;
ASSERT(!in_irq());
- if ( !(flags & ALLOC_DOM_DMA) )
+ if ( !(memflags & MEMF_dma) )
{
pg = alloc_heap_pages(MEMZONE_DOM, order);
/* Failure? Then check if we can fall back to the DMA pool. */
flush_tlb_mask(mask);
}
- if ( d == NULL )
- return pg;
-
- spin_lock(&d->page_alloc_lock);
-
- if ( unlikely(test_bit(_DOMF_dying, &d->domain_flags)) ||
- unlikely((d->tot_pages + (1 << order)) > d->max_pages) )
+ if ( (d != NULL) && assign_pages(d, pg, order, memflags) )
{
- DPRINTK("Over-allocation for domain %u: %u > %u\n",
- d->domain_id, d->tot_pages + (1 << order), d->max_pages);
- DPRINTK("...or the domain is dying (%d)\n",
- !!test_bit(_DOMF_dying, &d->domain_flags));
- spin_unlock(&d->page_alloc_lock);
free_heap_pages(pfn_dom_zone_type(page_to_mfn(pg)), pg, order);
return NULL;
}
-
- if ( unlikely(d->tot_pages == 0) )
- get_knownalive_domain(d);
-
- d->tot_pages += 1 << order;
-
- for ( i = 0; i < (1 << order); i++ )
- {
- page_set_owner(&pg[i], d);
- wmb(); /* Domain pointer must be visible before updating refcnt. */
- pg[i].count_info |= PGC_allocated | 1;
- list_add_tail(&pg[i].list, &d->page_list);
- }
-
- spin_unlock(&d->page_alloc_lock);
return pg;
}
#define create_grant_host_mapping(a, f, fl) 0
#define destroy_grant_host_mapping(a, f, fl) 0
-// for grant transfer
-#define steal_page_for_grant_transfer(d, p) 0
-
#else
// for grant map/unmap
int create_grant_host_mapping(unsigned long gpaddr, unsigned long mfn, unsigned int flags);
int destroy_grant_host_mapping(unsigned long gpaddr, unsigned long mfn, unsigned int flags);
// for grant transfer
-int steal_page_for_grant_transfer(struct domain *d, struct page_info *page);
void guest_physmap_add_page(struct domain *d, unsigned long gpfn, unsigned long mfn);
#endif
/* Arch-specific portion of memory_op hypercall. */
#define arch_memory_op(op, arg) (-ENOSYS)
+#ifndef CONFIG_XEN_IA64_DOM0_VP
+#define steal_page(d, p, f) 0
+#else
+int steal_page(
+ struct domain *d, struct page_info *page, unsigned int memflags);
+#endif
+
#endif /* __ASM_IA64_MM_H__ */
int destroy_grant_host_mapping(
unsigned long addr, unsigned long frame, unsigned int flags);
-int steal_page_for_grant_transfer(
- struct domain *d, struct page_info *page);
-
#define gnttab_create_shared_page(d, t, i) \
do { \
share_xen_page_with_guest( \
long arch_memory_op(int op, XEN_GUEST_HANDLE(void) arg);
long subarch_memory_op(int op, XEN_GUEST_HANDLE(void) arg);
+int steal_page(
+ struct domain *d, struct page_info *page, unsigned int memflags);
+
#endif /* __ASM_X86_MM_H__ */
#define __XEN_PUBLIC_MEMORY_H__
/*
- * Increase or decrease the specified domain's memory reservation. Returns a
- * -ve errcode on failure, or the # extents successfully allocated or freed.
+ * Increase or decrease the specified domain's memory reservation. Returns the
+ * number of extents successfully allocated or freed.
* arg == addr of struct xen_memory_reservation.
*/
#define XENMEM_increase_reservation 0
* Unprivileged domains can specify only DOMID_SELF.
*/
domid_t domid;
-
};
typedef struct xen_memory_reservation xen_memory_reservation_t;
DEFINE_XEN_GUEST_HANDLE(xen_memory_reservation_t);
+/*
+ * An atomic exchange of memory pages. If return code is zero then
+ * @out.extent_list provides GMFNs of the newly-allocated memory.
+ * Returns zero on complete success, otherwise a negative error code.
+ * On complete success then always @nr_exchanged == @in.nr_extents.
+ * On partial success @nr_exchanged indicates how much work was done.
+ */
+#define XENMEM_exchange 11
+struct xen_memory_exchange {
+ /*
+ * [IN] Details of memory extents to be exchanged (GMFN bases).
+ * Note that @in.address_bits is ignored and unused.
+ */
+ struct xen_memory_reservation in;
+
+ /*
+ * [IN/OUT] Details of new memory extents.
+ * We require that:
+ * 1. @in.domid == @out.domid
+ * 2. @in.nr_extents << @in.extent_order ==
+ * @out.nr_extents << @out.extent_order
+ * 3. @in.extent_start and @out.extent_start lists must not overlap
+ * 4. @out.extent_start lists GPFN bases to be populated
+ * 5. @out.extent_start is overwritten with allocated GMFN bases
+ */
+ struct xen_memory_reservation out;
+
+ /*
+ * [OUT] Number of input extents that were successfully exchanged:
+ * 1. The first @nr_exchanged input extents were successfully
+ * deallocated.
+ * 2. The corresponding first entries in the output extent list correctly
+ * indicate the GMFNs that were successfully exchanged.
+ * 3. All other input and output extents are untouched.
+ * 4. If not all input exents are exchanged then the return code of this
+ * command will be non-zero.
+ * 5. THIS FIELD MUST BE INITIALISED TO ZERO BY THE CALLER!
+ */
+ unsigned long nr_exchanged;
+};
+typedef struct xen_memory_exchange xen_memory_exchange_t;
+DEFINE_XEN_GUEST_HANDLE(xen_memory_exchange_t);
+
/*
* Returns the maximum machine frame number of mapped RAM in this system.
* This command always succeeds (it never returns an error code).
/* Domain suballocator. These functions are *not* interrupt-safe.*/
void init_domheap_pages(paddr_t ps, paddr_t pe);
struct page_info *alloc_domheap_pages(
- struct domain *d, unsigned int order, unsigned int flags);
+ struct domain *d, unsigned int order, unsigned int memflags);
void free_domheap_pages(struct page_info *pg, unsigned int order);
unsigned long avail_domheap_pages(void);
#define alloc_domheap_page(d) (alloc_domheap_pages(d,0,0))
#define free_domheap_page(p) (free_domheap_pages(p,0))
-#define ALLOC_DOM_DMA 1
+int assign_pages(
+ struct domain *d,
+ struct page_info *pg,
+ unsigned int order,
+ unsigned int memflags);
+
+/* memflags: */
+#define _MEMF_dma 0
+#define MEMF_dma (1U<<_MEMF_dma)
+#define _MEMF_no_refcount 1
+#define MEMF_no_refcount (1U<<_MEMF_no_refcount)
#ifdef CONFIG_PAGEALLOC_MAX_ORDER
#define MAX_ORDER CONFIG_PAGEALLOC_MAX_ORDER
projects / xen.git / commitdiff