123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516 |
- /*
- * Mach Operating System
- * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
- * Copyright (c) 1993,1994 The University of Utah and
- * the Computer Systems Laboratory (CSL).
- * All rights reserved.
- *
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- *
- * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
- * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
- * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
- * THIS SOFTWARE.
- *
- * Carnegie Mellon requests users of this software to return to
- *
- * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
- * School of Computer Science
- * Carnegie Mellon University
- * Pittsburgh PA 15213-3890
- *
- * any improvements or extensions that they make and grant Carnegie Mellon
- * the rights to redistribute these changes.
- */
- /*
- * File: vm/vm_pageout.c
- * Author: Avadis Tevanian, Jr., Michael Wayne Young
- * Date: 1985
- *
- * The proverbial page-out daemon.
- */
- #include <device/net_io.h>
- #include <mach/mach_types.h>
- #include <mach/memory_object.h>
- #include <vm/memory_object_default.user.h>
- #include <vm/memory_object_user.user.h>
- #include <mach/vm_param.h>
- #include <mach/vm_statistics.h>
- #include <kern/counters.h>
- #include <kern/debug.h>
- #include <kern/slab.h>
- #include <kern/task.h>
- #include <kern/thread.h>
- #include <kern/printf.h>
- #include <vm/memory_object.h>
- #include <vm/pmap.h>
- #include <vm/vm_map.h>
- #include <vm/vm_object.h>
- #include <vm/vm_page.h>
- #include <vm/vm_pageout.h>
- #include <machine/locore.h>
- #define DEBUG 0
- /*
- * Maximum delay, in milliseconds, between two pageout scans.
- */
- #define VM_PAGEOUT_TIMEOUT 50
- /*
- * Event placeholder for pageout requests, synchronized with
- * the free page queue lock.
- */
- static int vm_pageout_requested;
- /*
- * Event placeholder for pageout throttling, synchronized with
- * the free page queue lock.
- */
- static int vm_pageout_continue;
- /*
- * Routine: vm_pageout_setup
- * Purpose:
- * Set up a page for pageout.
- *
- * Move or copy the page to a new object, as part
- * of which it will be sent to its memory manager
- * in a memory_object_data_return or memory_object_initialize
- * message.
- *
- * The "paging_offset" argument specifies the offset
- * of the page within its external memory object.
- *
- * The "new_object" and "new_offset" arguments
- * indicate where the page should be moved.
- *
- * The "flush" argument specifies whether the page
- * should be flushed from its object. If not, a
- * copy of the page is moved to the new object.
- *
- * In/Out conditions:
- * The page in question must not be on any pageout queues,
- * and must be busy. The object to which it belongs
- * must be unlocked, and the caller must hold a paging
- * reference to it. The new_object must not be locked.
- *
- * If the page is flushed from its original object,
- * this routine returns a pointer to a place-holder page,
- * inserted at the same offset, to block out-of-order
- * requests for the page. The place-holder page must
- * be freed after the data_return or initialize message
- * has been sent. If the page is copied,
- * the holding page is VM_PAGE_NULL.
- *
- * The original page is put on a paging queue and marked
- * not busy on exit.
- */
- vm_page_t
- vm_pageout_setup(
- vm_page_t m,
- vm_offset_t paging_offset,
- vm_object_t new_object,
- vm_offset_t new_offset,
- boolean_t flush)
- {
- vm_object_t old_object = m->object;
- vm_page_t holding_page = 0; /*'=0'to quiet gcc warnings*/
- vm_page_t new_m;
- assert(m->busy && !m->absent && !m->fictitious);
- /*
- * If we are not flushing the page, allocate a
- * page in the object.
- */
- if (!flush) {
- for (;;) {
- vm_object_lock(new_object);
- new_m = vm_page_alloc(new_object, new_offset);
- vm_object_unlock(new_object);
- if (new_m != VM_PAGE_NULL) {
- break;
- }
- VM_PAGE_WAIT(NULL);
- }
- }
- if (flush) {
- /*
- * Create a place-holder page where the old one was,
- * to prevent anyone from attempting to page in this
- * page while we`re unlocked.
- */
- while ((holding_page = vm_page_grab_fictitious())
- == VM_PAGE_NULL)
- vm_page_more_fictitious();
- vm_object_lock(old_object);
- vm_page_lock_queues();
- vm_page_remove(m);
- vm_page_unlock_queues();
- PAGE_WAKEUP_DONE(m);
- vm_page_lock_queues();
- vm_page_insert(holding_page, old_object, m->offset);
- vm_page_unlock_queues();
- /*
- * Record that this page has been written out
- */
- #if MACH_PAGEMAP
- vm_external_state_set(old_object->existence_info,
- paging_offset,
- VM_EXTERNAL_STATE_EXISTS);
- #endif /* MACH_PAGEMAP */
- vm_object_unlock(old_object);
- vm_object_lock(new_object);
- /*
- * Move this page into the new object
- */
- vm_page_lock_queues();
- vm_page_insert(m, new_object, new_offset);
- vm_page_unlock_queues();
- m->dirty = TRUE;
- m->precious = FALSE;
- m->page_lock = VM_PROT_NONE;
- m->unlock_request = VM_PROT_NONE;
- }
- else {
- /*
- * Copy the data into the new page,
- * and mark the new page as clean.
- */
- vm_page_copy(m, new_m);
- vm_object_lock(old_object);
- m->dirty = FALSE;
- pmap_clear_modify(m->phys_addr);
- /*
- * Deactivate old page.
- */
- vm_page_lock_queues();
- vm_page_deactivate(m);
- vm_page_unlock_queues();
- PAGE_WAKEUP_DONE(m);
- /*
- * Record that this page has been written out
- */
- #if MACH_PAGEMAP
- vm_external_state_set(old_object->existence_info,
- paging_offset,
- VM_EXTERNAL_STATE_EXISTS);
- #endif /* MACH_PAGEMAP */
- vm_object_unlock(old_object);
- vm_object_lock(new_object);
- /*
- * Use the new page below.
- */
- m = new_m;
- m->dirty = TRUE;
- assert(!m->precious);
- PAGE_WAKEUP_DONE(m);
- }
- /*
- * Make the old page eligible for replacement again; if a
- * user-supplied memory manager fails to release the page,
- * it will be paged out again to the default memory manager.
- *
- * Note that pages written to the default memory manager
- * must be wired down -- in return, it guarantees to free
- * this page, rather than reusing it.
- */
- vm_page_lock_queues();
- vm_stat.pageouts++;
- if (m->laundry) {
- /*
- * The caller is telling us that it is going to
- * immediately double page this page to the default
- * pager.
- */
- assert(!old_object->internal);
- m->laundry = FALSE;
- } else if (old_object->internal ||
- memory_manager_default_port(old_object->pager)) {
- m->laundry = TRUE;
- vm_page_laundry_count++;
- vm_page_wire(m);
- } else {
- m->external_laundry = TRUE;
- /*
- * If vm_page_external_laundry_count is negative,
- * the pageout daemon isn't expecting to be
- * notified.
- */
- if (vm_page_external_laundry_count >= 0) {
- vm_page_external_laundry_count++;
- }
- vm_page_activate(m);
- }
- vm_page_unlock_queues();
- /*
- * Since IPC operations may block, we drop locks now.
- * [The placeholder page is busy, and we still have
- * paging_in_progress incremented.]
- */
- vm_object_unlock(new_object);
- /*
- * Return the placeholder page to simplify cleanup.
- */
- return (flush ? holding_page : VM_PAGE_NULL);
- }
- /*
- * Routine: vm_pageout_page
- * Purpose:
- * Causes the specified page to be written back to
- * the appropriate memory object.
- *
- * The "initial" argument specifies whether this
- * data is an initialization only, and should use
- * memory_object_data_initialize instead of
- * memory_object_data_return.
- *
- * The "flush" argument specifies whether the page
- * should be flushed from the object. If not, a
- * copy of the data is sent to the memory object.
- *
- * In/out conditions:
- * The page in question must not be on any pageout queues.
- * The object to which it belongs must be locked.
- * Implementation:
- * Move this page to a completely new object, if flushing;
- * copy to a new page in a new object, if not.
- */
- void
- vm_pageout_page(
- vm_page_t m,
- boolean_t initial,
- boolean_t flush)
- {
- vm_map_copy_t copy;
- vm_object_t old_object;
- vm_object_t new_object;
- vm_page_t holding_page;
- vm_offset_t paging_offset;
- kern_return_t rc;
- boolean_t precious_clean;
- assert(m->busy);
- /*
- * Cleaning but not flushing a clean precious page is a
- * no-op. Remember whether page is clean and precious now
- * because vm_pageout_setup will mark it dirty and not precious.
- *
- * XXX Check if precious_clean && !flush can really happen.
- */
- precious_clean = (!m->dirty) && m->precious;
- if (precious_clean && !flush) {
- PAGE_WAKEUP_DONE(m);
- return;
- }
- /*
- * Verify that we really want to clean this page.
- */
- if (m->absent || m->error || (!m->dirty && !m->precious)) {
- VM_PAGE_FREE(m);
- return;
- }
- /*
- * Create a paging reference to let us play with the object.
- */
- old_object = m->object;
- paging_offset = m->offset + old_object->paging_offset;
- vm_object_paging_begin(old_object);
- vm_object_unlock(old_object);
- /*
- * Allocate a new object into which we can put the page.
- */
- new_object = vm_object_allocate(PAGE_SIZE);
- new_object->used_for_pageout = TRUE;
- /*
- * Move the page into the new object.
- */
- holding_page = vm_pageout_setup(m,
- paging_offset,
- new_object,
- 0, /* new offset */
- flush); /* flush */
- rc = vm_map_copyin_object(new_object, 0, PAGE_SIZE, ©);
- assert(rc == KERN_SUCCESS);
- if (initial) {
- rc = memory_object_data_initialize(
- old_object->pager,
- old_object->pager_request,
- paging_offset, (pointer_t) copy, PAGE_SIZE);
- }
- else {
- rc = memory_object_data_return(
- old_object->pager,
- old_object->pager_request,
- paging_offset, (pointer_t) copy, PAGE_SIZE,
- !precious_clean, !flush);
- }
- if (rc != KERN_SUCCESS)
- vm_map_copy_discard(copy);
- /*
- * Clean up.
- */
- vm_object_lock(old_object);
- if (holding_page != VM_PAGE_NULL)
- VM_PAGE_FREE(holding_page);
- vm_object_paging_end(old_object);
- }
- /*
- * vm_pageout_scan does the dirty work for the pageout daemon.
- *
- * Return TRUE if the pageout daemon is done for now, FALSE otherwise,
- * in which case should_wait indicates whether the pageout daemon
- * should wait to allow pagers to keep up.
- *
- * It returns with vm_page_queue_free_lock held.
- */
- boolean_t vm_pageout_scan(boolean_t *should_wait)
- {
- boolean_t done;
- /*
- * Try balancing pages among segments first, since this
- * may be enough to resume unprivileged allocations.
- */
- /* This function returns with vm_page_queue_free_lock held */
- done = vm_page_balance();
- if (done) {
- return TRUE;
- }
- simple_unlock(&vm_page_queue_free_lock);
- /*
- * Balancing is not enough. Shrink caches and scan pages
- * for eviction.
- */
- stack_collect();
- net_kmsg_collect();
- consider_task_collect();
- if (0) /* XXX: pcb_collect doesn't do anything yet, so it is
- pointless to call consider_thread_collect. */
- consider_thread_collect();
- /*
- * slab_collect should be last, because the other operations
- * might return memory to caches.
- */
- slab_collect();
- vm_page_refill_inactive();
- /* This function returns with vm_page_queue_free_lock held */
- return vm_page_evict(should_wait);
- }
- void vm_pageout(void)
- {
- boolean_t done, should_wait;
- current_thread()->vm_privilege = 1;
- stack_privilege(current_thread());
- thread_set_own_priority(0);
- for (;;) {
- done = vm_pageout_scan(&should_wait);
- /* we hold vm_page_queue_free_lock now */
- if (done) {
- thread_sleep(&vm_pageout_requested,
- simple_lock_addr(vm_page_queue_free_lock),
- FALSE);
- } else if (should_wait) {
- assert_wait(&vm_pageout_continue, FALSE);
- thread_set_timeout(VM_PAGEOUT_TIMEOUT * hz / 1000);
- simple_unlock(&vm_page_queue_free_lock);
- thread_block(NULL);
- #if DEBUG
- if (current_thread()->wait_result != THREAD_AWAKENED) {
- printf("vm_pageout: timeout,"
- " vm_page_laundry_count:%d"
- " vm_page_external_laundry_count:%d\n",
- vm_page_laundry_count,
- vm_page_external_laundry_count);
- }
- #endif
- } else {
- simple_unlock(&vm_page_queue_free_lock);
- }
- }
- }
- /*
- * Start pageout
- *
- * The free page queue lock must be held before calling this function.
- */
- void vm_pageout_start(void)
- {
- if (!current_thread())
- return;
- thread_wakeup_one(&vm_pageout_requested);
- }
- /*
- * Resume pageout
- *
- * The free page queue lock must be held before calling this function.
- */
- void vm_pageout_resume(void)
- {
- thread_wakeup_one(&vm_pageout_continue);
- }
|