123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792 |
- /*-
- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
- *
- * Copyright (C) 2020 Justin Hibbits
- * Copyright (C) 2007-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com>
- * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
- * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * Some hw specific parts of this pmap were derived or influenced
- * by NetBSD's ibm4xx pmap module. More generic code is shared with
- * a few other pmap modules from the FreeBSD tree.
- */
- /*
- * VM layout notes:
- *
- * Kernel and user threads run within one common virtual address space
- * defined by AS=0.
- *
- * 64-bit pmap:
- * Virtual address space layout:
- * -----------------------------
- * 0x0000_0000_0000_0000 - 0x3fff_ffff_ffff_ffff : user process
- * 0x4000_0000_0000_0000 - 0x7fff_ffff_ffff_ffff : unused
- * 0x8000_0000_0000_0000 - 0xbfff_ffff_ffff_ffff : mmio region
- * 0xc000_0000_0000_0000 - 0xdfff_ffff_ffff_ffff : direct map
- * 0xe000_0000_0000_0000 - 0xffff_ffff_ffff_ffff : KVA
- */
- #include <sys/cdefs.h>
- __FBSDID("$FreeBSD$");
- #include "opt_ddb.h"
- #include "opt_kstack_pages.h"
- #include <sys/param.h>
- #include <sys/conf.h>
- #include <sys/malloc.h>
- #include <sys/ktr.h>
- #include <sys/proc.h>
- #include <sys/user.h>
- #include <sys/queue.h>
- #include <sys/systm.h>
- #include <sys/kernel.h>
- #include <sys/kerneldump.h>
- #include <sys/linker.h>
- #include <sys/msgbuf.h>
- #include <sys/lock.h>
- #include <sys/mutex.h>
- #include <sys/rwlock.h>
- #include <sys/sched.h>
- #include <sys/smp.h>
- #include <sys/vmmeter.h>
- #include <vm/vm.h>
- #include <vm/vm_page.h>
- #include <vm/vm_kern.h>
- #include <vm/vm_pageout.h>
- #include <vm/vm_extern.h>
- #include <vm/vm_object.h>
- #include <vm/vm_param.h>
- #include <vm/vm_map.h>
- #include <vm/vm_pager.h>
- #include <vm/vm_phys.h>
- #include <vm/vm_pagequeue.h>
- #include <vm/uma.h>
- #include <machine/_inttypes.h>
- #include <machine/cpu.h>
- #include <machine/pcb.h>
- #include <machine/platform.h>
- #include <machine/tlb.h>
- #include <machine/spr.h>
- #include <machine/md_var.h>
- #include <machine/mmuvar.h>
- #include <machine/pmap.h>
- #include <machine/pte.h>
- #include <ddb/ddb.h>
- #ifdef DEBUG
- #define debugf(fmt, args...) printf(fmt, ##args)
- #else
- #define debugf(fmt, args...)
- #endif
- #define PRI0ptrX "016lx"
- /**************************************************************************/
- /* PMAP */
- /**************************************************************************/
- unsigned int kernel_pdirs;
- static uma_zone_t ptbl_root_zone;
- static pte_t ****kernel_ptbl_root;
- /*
- * Base of the pmap_mapdev() region. On 32-bit it immediately follows the
- * userspace address range. On On 64-bit it's far above, at (1 << 63), and
- * ranges up to the DMAP, giving 62 bits of PA allowed. This is far larger than
- * the widest Book-E address bus, the e6500 has a 40-bit PA space. This allows
- * us to map akin to the DMAP, with addresses identical to the PA, offset by the
- * base.
- */
- #define VM_MAPDEV_BASE 0x8000000000000000
- #define VM_MAPDEV_PA_MAX 0x4000000000000000 /* Don't encroach on DMAP */
- static void tid_flush(tlbtid_t tid);
- static unsigned long ilog2(unsigned long);
- /**************************************************************************/
- /* Page table management */
- /**************************************************************************/
- #define PMAP_ROOT_SIZE (sizeof(pte_t****) * PG_ROOT_NENTRIES)
- static pte_t *ptbl_alloc(pmap_t pmap, vm_offset_t va,
- bool nosleep, bool *is_new);
- static void ptbl_hold(pmap_t, pte_t *);
- static int ptbl_unhold(pmap_t, vm_offset_t);
- static vm_paddr_t pte_vatopa(pmap_t, vm_offset_t);
- static int pte_enter(pmap_t, vm_page_t, vm_offset_t, uint32_t, boolean_t);
- static int pte_remove(pmap_t, vm_offset_t, uint8_t);
- static pte_t *pte_find(pmap_t, vm_offset_t);
- static pte_t *pte_find_next(pmap_t, vm_offset_t *);
- static void kernel_pte_alloc(vm_offset_t, vm_offset_t);
- /**************************************************************************/
- /* Page table related */
- /**************************************************************************/
- /* Allocate a page, to be used in a page table. */
- static vm_offset_t
- mmu_booke_alloc_page(pmap_t pmap, unsigned int idx, bool nosleep)
- {
- vm_page_t m;
- int req;
- req = VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO;
- while ((m = vm_page_alloc(NULL, idx, req)) == NULL) {
- if (nosleep)
- return (0);
- PMAP_UNLOCK(pmap);
- rw_wunlock(&pvh_global_lock);
- vm_wait(NULL);
- rw_wlock(&pvh_global_lock);
- PMAP_LOCK(pmap);
- }
- if (!(m->flags & PG_ZERO))
- /* Zero whole ptbl. */
- mmu_booke_zero_page(m);
- return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)));
- }
- /* Initialize pool of kva ptbl buffers. */
- static void
- ptbl_init(void)
- {
- }
- /* Get a pointer to a PTE in a page table. */
- static __inline pte_t *
- pte_find(pmap_t pmap, vm_offset_t va)
- {
- pte_t ***pdir_l1;
- pte_t **pdir;
- pte_t *ptbl;
- KASSERT((pmap != NULL), ("pte_find: invalid pmap"));
- pdir_l1 = pmap->pm_root[PG_ROOT_IDX(va)];
- if (pdir_l1 == NULL)
- return (NULL);
- pdir = pdir_l1[PDIR_L1_IDX(va)];
- if (pdir == NULL)
- return (NULL);
- ptbl = pdir[PDIR_IDX(va)];
- return ((ptbl != NULL) ? &ptbl[PTBL_IDX(va)] : NULL);
- }
- /* Get a pointer to a PTE in a page table, or the next closest (greater) one. */
- static __inline pte_t *
- pte_find_next(pmap_t pmap, vm_offset_t *pva)
- {
- vm_offset_t va;
- pte_t ****pm_root;
- pte_t *pte;
- unsigned long i, j, k, l;
- KASSERT((pmap != NULL), ("pte_find: invalid pmap"));
- va = *pva;
- i = PG_ROOT_IDX(va);
- j = PDIR_L1_IDX(va);
- k = PDIR_IDX(va);
- l = PTBL_IDX(va);
- pm_root = pmap->pm_root;
- /* truncate the VA for later. */
- va &= ~((1UL << (PG_ROOT_H + 1)) - 1);
- for (; i < PG_ROOT_NENTRIES; i++, j = 0, k = 0, l = 0) {
- if (pm_root[i] == 0)
- continue;
- for (; j < PDIR_L1_NENTRIES; j++, k = 0, l = 0) {
- if (pm_root[i][j] == 0)
- continue;
- for (; k < PDIR_NENTRIES; k++, l = 0) {
- if (pm_root[i][j][k] == NULL)
- continue;
- for (; l < PTBL_NENTRIES; l++) {
- pte = &pm_root[i][j][k][l];
- if (!PTE_ISVALID(pte))
- continue;
- *pva = va + PG_ROOT_SIZE * i +
- PDIR_L1_SIZE * j +
- PDIR_SIZE * k +
- PAGE_SIZE * l;
- return (pte);
- }
- }
- }
- }
- return (NULL);
- }
- static bool
- unhold_free_page(pmap_t pmap, vm_page_t m)
- {
- if (vm_page_unwire_noq(m)) {
- vm_page_free_zero(m);
- return (true);
- }
- return (false);
- }
- static vm_offset_t
- get_pgtbl_page(pmap_t pmap, vm_offset_t *ptr_tbl, uint32_t index,
- bool nosleep, bool hold_parent, bool *isnew)
- {
- vm_offset_t page;
- vm_page_t m;
- page = ptr_tbl[index];
- KASSERT(page != 0 || pmap != kernel_pmap,
- ("NULL page table page found in kernel pmap!"));
- if (page == 0) {
- page = mmu_booke_alloc_page(pmap, index, nosleep);
- if (ptr_tbl[index] == 0) {
- *isnew = true;
- ptr_tbl[index] = page;
- if (hold_parent) {
- m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)ptr_tbl));
- m->ref_count++;
- }
- return (page);
- }
- m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS(page));
- page = ptr_tbl[index];
- vm_page_unwire_noq(m);
- vm_page_free_zero(m);
- }
- *isnew = false;
- return (page);
- }
- /* Allocate page table. */
- static pte_t*
- ptbl_alloc(pmap_t pmap, vm_offset_t va, bool nosleep, bool *is_new)
- {
- unsigned int pg_root_idx = PG_ROOT_IDX(va);
- unsigned int pdir_l1_idx = PDIR_L1_IDX(va);
- unsigned int pdir_idx = PDIR_IDX(va);
- vm_offset_t pdir_l1, pdir, ptbl;
- /* When holding a parent, no need to hold the root index pages. */
- pdir_l1 = get_pgtbl_page(pmap, (vm_offset_t *)pmap->pm_root,
- pg_root_idx, nosleep, false, is_new);
- if (pdir_l1 == 0)
- return (NULL);
- pdir = get_pgtbl_page(pmap, (vm_offset_t *)pdir_l1, pdir_l1_idx,
- nosleep, !*is_new, is_new);
- if (pdir == 0)
- return (NULL);
- ptbl = get_pgtbl_page(pmap, (vm_offset_t *)pdir, pdir_idx,
- nosleep, !*is_new, is_new);
- return ((pte_t *)ptbl);
- }
- /*
- * Decrement ptbl pages hold count and attempt to free ptbl pages. Called
- * when removing pte entry from ptbl.
- *
- * Return 1 if ptbl pages were freed.
- */
- static int
- ptbl_unhold(pmap_t pmap, vm_offset_t va)
- {
- pte_t *ptbl;
- vm_page_t m;
- u_int pg_root_idx;
- pte_t ***pdir_l1;
- u_int pdir_l1_idx;
- pte_t **pdir;
- u_int pdir_idx;
- pg_root_idx = PG_ROOT_IDX(va);
- pdir_l1_idx = PDIR_L1_IDX(va);
- pdir_idx = PDIR_IDX(va);
- KASSERT((pmap != kernel_pmap),
- ("ptbl_unhold: unholding kernel ptbl!"));
- pdir_l1 = pmap->pm_root[pg_root_idx];
- pdir = pdir_l1[pdir_l1_idx];
- ptbl = pdir[pdir_idx];
- /* decrement hold count */
- m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) ptbl));
- if (!unhold_free_page(pmap, m))
- return (0);
- pdir[pdir_idx] = NULL;
- m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) pdir));
- if (!unhold_free_page(pmap, m))
- return (1);
- pdir_l1[pdir_l1_idx] = NULL;
- m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) pdir_l1));
- if (!unhold_free_page(pmap, m))
- return (1);
- pmap->pm_root[pg_root_idx] = NULL;
- return (1);
- }
- /*
- * Increment hold count for ptbl pages. This routine is used when new pte
- * entry is being inserted into ptbl.
- */
- static void
- ptbl_hold(pmap_t pmap, pte_t *ptbl)
- {
- vm_page_t m;
- KASSERT((pmap != kernel_pmap),
- ("ptbl_hold: holding kernel ptbl!"));
- m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) ptbl));
- m->ref_count++;
- }
- /*
- * Clean pte entry, try to free page table page if requested.
- *
- * Return 1 if ptbl pages were freed, otherwise return 0.
- */
- static int
- pte_remove(pmap_t pmap, vm_offset_t va, u_int8_t flags)
- {
- vm_page_t m;
- pte_t *pte;
- pte = pte_find(pmap, va);
- KASSERT(pte != NULL, ("%s: NULL pte for va %#jx, pmap %p",
- __func__, (uintmax_t)va, pmap));
- if (!PTE_ISVALID(pte))
- return (0);
- /* Get vm_page_t for mapped pte. */
- m = PHYS_TO_VM_PAGE(PTE_PA(pte));
- if (PTE_ISWIRED(pte))
- pmap->pm_stats.wired_count--;
- /* Handle managed entry. */
- if (PTE_ISMANAGED(pte)) {
- /* Handle modified pages. */
- if (PTE_ISMODIFIED(pte))
- vm_page_dirty(m);
- /* Referenced pages. */
- if (PTE_ISREFERENCED(pte))
- vm_page_aflag_set(m, PGA_REFERENCED);
- /* Remove pv_entry from pv_list. */
- pv_remove(pmap, va, m);
- } else if (pmap == kernel_pmap && m && m->md.pv_tracked) {
- pv_remove(pmap, va, m);
- if (TAILQ_EMPTY(&m->md.pv_list))
- m->md.pv_tracked = false;
- }
- mtx_lock_spin(&tlbivax_mutex);
- tlb_miss_lock();
- tlb0_flush_entry(va);
- *pte = 0;
- tlb_miss_unlock();
- mtx_unlock_spin(&tlbivax_mutex);
- pmap->pm_stats.resident_count--;
- if (flags & PTBL_UNHOLD) {
- return (ptbl_unhold(pmap, va));
- }
- return (0);
- }
- /*
- * Insert PTE for a given page and virtual address.
- */
- static int
- pte_enter(pmap_t pmap, vm_page_t m, vm_offset_t va, uint32_t flags,
- boolean_t nosleep)
- {
- unsigned int ptbl_idx = PTBL_IDX(va);
- pte_t *ptbl, *pte, pte_tmp;
- bool is_new;
- /* Get the page directory pointer. */
- ptbl = ptbl_alloc(pmap, va, nosleep, &is_new);
- if (ptbl == NULL) {
- KASSERT(nosleep, ("nosleep and NULL ptbl"));
- return (ENOMEM);
- }
- if (is_new) {
- pte = &ptbl[ptbl_idx];
- } else {
- /*
- * Check if there is valid mapping for requested va, if there
- * is, remove it.
- */
- pte = &ptbl[ptbl_idx];
- if (PTE_ISVALID(pte)) {
- pte_remove(pmap, va, PTBL_HOLD);
- } else {
- /*
- * pte is not used, increment hold count for ptbl
- * pages.
- */
- if (pmap != kernel_pmap)
- ptbl_hold(pmap, ptbl);
- }
- }
- /*
- * Insert pv_entry into pv_list for mapped page if part of managed
- * memory.
- */
- if ((m->oflags & VPO_UNMANAGED) == 0) {
- flags |= PTE_MANAGED;
- /* Create and insert pv entry. */
- pv_insert(pmap, va, m);
- }
- pmap->pm_stats.resident_count++;
- pte_tmp = PTE_RPN_FROM_PA(VM_PAGE_TO_PHYS(m));
- pte_tmp |= (PTE_VALID | flags);
- mtx_lock_spin(&tlbivax_mutex);
- tlb_miss_lock();
- tlb0_flush_entry(va);
- *pte = pte_tmp;
- tlb_miss_unlock();
- mtx_unlock_spin(&tlbivax_mutex);
- return (0);
- }
- /* Return the pa for the given pmap/va. */
- static vm_paddr_t
- pte_vatopa(pmap_t pmap, vm_offset_t va)
- {
- vm_paddr_t pa = 0;
- pte_t *pte;
- pte = pte_find(pmap, va);
- if ((pte != NULL) && PTE_ISVALID(pte))
- pa = (PTE_PA(pte) | (va & PTE_PA_MASK));
- return (pa);
- }
- /* allocate pte entries to manage (addr & mask) to (addr & mask) + size */
- static void
- kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr)
- {
- pte_t *pte;
- vm_size_t kva_size;
- int kernel_pdirs, kernel_pgtbls, pdir_l1s;
- vm_offset_t va, l1_va, pdir_va, ptbl_va;
- int i, j, k;
- kva_size = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS;
- kernel_pmap->pm_root = kernel_ptbl_root;
- pdir_l1s = howmany(kva_size, PG_ROOT_SIZE);
- kernel_pdirs = howmany(kva_size, PDIR_L1_SIZE);
- kernel_pgtbls = howmany(kva_size, PDIR_SIZE);
- /* Initialize kernel pdir */
- l1_va = (vm_offset_t)kernel_ptbl_root +
- round_page(PG_ROOT_NENTRIES * sizeof(pte_t ***));
- pdir_va = l1_va + pdir_l1s * PAGE_SIZE;
- ptbl_va = pdir_va + kernel_pdirs * PAGE_SIZE;
- if (bootverbose) {
- printf("ptbl_root_va: %#lx\n", (vm_offset_t)kernel_ptbl_root);
- printf("l1_va: %#lx (%d entries)\n", l1_va, pdir_l1s);
- printf("pdir_va: %#lx(%d entries)\n", pdir_va, kernel_pdirs);
- printf("ptbl_va: %#lx(%d entries)\n", ptbl_va, kernel_pgtbls);
- }
- va = VM_MIN_KERNEL_ADDRESS;
- for (i = PG_ROOT_IDX(va); i < PG_ROOT_IDX(va) + pdir_l1s;
- i++, l1_va += PAGE_SIZE) {
- kernel_pmap->pm_root[i] = (pte_t ***)l1_va;
- for (j = 0;
- j < PDIR_L1_NENTRIES && va < VM_MAX_KERNEL_ADDRESS;
- j++, pdir_va += PAGE_SIZE) {
- kernel_pmap->pm_root[i][j] = (pte_t **)pdir_va;
- for (k = 0;
- k < PDIR_NENTRIES && va < VM_MAX_KERNEL_ADDRESS;
- k++, va += PDIR_SIZE, ptbl_va += PAGE_SIZE)
- kernel_pmap->pm_root[i][j][k] = (pte_t *)ptbl_va;
- }
- }
- /*
- * Fill in PTEs covering kernel code and data. They are not required
- * for address translation, as this area is covered by static TLB1
- * entries, but for pte_vatopa() to work correctly with kernel area
- * addresses.
- */
- for (va = addr; va < data_end; va += PAGE_SIZE) {
- pte = &(kernel_pmap->pm_root[PG_ROOT_IDX(va)][PDIR_L1_IDX(va)][PDIR_IDX(va)][PTBL_IDX(va)]);
- *pte = PTE_RPN_FROM_PA(kernload + (va - kernstart));
- *pte |= PTE_M | PTE_SR | PTE_SW | PTE_SX | PTE_WIRED |
- PTE_VALID | PTE_PS_4KB;
- }
- }
- static vm_offset_t
- mmu_booke_alloc_kernel_pgtables(vm_offset_t data_end)
- {
- vm_size_t kva_size = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS;
- kernel_ptbl_root = (pte_t ****)data_end;
- data_end += round_page(PG_ROOT_NENTRIES * sizeof(pte_t ***));
- data_end += howmany(kva_size, PG_ROOT_SIZE) * PAGE_SIZE;
- data_end += howmany(kva_size, PDIR_L1_SIZE) * PAGE_SIZE;
- data_end += howmany(kva_size, PDIR_SIZE) * PAGE_SIZE;
- return (data_end);
- }
- /*
- * Initialize a preallocated and zeroed pmap structure,
- * such as one in a vmspace structure.
- */
- static int
- mmu_booke_pinit(pmap_t pmap)
- {
- int i;
- CTR4(KTR_PMAP, "%s: pmap = %p, proc %d '%s'", __func__, pmap,
- curthread->td_proc->p_pid, curthread->td_proc->p_comm);
- KASSERT((pmap != kernel_pmap), ("pmap_pinit: initializing kernel_pmap"));
- for (i = 0; i < MAXCPU; i++)
- pmap->pm_tid[i] = TID_NONE;
- CPU_ZERO(&kernel_pmap->pm_active);
- bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));
- pmap->pm_root = uma_zalloc(ptbl_root_zone, M_WAITOK);
- bzero(pmap->pm_root, sizeof(pte_t **) * PG_ROOT_NENTRIES);
- return (1);
- }
- /*
- * Release any resources held by the given physical map.
- * Called when a pmap initialized by mmu_booke_pinit is being released.
- * Should only be called if the map contains no valid mappings.
- */
- static void
- mmu_booke_release(pmap_t pmap)
- {
- KASSERT(pmap->pm_stats.resident_count == 0,
- ("pmap_release: pmap resident count %ld != 0",
- pmap->pm_stats.resident_count));
- #ifdef INVARIANTS
- /*
- * Verify that all page directories are gone.
- * Protects against reference count leakage.
- */
- for (int i = 0; i < PG_ROOT_NENTRIES; i++)
- KASSERT(pmap->pm_root[i] == 0,
- ("Index %d on root page %p is non-zero!\n", i, pmap->pm_root));
- #endif
- uma_zfree(ptbl_root_zone, pmap->pm_root);
- }
- static void
- mmu_booke_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
- {
- pte_t *pte;
- vm_paddr_t pa = 0;
- int sync_sz, valid;
- while (sz > 0) {
- PMAP_LOCK(pm);
- pte = pte_find(pm, va);
- valid = (pte != NULL && PTE_ISVALID(pte)) ? 1 : 0;
- if (valid)
- pa = PTE_PA(pte);
- PMAP_UNLOCK(pm);
- sync_sz = PAGE_SIZE - (va & PAGE_MASK);
- sync_sz = min(sync_sz, sz);
- if (valid) {
- pa += (va & PAGE_MASK);
- __syncicache((void *)PHYS_TO_DMAP(pa), sync_sz);
- }
- va += sync_sz;
- sz -= sync_sz;
- }
- }
- /*
- * mmu_booke_zero_page_area zeros the specified hardware page by
- * mapping it into virtual memory and using bzero to clear
- * its contents.
- *
- * off and size must reside within a single page.
- */
- static void
- mmu_booke_zero_page_area(vm_page_t m, int off, int size)
- {
- vm_offset_t va;
- /* XXX KASSERT off and size are within a single page? */
- va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
- bzero((caddr_t)va + off, size);
- }
- /*
- * mmu_booke_zero_page zeros the specified hardware page.
- */
- static void
- mmu_booke_zero_page(vm_page_t m)
- {
- vm_offset_t off, va;
- va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
- for (off = 0; off < PAGE_SIZE; off += cacheline_size)
- __asm __volatile("dcbz 0,%0" :: "r"(va + off));
- }
- /*
- * mmu_booke_copy_page copies the specified (machine independent) page by
- * mapping the page into virtual memory and using memcopy to copy the page,
- * one machine dependent page at a time.
- */
- static void
- mmu_booke_copy_page(vm_page_t sm, vm_page_t dm)
- {
- vm_offset_t sva, dva;
- sva = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(sm));
- dva = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dm));
- memcpy((caddr_t)dva, (caddr_t)sva, PAGE_SIZE);
- }
- static inline void
- mmu_booke_copy_pages(vm_page_t *ma, vm_offset_t a_offset,
- vm_page_t *mb, vm_offset_t b_offset, int xfersize)
- {
- void *a_cp, *b_cp;
- vm_offset_t a_pg_offset, b_pg_offset;
- int cnt;
- vm_page_t pa, pb;
- while (xfersize > 0) {
- a_pg_offset = a_offset & PAGE_MASK;
- pa = ma[a_offset >> PAGE_SHIFT];
- b_pg_offset = b_offset & PAGE_MASK;
- pb = mb[b_offset >> PAGE_SHIFT];
- cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
- cnt = min(cnt, PAGE_SIZE - b_pg_offset);
- a_cp = (caddr_t)((uintptr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pa)) +
- a_pg_offset);
- b_cp = (caddr_t)((uintptr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pb)) +
- b_pg_offset);
- bcopy(a_cp, b_cp, cnt);
- a_offset += cnt;
- b_offset += cnt;
- xfersize -= cnt;
- }
- }
- static vm_offset_t
- mmu_booke_quick_enter_page(vm_page_t m)
- {
- return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)));
- }
- static void
- mmu_booke_quick_remove_page(vm_offset_t addr)
- {
- }
- /**************************************************************************/
- /* TID handling */
- /**************************************************************************/
- /*
- * Return the largest uint value log such that 2^log <= num.
- */
- static unsigned long
- ilog2(unsigned long num)
- {
- long lz;
- __asm ("cntlzd %0, %1" : "=r" (lz) : "r" (num));
- return (63 - lz);
- }
- /*
- * Invalidate all TLB0 entries which match the given TID. Note this is
- * dedicated for cases when invalidations should NOT be propagated to other
- * CPUs.
- */
- static void
- tid_flush(tlbtid_t tid)
- {
- register_t msr;
- /* Don't evict kernel translations */
- if (tid == TID_KERNEL)
- return;
- msr = mfmsr();
- __asm __volatile("wrteei 0");
- /*
- * Newer (e500mc and later) have tlbilx, which doesn't broadcast, so use
- * it for PID invalidation.
- */
- mtspr(SPR_MAS6, tid << MAS6_SPID0_SHIFT);
- __asm __volatile("isync; .long 0x7c200024; isync; msync");
- __asm __volatile("wrtee %0" :: "r"(msr));
- }
|