123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291 |
- /*-
- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
- *
- * Copyright (C) 2010 Nathan Whitehorn
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
- * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
- #include <sys/cdefs.h>
- __FBSDID("$FreeBSD$");
- #include <sys/param.h>
- #include <sys/kernel.h>
- #include <sys/ktr.h>
- #include <sys/lock.h>
- #include <sys/msgbuf.h>
- #include <sys/mutex.h>
- #include <sys/proc.h>
- #include <sys/sysctl.h>
- #include <sys/systm.h>
- #include <sys/vmmeter.h>
- #include <vm/vm.h>
- #include <vm/vm_param.h>
- #include <vm/vm_kern.h>
- #include <vm/vm_page.h>
- #include <vm/vm_map.h>
- #include <vm/vm_object.h>
- #include <vm/vm_extern.h>
- #include <vm/vm_pageout.h>
- #include <vm/uma.h>
- #include <powerpc/aim/mmu_oea64.h>
- #include "ps3-hvcall.h"
- #define VSID_HASH_MASK 0x0000007fffffffffUL
- #define PTESYNC() __asm __volatile("ptesync")
- extern int ps3fb_remap(void);
- static uint64_t mps3_vas_id;
- /*
- * Kernel MMU interface
- */
- static void mps3_install(void);
- static void mps3_bootstrap(vm_offset_t kernelstart,
- vm_offset_t kernelend);
- static void mps3_cpu_bootstrap(int ap);
- static int64_t mps3_pte_synch(struct pvo_entry *);
- static int64_t mps3_pte_clear(struct pvo_entry *, uint64_t ptebit);
- static int64_t mps3_pte_unset(struct pvo_entry *);
- static int64_t mps3_pte_insert(struct pvo_entry *);
- static struct pmap_funcs mps3_methods = {
- .install = mps3_install,
- .bootstrap = mps3_bootstrap,
- .cpu_bootstrap = mps3_cpu_bootstrap,
- };
- static struct moea64_funcs mps3_funcs = {
- .pte_synch = mps3_pte_synch,
- .pte_clear = mps3_pte_clear,
- .pte_unset = mps3_pte_unset,
- .pte_insert = mps3_pte_insert,
- };
- MMU_DEF_INHERIT(ps3_mmu, "mmu_ps3", mps3_methods, oea64_mmu);
- static struct mtx mps3_table_lock;
- static void
- mps3_install()
- {
- moea64_ops = &mps3_funcs;
- }
- static void
- mps3_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend)
- {
- uint64_t final_pteg_count;
- mtx_init(&mps3_table_lock, "page table", NULL, MTX_DEF);
- moea64_early_bootstrap(kernelstart, kernelend);
- /* In case we had a page table already */
- lv1_destruct_virtual_address_space(0);
- /* Allocate new hardware page table */
- lv1_construct_virtual_address_space(
- 20 /* log_2(moea64_pteg_count) */, 2 /* n page sizes */,
- (24UL << 56) | (16UL << 48) /* page sizes 16 MB + 64 KB */,
- &mps3_vas_id, &final_pteg_count
- );
- lv1_select_virtual_address_space(mps3_vas_id);
- moea64_pteg_count = final_pteg_count / sizeof(struct lpteg);
- moea64_mid_bootstrap(kernelstart, kernelend);
- moea64_late_bootstrap(kernelstart, kernelend);
- }
- static void
- mps3_cpu_bootstrap(int ap)
- {
- struct slb *slb = PCPU_GET(aim.slb);
- register_t seg0;
- int i;
- mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR);
- /*
- * Select the page table we configured above and set up the FB mapping
- * so we can have a console.
- */
- lv1_select_virtual_address_space(mps3_vas_id);
- if (!ap)
- ps3fb_remap();
- /*
- * Install kernel SLB entries
- */
- __asm __volatile ("slbia");
- __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0));
- for (i = 0; i < 64; i++) {
- if (!(slb[i].slbe & SLBE_VALID))
- continue;
- __asm __volatile ("slbmte %0, %1" ::
- "r"(slb[i].slbv), "r"(slb[i].slbe));
- }
- }
- static int64_t
- mps3_pte_synch_locked(struct pvo_entry *pvo)
- {
- uint64_t halfbucket[4], rcbits;
- PTESYNC();
- lv1_read_htab_entries(mps3_vas_id, pvo->pvo_pte.slot & ~0x3UL,
- &halfbucket[0], &halfbucket[1], &halfbucket[2], &halfbucket[3],
- &rcbits);
- /* Check if present in page table */
- if ((halfbucket[pvo->pvo_pte.slot & 0x3] & LPTE_AVPN_MASK) !=
- ((pvo->pvo_vpn >> (ADDR_API_SHFT64 - ADDR_PIDX_SHFT)) &
- LPTE_AVPN_MASK))
- return (-1);
- if (!(halfbucket[pvo->pvo_pte.slot & 0x3] & LPTE_VALID))
- return (-1);
- /*
- * rcbits contains the low 12 bits of each PTE's 2nd part,
- * spaced at 16-bit intervals
- */
- return ((rcbits >> ((3 - (pvo->pvo_pte.slot & 0x3))*16)) &
- (LPTE_CHG | LPTE_REF));
- }
- static int64_t
- mps3_pte_synch(struct pvo_entry *pvo)
- {
- int64_t retval;
- mtx_lock(&mps3_table_lock);
- retval = mps3_pte_synch_locked(pvo);
- mtx_unlock(&mps3_table_lock);
- return (retval);
- }
- static int64_t
- mps3_pte_clear(struct pvo_entry *pvo, uint64_t ptebit)
- {
- int64_t refchg;
- struct lpte pte;
- mtx_lock(&mps3_table_lock);
- refchg = mps3_pte_synch_locked(pvo);
- if (refchg < 0) {
- mtx_unlock(&mps3_table_lock);
- return (refchg);
- }
- moea64_pte_from_pvo(pvo, &pte);
- pte.pte_lo |= refchg;
- pte.pte_lo &= ~ptebit;
- /* XXX: race on RC bits between write and sync. Anything to do? */
- lv1_write_htab_entry(mps3_vas_id, pvo->pvo_pte.slot, pte.pte_hi,
- pte.pte_lo);
- mtx_unlock(&mps3_table_lock);
- return (refchg);
- }
- static int64_t
- mps3_pte_unset(struct pvo_entry *pvo)
- {
- int64_t refchg;
- mtx_lock(&mps3_table_lock);
- refchg = mps3_pte_synch_locked(pvo);
- if (refchg < 0) {
- STAT_MOEA64(moea64_pte_overflow--);
- mtx_unlock(&mps3_table_lock);
- return (-1);
- }
- /* XXX: race on RC bits between unset and sync. Anything to do? */
- lv1_write_htab_entry(mps3_vas_id, pvo->pvo_pte.slot, 0, 0);
- mtx_unlock(&mps3_table_lock);
- STAT_MOEA64(moea64_pte_valid--);
- return (refchg & (LPTE_REF | LPTE_CHG));
- }
- static int64_t
- mps3_pte_insert(struct pvo_entry *pvo)
- {
- int result;
- struct lpte pte, evicted;
- uint64_t index;
- if (pvo->pvo_vaddr & PVO_HID) {
- /* Hypercall needs primary PTEG */
- pvo->pvo_vaddr &= ~PVO_HID;
- pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3);
- }
- pvo->pvo_pte.slot &= ~7UL;
- moea64_pte_from_pvo(pvo, &pte);
- evicted.pte_hi = 0;
- PTESYNC();
- mtx_lock(&mps3_table_lock);
- result = lv1_insert_htab_entry(mps3_vas_id, pvo->pvo_pte.slot,
- pte.pte_hi, pte.pte_lo, LPTE_LOCKED | LPTE_WIRED, 0,
- &index, &evicted.pte_hi, &evicted.pte_lo);
- mtx_unlock(&mps3_table_lock);
- if (result != 0) {
- /* No freeable slots in either PTEG? We're hosed. */
- panic("mps3_pte_insert: overflow (%d)", result);
- return (-1);
- }
- /*
- * See where we ended up.
- */
- if ((index & ~7UL) != pvo->pvo_pte.slot)
- pvo->pvo_vaddr |= PVO_HID;
- pvo->pvo_pte.slot = index;
- STAT_MOEA64(moea64_pte_valid++);
- if (evicted.pte_hi) {
- KASSERT((evicted.pte_hi & (LPTE_WIRED | LPTE_LOCKED)) == 0,
- ("Evicted a wired PTE"));
- STAT_MOEA64(moea64_pte_valid--);
- STAT_MOEA64(moea64_pte_overflow++);
- }
- return (0);
- }
|