1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109 |
- /*-
- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
- *
- * Copyright (C) 2007-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com>
- * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
- * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * Some hw specific parts of this pmap were derived or influenced
- * by NetBSD's ibm4xx pmap module. More generic code is shared with
- * a few other pmap modules from the FreeBSD tree.
- */
- /*
- * VM layout notes:
- *
- * Kernel and user threads run within one common virtual address space
- * defined by AS=0.
- *
- * 32-bit pmap:
- * Virtual address space layout:
- * -----------------------------
- * 0x0000_0000 - 0x7fff_ffff : user process
- * 0x8000_0000 - 0xbfff_ffff : pmap_mapdev()-ed area (PCI/PCIE etc.)
- * 0xc000_0000 - 0xc0ff_ffff : kernel reserved
- * 0xc000_0000 - data_end : kernel code+data, env, metadata etc.
- * 0xc100_0000 - 0xffff_ffff : KVA
- * 0xc100_0000 - 0xc100_3fff : reserved for page zero/copy
- * 0xc100_4000 - 0xc200_3fff : reserved for ptbl bufs
- * 0xc200_4000 - 0xc200_8fff : guard page + kstack0
- * 0xc200_9000 - 0xfeef_ffff : actual free KVA space
- *
- * 64-bit pmap:
- * Virtual address space layout:
- * -----------------------------
- * 0x0000_0000_0000_0000 - 0xbfff_ffff_ffff_ffff : user process
- * 0x0000_0000_0000_0000 - 0x8fff_ffff_ffff_ffff : text, data, heap, maps, libraries
- * 0x9000_0000_0000_0000 - 0xafff_ffff_ffff_ffff : mmio region
- * 0xb000_0000_0000_0000 - 0xbfff_ffff_ffff_ffff : stack
- * 0xc000_0000_0000_0000 - 0xcfff_ffff_ffff_ffff : kernel reserved
- * 0xc000_0000_0000_0000 - endkernel-1 : kernel code & data
- * endkernel - msgbufp-1 : flat device tree
- * msgbufp - kernel_pdir-1 : message buffer
- * kernel_pdir - kernel_pp2d-1 : kernel page directory
- * kernel_pp2d - . : kernel pointers to page directory
- * pmap_zero_copy_min - crashdumpmap-1 : reserved for page zero/copy
- * crashdumpmap - ptbl_buf_pool_vabase-1 : reserved for ptbl bufs
- * ptbl_buf_pool_vabase - virtual_avail-1 : user page directories and page tables
- * virtual_avail - 0xcfff_ffff_ffff_ffff : actual free KVA space
- * 0xd000_0000_0000_0000 - 0xdfff_ffff_ffff_ffff : coprocessor region
- * 0xe000_0000_0000_0000 - 0xefff_ffff_ffff_ffff : mmio region
- * 0xf000_0000_0000_0000 - 0xffff_ffff_ffff_ffff : direct map
- * 0xf000_0000_0000_0000 - +Maxmem : physmem map
- * - 0xffff_ffff_ffff_ffff : device direct map
- */
- #include <sys/cdefs.h>
- __FBSDID("$FreeBSD$");
- #include "opt_ddb.h"
- #include "opt_kstack_pages.h"
- #include <sys/param.h>
- #include <sys/conf.h>
- #include <sys/malloc.h>
- #include <sys/ktr.h>
- #include <sys/proc.h>
- #include <sys/user.h>
- #include <sys/queue.h>
- #include <sys/systm.h>
- #include <sys/kernel.h>
- #include <sys/kerneldump.h>
- #include <sys/linker.h>
- #include <sys/msgbuf.h>
- #include <sys/lock.h>
- #include <sys/mutex.h>
- #include <sys/rwlock.h>
- #include <sys/sched.h>
- #include <sys/smp.h>
- #include <sys/vmmeter.h>
- #include <vm/vm.h>
- #include <vm/vm_param.h>
- #include <vm/vm_page.h>
- #include <vm/vm_kern.h>
- #include <vm/vm_pageout.h>
- #include <vm/vm_extern.h>
- #include <vm/vm_object.h>
- #include <vm/vm_map.h>
- #include <vm/vm_pager.h>
- #include <vm/vm_phys.h>
- #include <vm/vm_pagequeue.h>
- #include <vm/vm_dumpset.h>
- #include <vm/uma.h>
- #include <machine/_inttypes.h>
- #include <machine/cpu.h>
- #include <machine/pcb.h>
- #include <machine/platform.h>
- #include <machine/tlb.h>
- #include <machine/spr.h>
- #include <machine/md_var.h>
- #include <machine/mmuvar.h>
- #include <machine/pmap.h>
- #include <machine/pte.h>
- #include <ddb/ddb.h>
- #define SPARSE_MAPDEV
- /* Use power-of-two mappings in mmu_booke_mapdev(), to save entries. */
- #define POW2_MAPPINGS
- #ifdef DEBUG
- #define debugf(fmt, args...) printf(fmt, ##args)
- #else
- #define debugf(fmt, args...)
- #endif
- #ifdef __powerpc64__
- #define PRI0ptrX "016lx"
- #else
- #define PRI0ptrX "08x"
- #endif
- #define TODO panic("%s: not implemented", __func__);
- extern unsigned char _etext[];
- extern unsigned char _end[];
- extern uint32_t *bootinfo;
- vm_paddr_t kernload;
- vm_offset_t kernstart;
- vm_size_t kernsize;
- /* Message buffer and tables. */
- static vm_offset_t data_start;
- static vm_size_t data_end;
- /* Phys/avail memory regions. */
- static struct mem_region *availmem_regions;
- static int availmem_regions_sz;
- static struct mem_region *physmem_regions;
- static int physmem_regions_sz;
- #ifndef __powerpc64__
- /* Reserved KVA space and mutex for mmu_booke_zero_page. */
- static vm_offset_t zero_page_va;
- static struct mtx zero_page_mutex;
- /* Reserved KVA space and mutex for mmu_booke_copy_page. */
- static vm_offset_t copy_page_src_va;
- static vm_offset_t copy_page_dst_va;
- static struct mtx copy_page_mutex;
- #endif
- static struct mtx tlbivax_mutex;
- /**************************************************************************/
- /* PMAP */
- /**************************************************************************/
- static int mmu_booke_enter_locked(pmap_t, vm_offset_t, vm_page_t,
- vm_prot_t, u_int flags, int8_t psind);
- unsigned int kptbl_min; /* Index of the first kernel ptbl. */
- static uma_zone_t ptbl_root_zone;
- /*
- * If user pmap is processed with mmu_booke_remove and the resident count
- * drops to 0, there are no more pages to remove, so we need not continue.
- */
- #define PMAP_REMOVE_DONE(pmap) \
- ((pmap) != kernel_pmap && (pmap)->pm_stats.resident_count == 0)
- #if defined(COMPAT_FREEBSD32) || !defined(__powerpc64__)
- extern int elf32_nxstack;
- #endif
- /**************************************************************************/
- /* TLB and TID handling */
- /**************************************************************************/
- /* Translation ID busy table */
- static volatile pmap_t tidbusy[MAXCPU][TID_MAX + 1];
- /*
- * TLB0 capabilities (entry, way numbers etc.). These can vary between e500
- * core revisions and should be read from h/w registers during early config.
- */
- uint32_t tlb0_entries;
- uint32_t tlb0_ways;
- uint32_t tlb0_entries_per_way;
- uint32_t tlb1_entries;
- #define TLB0_ENTRIES (tlb0_entries)
- #define TLB0_WAYS (tlb0_ways)
- #define TLB0_ENTRIES_PER_WAY (tlb0_entries_per_way)
- #define TLB1_ENTRIES (tlb1_entries)
- static tlbtid_t tid_alloc(struct pmap *);
- #ifdef DDB
- #ifdef __powerpc64__
- static void tlb_print_entry(int, uint32_t, uint64_t, uint32_t, uint32_t);
- #else
- static void tlb_print_entry(int, uint32_t, uint32_t, uint32_t, uint32_t);
- #endif
- #endif
- static void tlb1_read_entry(tlb_entry_t *, unsigned int);
- static void tlb1_write_entry(tlb_entry_t *, unsigned int);
- static int tlb1_iomapped(int, vm_paddr_t, vm_size_t, vm_offset_t *);
- static vm_size_t tlb1_mapin_region(vm_offset_t, vm_paddr_t, vm_size_t, int);
- static __inline uint32_t tlb_calc_wimg(vm_paddr_t pa, vm_memattr_t ma);
- static vm_size_t tsize2size(unsigned int);
- static unsigned int size2tsize(vm_size_t);
- static unsigned long ilog2(unsigned long);
- static void set_mas4_defaults(void);
- static inline void tlb0_flush_entry(vm_offset_t);
- static inline unsigned int tlb0_tableidx(vm_offset_t, unsigned int);
- /**************************************************************************/
- /* Page table management */
- /**************************************************************************/
- static struct rwlock_padalign pvh_global_lock;
- /* Data for the pv entry allocation mechanism */
- static uma_zone_t pvzone;
- static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
- #define PV_ENTRY_ZONE_MIN 2048 /* min pv entries in uma zone */
- #ifndef PMAP_SHPGPERPROC
- #define PMAP_SHPGPERPROC 200
- #endif
- static vm_paddr_t pte_vatopa(pmap_t, vm_offset_t);
- static int pte_enter(pmap_t, vm_page_t, vm_offset_t, uint32_t, boolean_t);
- static int pte_remove(pmap_t, vm_offset_t, uint8_t);
- static pte_t *pte_find(pmap_t, vm_offset_t);
- static void kernel_pte_alloc(vm_offset_t, vm_offset_t);
- static pv_entry_t pv_alloc(void);
- static void pv_free(pv_entry_t);
- static void pv_insert(pmap_t, vm_offset_t, vm_page_t);
- static void pv_remove(pmap_t, vm_offset_t, vm_page_t);
- static void booke_pmap_init_qpages(void);
- static inline void tlb_miss_lock(void);
- static inline void tlb_miss_unlock(void);
- #ifdef SMP
- extern tlb_entry_t __boot_tlb1[];
- void pmap_bootstrap_ap(volatile uint32_t *);
- #endif
- /*
- * Kernel MMU interface
- */
- static void mmu_booke_clear_modify(vm_page_t);
- static void mmu_booke_copy(pmap_t, pmap_t, vm_offset_t,
- vm_size_t, vm_offset_t);
- static void mmu_booke_copy_page(vm_page_t, vm_page_t);
- static void mmu_booke_copy_pages(vm_page_t *,
- vm_offset_t, vm_page_t *, vm_offset_t, int);
- static int mmu_booke_enter(pmap_t, vm_offset_t, vm_page_t,
- vm_prot_t, u_int flags, int8_t psind);
- static void mmu_booke_enter_object(pmap_t, vm_offset_t, vm_offset_t,
- vm_page_t, vm_prot_t);
- static void mmu_booke_enter_quick(pmap_t, vm_offset_t, vm_page_t,
- vm_prot_t);
- static vm_paddr_t mmu_booke_extract(pmap_t, vm_offset_t);
- static vm_page_t mmu_booke_extract_and_hold(pmap_t, vm_offset_t,
- vm_prot_t);
- static void mmu_booke_init(void);
- static boolean_t mmu_booke_is_modified(vm_page_t);
- static boolean_t mmu_booke_is_prefaultable(pmap_t, vm_offset_t);
- static boolean_t mmu_booke_is_referenced(vm_page_t);
- static int mmu_booke_ts_referenced(vm_page_t);
- static vm_offset_t mmu_booke_map(vm_offset_t *, vm_paddr_t, vm_paddr_t,
- int);
- static int mmu_booke_mincore(pmap_t, vm_offset_t,
- vm_paddr_t *);
- static void mmu_booke_object_init_pt(pmap_t, vm_offset_t,
- vm_object_t, vm_pindex_t, vm_size_t);
- static boolean_t mmu_booke_page_exists_quick(pmap_t, vm_page_t);
- static void mmu_booke_page_init(vm_page_t);
- static int mmu_booke_page_wired_mappings(vm_page_t);
- static int mmu_booke_pinit(pmap_t);
- static void mmu_booke_pinit0(pmap_t);
- static void mmu_booke_protect(pmap_t, vm_offset_t, vm_offset_t,
- vm_prot_t);
- static void mmu_booke_qenter(vm_offset_t, vm_page_t *, int);
- static void mmu_booke_qremove(vm_offset_t, int);
- static void mmu_booke_release(pmap_t);
- static void mmu_booke_remove(pmap_t, vm_offset_t, vm_offset_t);
- static void mmu_booke_remove_all(vm_page_t);
- static void mmu_booke_remove_write(vm_page_t);
- static void mmu_booke_unwire(pmap_t, vm_offset_t, vm_offset_t);
- static void mmu_booke_zero_page(vm_page_t);
- static void mmu_booke_zero_page_area(vm_page_t, int, int);
- static void mmu_booke_activate(struct thread *);
- static void mmu_booke_deactivate(struct thread *);
- static void mmu_booke_bootstrap(vm_offset_t, vm_offset_t);
- static void *mmu_booke_mapdev(vm_paddr_t, vm_size_t);
- static void *mmu_booke_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t);
- static void mmu_booke_unmapdev(vm_offset_t, vm_size_t);
- static vm_paddr_t mmu_booke_kextract(vm_offset_t);
- static void mmu_booke_kenter(vm_offset_t, vm_paddr_t);
- static void mmu_booke_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t);
- static void mmu_booke_kremove(vm_offset_t);
- static boolean_t mmu_booke_dev_direct_mapped(vm_paddr_t, vm_size_t);
- static void mmu_booke_sync_icache(pmap_t, vm_offset_t,
- vm_size_t);
- static void mmu_booke_dumpsys_map(vm_paddr_t pa, size_t,
- void **);
- static void mmu_booke_dumpsys_unmap(vm_paddr_t pa, size_t,
- void *);
- static void mmu_booke_scan_init(void);
- static vm_offset_t mmu_booke_quick_enter_page(vm_page_t m);
- static void mmu_booke_quick_remove_page(vm_offset_t addr);
- static int mmu_booke_change_attr(vm_offset_t addr,
- vm_size_t sz, vm_memattr_t mode);
- static int mmu_booke_decode_kernel_ptr(vm_offset_t addr,
- int *is_user, vm_offset_t *decoded_addr);
- static void mmu_booke_page_array_startup(long);
- static boolean_t mmu_booke_page_is_mapped(vm_page_t m);
- static struct pmap_funcs mmu_booke_methods = {
- /* pmap dispatcher interface */
- .clear_modify = mmu_booke_clear_modify,
- .copy = mmu_booke_copy,
- .copy_page = mmu_booke_copy_page,
- .copy_pages = mmu_booke_copy_pages,
- .enter = mmu_booke_enter,
- .enter_object = mmu_booke_enter_object,
- .enter_quick = mmu_booke_enter_quick,
- .extract = mmu_booke_extract,
- .extract_and_hold = mmu_booke_extract_and_hold,
- .init = mmu_booke_init,
- .is_modified = mmu_booke_is_modified,
- .is_prefaultable = mmu_booke_is_prefaultable,
- .is_referenced = mmu_booke_is_referenced,
- .ts_referenced = mmu_booke_ts_referenced,
- .map = mmu_booke_map,
- .mincore = mmu_booke_mincore,
- .object_init_pt = mmu_booke_object_init_pt,
- .page_exists_quick = mmu_booke_page_exists_quick,
- .page_init = mmu_booke_page_init,
- .page_wired_mappings = mmu_booke_page_wired_mappings,
- .pinit = mmu_booke_pinit,
- .pinit0 = mmu_booke_pinit0,
- .protect = mmu_booke_protect,
- .qenter = mmu_booke_qenter,
- .qremove = mmu_booke_qremove,
- .release = mmu_booke_release,
- .remove = mmu_booke_remove,
- .remove_all = mmu_booke_remove_all,
- .remove_write = mmu_booke_remove_write,
- .sync_icache = mmu_booke_sync_icache,
- .unwire = mmu_booke_unwire,
- .zero_page = mmu_booke_zero_page,
- .zero_page_area = mmu_booke_zero_page_area,
- .activate = mmu_booke_activate,
- .deactivate = mmu_booke_deactivate,
- .quick_enter_page = mmu_booke_quick_enter_page,
- .quick_remove_page = mmu_booke_quick_remove_page,
- .page_array_startup = mmu_booke_page_array_startup,
- .page_is_mapped = mmu_booke_page_is_mapped,
- /* Internal interfaces */
- .bootstrap = mmu_booke_bootstrap,
- .dev_direct_mapped = mmu_booke_dev_direct_mapped,
- .mapdev = mmu_booke_mapdev,
- .mapdev_attr = mmu_booke_mapdev_attr,
- .kenter = mmu_booke_kenter,
- .kenter_attr = mmu_booke_kenter_attr,
- .kextract = mmu_booke_kextract,
- .kremove = mmu_booke_kremove,
- .unmapdev = mmu_booke_unmapdev,
- .change_attr = mmu_booke_change_attr,
- .decode_kernel_ptr = mmu_booke_decode_kernel_ptr,
- /* dumpsys() support */
- .dumpsys_map_chunk = mmu_booke_dumpsys_map,
- .dumpsys_unmap_chunk = mmu_booke_dumpsys_unmap,
- .dumpsys_pa_init = mmu_booke_scan_init,
- };
- MMU_DEF(booke_mmu, MMU_TYPE_BOOKE, mmu_booke_methods);
- #ifdef __powerpc64__
- #include "pmap_64.c"
- #else
- #include "pmap_32.c"
- #endif
- static vm_offset_t tlb1_map_base = VM_MAPDEV_BASE;
- static __inline uint32_t
- tlb_calc_wimg(vm_paddr_t pa, vm_memattr_t ma)
- {
- uint32_t attrib;
- int i;
- if (ma != VM_MEMATTR_DEFAULT) {
- switch (ma) {
- case VM_MEMATTR_UNCACHEABLE:
- return (MAS2_I | MAS2_G);
- case VM_MEMATTR_WRITE_COMBINING:
- case VM_MEMATTR_WRITE_BACK:
- case VM_MEMATTR_PREFETCHABLE:
- return (MAS2_I);
- case VM_MEMATTR_WRITE_THROUGH:
- return (MAS2_W | MAS2_M);
- case VM_MEMATTR_CACHEABLE:
- return (MAS2_M);
- }
- }
- /*
- * Assume the page is cache inhibited and access is guarded unless
- * it's in our available memory array.
- */
- attrib = _TLB_ENTRY_IO;
- for (i = 0; i < physmem_regions_sz; i++) {
- if ((pa >= physmem_regions[i].mr_start) &&
- (pa < (physmem_regions[i].mr_start +
- physmem_regions[i].mr_size))) {
- attrib = _TLB_ENTRY_MEM;
- break;
- }
- }
- return (attrib);
- }
- static inline void
- tlb_miss_lock(void)
- {
- #ifdef SMP
- struct pcpu *pc;
- if (!smp_started)
- return;
- STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
- if (pc != pcpup) {
- CTR3(KTR_PMAP, "%s: tlb miss LOCK of CPU=%d, "
- "tlb_lock=%p", __func__, pc->pc_cpuid, pc->pc_booke.tlb_lock);
- KASSERT((pc->pc_cpuid != PCPU_GET(cpuid)),
- ("tlb_miss_lock: tried to lock self"));
- tlb_lock(pc->pc_booke.tlb_lock);
- CTR1(KTR_PMAP, "%s: locked", __func__);
- }
- }
- #endif
- }
- static inline void
- tlb_miss_unlock(void)
- {
- #ifdef SMP
- struct pcpu *pc;
- if (!smp_started)
- return;
- STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
- if (pc != pcpup) {
- CTR2(KTR_PMAP, "%s: tlb miss UNLOCK of CPU=%d",
- __func__, pc->pc_cpuid);
- tlb_unlock(pc->pc_booke.tlb_lock);
- CTR1(KTR_PMAP, "%s: unlocked", __func__);
- }
- }
- #endif
- }
- /* Return number of entries in TLB0. */
- static __inline void
- tlb0_get_tlbconf(void)
- {
- uint32_t tlb0_cfg;
- tlb0_cfg = mfspr(SPR_TLB0CFG);
- tlb0_entries = tlb0_cfg & TLBCFG_NENTRY_MASK;
- tlb0_ways = (tlb0_cfg & TLBCFG_ASSOC_MASK) >> TLBCFG_ASSOC_SHIFT;
- tlb0_entries_per_way = tlb0_entries / tlb0_ways;
- }
- /* Return number of entries in TLB1. */
- static __inline void
- tlb1_get_tlbconf(void)
- {
- uint32_t tlb1_cfg;
- tlb1_cfg = mfspr(SPR_TLB1CFG);
- tlb1_entries = tlb1_cfg & TLBCFG_NENTRY_MASK;
- }
- /**************************************************************************/
- /* Page table related */
- /**************************************************************************/
- /* Allocate pv_entry structure. */
- pv_entry_t
- pv_alloc(void)
- {
- pv_entry_t pv;
- pv_entry_count++;
- if (pv_entry_count > pv_entry_high_water)
- pagedaemon_wakeup(0); /* XXX powerpc NUMA */
- pv = uma_zalloc(pvzone, M_NOWAIT);
- return (pv);
- }
- /* Free pv_entry structure. */
- static __inline void
- pv_free(pv_entry_t pve)
- {
- pv_entry_count--;
- uma_zfree(pvzone, pve);
- }
- /* Allocate and initialize pv_entry structure. */
- static void
- pv_insert(pmap_t pmap, vm_offset_t va, vm_page_t m)
- {
- pv_entry_t pve;
- //int su = (pmap == kernel_pmap);
- //debugf("pv_insert: s (su = %d pmap = 0x%08x va = 0x%08x m = 0x%08x)\n", su,
- // (u_int32_t)pmap, va, (u_int32_t)m);
- pve = pv_alloc();
- if (pve == NULL)
- panic("pv_insert: no pv entries!");
- pve->pv_pmap = pmap;
- pve->pv_va = va;
- /* add to pv_list */
- PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- rw_assert(&pvh_global_lock, RA_WLOCKED);
- TAILQ_INSERT_TAIL(&m->md.pv_list, pve, pv_link);
- //debugf("pv_insert: e\n");
- }
- /* Destroy pv entry. */
- static void
- pv_remove(pmap_t pmap, vm_offset_t va, vm_page_t m)
- {
- pv_entry_t pve;
- //int su = (pmap == kernel_pmap);
- //debugf("pv_remove: s (su = %d pmap = 0x%08x va = 0x%08x)\n", su, (u_int32_t)pmap, va);
- PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- rw_assert(&pvh_global_lock, RA_WLOCKED);
- /* find pv entry */
- TAILQ_FOREACH(pve, &m->md.pv_list, pv_link) {
- if ((pmap == pve->pv_pmap) && (va == pve->pv_va)) {
- /* remove from pv_list */
- TAILQ_REMOVE(&m->md.pv_list, pve, pv_link);
- if (TAILQ_EMPTY(&m->md.pv_list))
- vm_page_aflag_clear(m, PGA_WRITEABLE);
- /* free pv entry struct */
- pv_free(pve);
- break;
- }
- }
- //debugf("pv_remove: e\n");
- }
- /**************************************************************************/
- /* PMAP related */
- /**************************************************************************/
- /*
- * This is called during booke_init, before the system is really initialized.
- */
- static void
- mmu_booke_bootstrap(vm_offset_t start, vm_offset_t kernelend)
- {
- vm_paddr_t phys_kernelend;
- struct mem_region *mp, *mp1;
- int cnt, i, j;
- vm_paddr_t s, e, sz;
- vm_paddr_t physsz, hwphyssz;
- u_int phys_avail_count;
- vm_size_t kstack0_sz;
- vm_paddr_t kstack0_phys;
- vm_offset_t kstack0;
- void *dpcpu;
- debugf("mmu_booke_bootstrap: entered\n");
- /* Set interesting system properties */
- #ifdef __powerpc64__
- hw_direct_map = 1;
- #else
- hw_direct_map = 0;
- #endif
- #if defined(COMPAT_FREEBSD32) || !defined(__powerpc64__)
- elf32_nxstack = 1;
- #endif
- /* Initialize invalidation mutex */
- mtx_init(&tlbivax_mutex, "tlbivax", NULL, MTX_SPIN);
- /* Read TLB0 size and associativity. */
- tlb0_get_tlbconf();
- /*
- * Align kernel start and end address (kernel image).
- * Note that kernel end does not necessarily relate to kernsize.
- * kernsize is the size of the kernel that is actually mapped.
- */
- data_start = round_page(kernelend);
- data_end = data_start;
- /* Allocate the dynamic per-cpu area. */
- dpcpu = (void *)data_end;
- data_end += DPCPU_SIZE;
- /* Allocate space for the message buffer. */
- msgbufp = (struct msgbuf *)data_end;
- data_end += msgbufsize;
- debugf(" msgbufp at 0x%"PRI0ptrX" end = 0x%"PRI0ptrX"\n",
- (uintptr_t)msgbufp, data_end);
- data_end = round_page(data_end);
- data_end = round_page(mmu_booke_alloc_kernel_pgtables(data_end));
- /* Retrieve phys/avail mem regions */
- mem_regions(&physmem_regions, &physmem_regions_sz,
- &availmem_regions, &availmem_regions_sz);
- if (PHYS_AVAIL_ENTRIES < availmem_regions_sz)
- panic("mmu_booke_bootstrap: phys_avail too small");
- data_end = round_page(data_end);
- vm_page_array = (vm_page_t)data_end;
- /*
- * Get a rough idea (upper bound) on the size of the page array. The
- * vm_page_array will not handle any more pages than we have in the
- * avail_regions array, and most likely much less.
- */
- sz = 0;
- for (mp = availmem_regions; mp->mr_size; mp++) {
- sz += mp->mr_size;
- }
- sz = (round_page(sz) / (PAGE_SIZE + sizeof(struct vm_page)));
- data_end += round_page(sz * sizeof(struct vm_page));
- /* Pre-round up to 1MB. This wastes some space, but saves TLB entries */
- data_end = roundup2(data_end, 1 << 20);
- debugf(" data_end: 0x%"PRI0ptrX"\n", data_end);
- debugf(" kernstart: %#zx\n", kernstart);
- debugf(" kernsize: %#zx\n", kernsize);
- if (data_end - kernstart > kernsize) {
- kernsize += tlb1_mapin_region(kernstart + kernsize,
- kernload + kernsize, (data_end - kernstart) - kernsize,
- _TLB_ENTRY_MEM);
- }
- data_end = kernstart + kernsize;
- debugf(" updated data_end: 0x%"PRI0ptrX"\n", data_end);
- /*
- * Clear the structures - note we can only do it safely after the
- * possible additional TLB1 translations are in place (above) so that
- * all range up to the currently calculated 'data_end' is covered.
- */
- bzero((void *)data_start, data_end - data_start);
- dpcpu_init(dpcpu, 0);
- /*******************************************************/
- /* Set the start and end of kva. */
- /*******************************************************/
- virtual_avail = round_page(data_end);
- virtual_end = VM_MAX_KERNEL_ADDRESS;
- #ifndef __powerpc64__
- /* Allocate KVA space for page zero/copy operations. */
- zero_page_va = virtual_avail;
- virtual_avail += PAGE_SIZE;
- copy_page_src_va = virtual_avail;
- virtual_avail += PAGE_SIZE;
- copy_page_dst_va = virtual_avail;
- virtual_avail += PAGE_SIZE;
- debugf("zero_page_va = 0x%"PRI0ptrX"\n", zero_page_va);
- debugf("copy_page_src_va = 0x%"PRI0ptrX"\n", copy_page_src_va);
- debugf("copy_page_dst_va = 0x%"PRI0ptrX"\n", copy_page_dst_va);
- /* Initialize page zero/copy mutexes. */
- mtx_init(&zero_page_mutex, "mmu_booke_zero_page", NULL, MTX_DEF);
- mtx_init(©_page_mutex, "mmu_booke_copy_page", NULL, MTX_DEF);
- /* Allocate KVA space for ptbl bufs. */
- ptbl_buf_pool_vabase = virtual_avail;
- virtual_avail += PTBL_BUFS * PTBL_PAGES * PAGE_SIZE;
- debugf("ptbl_buf_pool_vabase = 0x%"PRI0ptrX" end = 0x%"PRI0ptrX"\n",
- ptbl_buf_pool_vabase, virtual_avail);
- #endif
- /* Calculate corresponding physical addresses for the kernel region. */
- phys_kernelend = kernload + kernsize;
- debugf("kernel image and allocated data:\n");
- debugf(" kernload = 0x%09jx\n", (uintmax_t)kernload);
- debugf(" kernstart = 0x%"PRI0ptrX"\n", kernstart);
- debugf(" kernsize = 0x%"PRI0ptrX"\n", kernsize);
- /*
- * Remove kernel physical address range from avail regions list. Page
- * align all regions. Non-page aligned memory isn't very interesting
- * to us. Also, sort the entries for ascending addresses.
- */
- sz = 0;
- cnt = availmem_regions_sz;
- debugf("processing avail regions:\n");
- for (mp = availmem_regions; mp->mr_size; mp++) {
- s = mp->mr_start;
- e = mp->mr_start + mp->mr_size;
- debugf(" %09jx-%09jx -> ", (uintmax_t)s, (uintmax_t)e);
- /* Check whether this region holds all of the kernel. */
- if (s < kernload && e > phys_kernelend) {
- availmem_regions[cnt].mr_start = phys_kernelend;
- availmem_regions[cnt++].mr_size = e - phys_kernelend;
- e = kernload;
- }
- /* Look whether this regions starts within the kernel. */
- if (s >= kernload && s < phys_kernelend) {
- if (e <= phys_kernelend)
- goto empty;
- s = phys_kernelend;
- }
- /* Now look whether this region ends within the kernel. */
- if (e > kernload && e <= phys_kernelend) {
- if (s >= kernload)
- goto empty;
- e = kernload;
- }
- /* Now page align the start and size of the region. */
- s = round_page(s);
- e = trunc_page(e);
- if (e < s)
- e = s;
- sz = e - s;
- debugf("%09jx-%09jx = %jx\n",
- (uintmax_t)s, (uintmax_t)e, (uintmax_t)sz);
- /* Check whether some memory is left here. */
- if (sz == 0) {
- empty:
- memmove(mp, mp + 1,
- (cnt - (mp - availmem_regions)) * sizeof(*mp));
- cnt--;
- mp--;
- continue;
- }
- /* Do an insertion sort. */
- for (mp1 = availmem_regions; mp1 < mp; mp1++)
- if (s < mp1->mr_start)
- break;
- if (mp1 < mp) {
- memmove(mp1 + 1, mp1, (char *)mp - (char *)mp1);
- mp1->mr_start = s;
- mp1->mr_size = sz;
- } else {
- mp->mr_start = s;
- mp->mr_size = sz;
- }
- }
- availmem_regions_sz = cnt;
- /*******************************************************/
- /* Steal physical memory for kernel stack from the end */
- /* of the first avail region */
- /*******************************************************/
- kstack0_sz = kstack_pages * PAGE_SIZE;
- kstack0_phys = availmem_regions[0].mr_start +
- availmem_regions[0].mr_size;
- kstack0_phys -= kstack0_sz;
- availmem_regions[0].mr_size -= kstack0_sz;
- /*******************************************************/
- /* Fill in phys_avail table, based on availmem_regions */
- /*******************************************************/
- phys_avail_count = 0;
- physsz = 0;
- hwphyssz = 0;
- TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz);
- debugf("fill in phys_avail:\n");
- for (i = 0, j = 0; i < availmem_regions_sz; i++, j += 2) {
- debugf(" region: 0x%jx - 0x%jx (0x%jx)\n",
- (uintmax_t)availmem_regions[i].mr_start,
- (uintmax_t)availmem_regions[i].mr_start +
- availmem_regions[i].mr_size,
- (uintmax_t)availmem_regions[i].mr_size);
- if (hwphyssz != 0 &&
- (physsz + availmem_regions[i].mr_size) >= hwphyssz) {
- debugf(" hw.physmem adjust\n");
- if (physsz < hwphyssz) {
- phys_avail[j] = availmem_regions[i].mr_start;
- phys_avail[j + 1] =
- availmem_regions[i].mr_start +
- hwphyssz - physsz;
- physsz = hwphyssz;
- phys_avail_count++;
- dump_avail[j] = phys_avail[j];
- dump_avail[j + 1] = phys_avail[j + 1];
- }
- break;
- }
- phys_avail[j] = availmem_regions[i].mr_start;
- phys_avail[j + 1] = availmem_regions[i].mr_start +
- availmem_regions[i].mr_size;
- phys_avail_count++;
- physsz += availmem_regions[i].mr_size;
- dump_avail[j] = phys_avail[j];
- dump_avail[j + 1] = phys_avail[j + 1];
- }
- physmem = btoc(physsz);
- /* Calculate the last available physical address. */
- for (i = 0; phys_avail[i + 2] != 0; i += 2)
- ;
- Maxmem = powerpc_btop(phys_avail[i + 1]);
- debugf("Maxmem = 0x%08lx\n", Maxmem);
- debugf("phys_avail_count = %d\n", phys_avail_count);
- debugf("physsz = 0x%09jx physmem = %jd (0x%09jx)\n",
- (uintmax_t)physsz, (uintmax_t)physmem, (uintmax_t)physmem);
- #ifdef __powerpc64__
- /*
- * Map the physical memory contiguously in TLB1.
- * Round so it fits into a single mapping.
- */
- tlb1_mapin_region(DMAP_BASE_ADDRESS, 0,
- phys_avail[i + 1], _TLB_ENTRY_MEM);
- #endif
- /*******************************************************/
- /* Initialize (statically allocated) kernel pmap. */
- /*******************************************************/
- PMAP_LOCK_INIT(kernel_pmap);
- debugf("kernel_pmap = 0x%"PRI0ptrX"\n", (uintptr_t)kernel_pmap);
- kernel_pte_alloc(virtual_avail, kernstart);
- for (i = 0; i < MAXCPU; i++) {
- kernel_pmap->pm_tid[i] = TID_KERNEL;
-
- /* Initialize each CPU's tidbusy entry 0 with kernel_pmap */
- tidbusy[i][TID_KERNEL] = kernel_pmap;
- }
- /* Mark kernel_pmap active on all CPUs */
- CPU_FILL(&kernel_pmap->pm_active);
- /*
- * Initialize the global pv list lock.
- */
- rw_init(&pvh_global_lock, "pmap pv global");
- /*******************************************************/
- /* Final setup */
- /*******************************************************/
- /* Enter kstack0 into kernel map, provide guard page */
- kstack0 = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE;
- thread0.td_kstack = kstack0;
- thread0.td_kstack_pages = kstack_pages;
- debugf("kstack_sz = 0x%08jx\n", (uintmax_t)kstack0_sz);
- debugf("kstack0_phys at 0x%09jx - 0x%09jx\n",
- (uintmax_t)kstack0_phys, (uintmax_t)kstack0_phys + kstack0_sz);
- debugf("kstack0 at 0x%"PRI0ptrX" - 0x%"PRI0ptrX"\n",
- kstack0, kstack0 + kstack0_sz);
- virtual_avail += KSTACK_GUARD_PAGES * PAGE_SIZE + kstack0_sz;
- for (i = 0; i < kstack_pages; i++) {
- mmu_booke_kenter(kstack0, kstack0_phys);
- kstack0 += PAGE_SIZE;
- kstack0_phys += PAGE_SIZE;
- }
- pmap_bootstrapped = 1;
- debugf("virtual_avail = %"PRI0ptrX"\n", virtual_avail);
- debugf("virtual_end = %"PRI0ptrX"\n", virtual_end);
- debugf("mmu_booke_bootstrap: exit\n");
- }
- #ifdef SMP
- void
- tlb1_ap_prep(void)
- {
- tlb_entry_t *e, tmp;
- unsigned int i;
- /* Prepare TLB1 image for AP processors */
- e = __boot_tlb1;
- for (i = 0; i < TLB1_ENTRIES; i++) {
- tlb1_read_entry(&tmp, i);
- if ((tmp.mas1 & MAS1_VALID) && (tmp.mas2 & _TLB_ENTRY_SHARED))
- memcpy(e++, &tmp, sizeof(tmp));
- }
- }
- void
- pmap_bootstrap_ap(volatile uint32_t *trcp __unused)
- {
- int i;
- /*
- * Finish TLB1 configuration: the BSP already set up its TLB1 and we
- * have the snapshot of its contents in the s/w __boot_tlb1[] table
- * created by tlb1_ap_prep(), so use these values directly to
- * (re)program AP's TLB1 hardware.
- *
- * Start at index 1 because index 0 has the kernel map.
- */
- for (i = 1; i < TLB1_ENTRIES; i++) {
- if (__boot_tlb1[i].mas1 & MAS1_VALID)
- tlb1_write_entry(&__boot_tlb1[i], i);
- }
- set_mas4_defaults();
- }
- #endif
- static void
- booke_pmap_init_qpages(void)
- {
- struct pcpu *pc;
- int i;
- CPU_FOREACH(i) {
- pc = pcpu_find(i);
- pc->pc_qmap_addr = kva_alloc(PAGE_SIZE);
- if (pc->pc_qmap_addr == 0)
- panic("pmap_init_qpages: unable to allocate KVA");
- }
- }
- SYSINIT(qpages_init, SI_SUB_CPU, SI_ORDER_ANY, booke_pmap_init_qpages, NULL);
- /*
- * Get the physical page address for the given pmap/virtual address.
- */
- static vm_paddr_t
- mmu_booke_extract(pmap_t pmap, vm_offset_t va)
- {
- vm_paddr_t pa;
- PMAP_LOCK(pmap);
- pa = pte_vatopa(pmap, va);
- PMAP_UNLOCK(pmap);
- return (pa);
- }
- /*
- * Extract the physical page address associated with the given
- * kernel virtual address.
- */
- static vm_paddr_t
- mmu_booke_kextract(vm_offset_t va)
- {
- tlb_entry_t e;
- vm_paddr_t p = 0;
- int i;
- #ifdef __powerpc64__
- if (va >= DMAP_BASE_ADDRESS && va <= DMAP_MAX_ADDRESS)
- return (DMAP_TO_PHYS(va));
- #endif
- if (va >= VM_MIN_KERNEL_ADDRESS && va <= VM_MAX_KERNEL_ADDRESS)
- p = pte_vatopa(kernel_pmap, va);
- if (p == 0) {
- /* Check TLB1 mappings */
- for (i = 0; i < TLB1_ENTRIES; i++) {
- tlb1_read_entry(&e, i);
- if (!(e.mas1 & MAS1_VALID))
- continue;
- if (va >= e.virt && va < e.virt + e.size)
- return (e.phys + (va - e.virt));
- }
- }
- return (p);
- }
- /*
- * Initialize the pmap module.
- * Called by vm_init, to initialize any structures that the pmap
- * system needs to map virtual memory.
- */
- static void
- mmu_booke_init()
- {
- int shpgperproc = PMAP_SHPGPERPROC;
- /*
- * Initialize the address space (zone) for the pv entries. Set a
- * high water mark so that the system can recover from excessive
- * numbers of pv entries.
- */
- pvzone = uma_zcreate("PV ENTRY", sizeof(struct pv_entry), NULL, NULL,
- NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE);
- TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
- pv_entry_max = shpgperproc * maxproc + vm_cnt.v_page_count;
- TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
- pv_entry_high_water = 9 * (pv_entry_max / 10);
- uma_zone_reserve_kva(pvzone, pv_entry_max);
- /* Pre-fill pvzone with initial number of pv entries. */
- uma_prealloc(pvzone, PV_ENTRY_ZONE_MIN);
- /* Create a UMA zone for page table roots. */
- ptbl_root_zone = uma_zcreate("pmap root", PMAP_ROOT_SIZE,
- NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, UMA_ZONE_VM);
- /* Initialize ptbl allocation. */
- ptbl_init();
- }
- /*
- * Map a list of wired pages into kernel virtual address space. This is
- * intended for temporary mappings which do not need page modification or
- * references recorded. Existing mappings in the region are overwritten.
- */
- static void
- mmu_booke_qenter(vm_offset_t sva, vm_page_t *m, int count)
- {
- vm_offset_t va;
- va = sva;
- while (count-- > 0) {
- mmu_booke_kenter(va, VM_PAGE_TO_PHYS(*m));
- va += PAGE_SIZE;
- m++;
- }
- }
- /*
- * Remove page mappings from kernel virtual address space. Intended for
- * temporary mappings entered by mmu_booke_qenter.
- */
- static void
- mmu_booke_qremove(vm_offset_t sva, int count)
- {
- vm_offset_t va;
- va = sva;
- while (count-- > 0) {
- mmu_booke_kremove(va);
- va += PAGE_SIZE;
- }
- }
- /*
- * Map a wired page into kernel virtual address space.
- */
- static void
- mmu_booke_kenter(vm_offset_t va, vm_paddr_t pa)
- {
- mmu_booke_kenter_attr(va, pa, VM_MEMATTR_DEFAULT);
- }
- static void
- mmu_booke_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma)
- {
- uint32_t flags;
- pte_t *pte;
- KASSERT(((va >= VM_MIN_KERNEL_ADDRESS) &&
- (va <= VM_MAX_KERNEL_ADDRESS)), ("mmu_booke_kenter: invalid va"));
- flags = PTE_SR | PTE_SW | PTE_SX | PTE_WIRED | PTE_VALID;
- flags |= tlb_calc_wimg(pa, ma) << PTE_MAS2_SHIFT;
- flags |= PTE_PS_4KB;
- pte = pte_find(kernel_pmap, va);
- KASSERT((pte != NULL), ("mmu_booke_kenter: invalid va. NULL PTE"));
- mtx_lock_spin(&tlbivax_mutex);
- tlb_miss_lock();
- if (PTE_ISVALID(pte)) {
- CTR1(KTR_PMAP, "%s: replacing entry!", __func__);
- /* Flush entry from TLB0 */
- tlb0_flush_entry(va);
- }
- *pte = PTE_RPN_FROM_PA(pa) | flags;
- //debugf("mmu_booke_kenter: pdir_idx = %d ptbl_idx = %d va=0x%08x "
- // "pa=0x%08x rpn=0x%08x flags=0x%08x\n",
- // pdir_idx, ptbl_idx, va, pa, pte->rpn, pte->flags);
- /* Flush the real memory from the instruction cache. */
- if ((flags & (PTE_I | PTE_G)) == 0)
- __syncicache((void *)va, PAGE_SIZE);
- tlb_miss_unlock();
- mtx_unlock_spin(&tlbivax_mutex);
- }
- /*
- * Remove a page from kernel page table.
- */
- static void
- mmu_booke_kremove(vm_offset_t va)
- {
- pte_t *pte;
- CTR2(KTR_PMAP,"%s: s (va = 0x%"PRI0ptrX")\n", __func__, va);
- KASSERT(((va >= VM_MIN_KERNEL_ADDRESS) &&
- (va <= VM_MAX_KERNEL_ADDRESS)),
- ("mmu_booke_kremove: invalid va"));
- pte = pte_find(kernel_pmap, va);
- if (!PTE_ISVALID(pte)) {
- CTR1(KTR_PMAP, "%s: invalid pte", __func__);
- return;
- }
- mtx_lock_spin(&tlbivax_mutex);
- tlb_miss_lock();
- /* Invalidate entry in TLB0, update PTE. */
- tlb0_flush_entry(va);
- *pte = 0;
- tlb_miss_unlock();
- mtx_unlock_spin(&tlbivax_mutex);
- }
- /*
- * Figure out where a given kernel pointer (usually in a fault) points
- * to from the VM's perspective, potentially remapping into userland's
- * address space.
- */
- static int
- mmu_booke_decode_kernel_ptr(vm_offset_t addr, int *is_user,
- vm_offset_t *decoded_addr)
- {
- if (trunc_page(addr) <= VM_MAXUSER_ADDRESS)
- *is_user = 1;
- else
- *is_user = 0;
- *decoded_addr = addr;
- return (0);
- }
- static boolean_t
- mmu_booke_page_is_mapped(vm_page_t m)
- {
- return (!TAILQ_EMPTY(&(m)->md.pv_list));
- }
- /*
- * Initialize pmap associated with process 0.
- */
- static void
- mmu_booke_pinit0(pmap_t pmap)
- {
- PMAP_LOCK_INIT(pmap);
- mmu_booke_pinit(pmap);
- PCPU_SET(curpmap, pmap);
- }
- /*
- * Insert the given physical page at the specified virtual address in the
- * target physical map with the protection requested. If specified the page
- * will be wired down.
- */
- static int
- mmu_booke_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,
- vm_prot_t prot, u_int flags, int8_t psind)
- {
- int error;
- rw_wlock(&pvh_global_lock);
- PMAP_LOCK(pmap);
- error = mmu_booke_enter_locked(pmap, va, m, prot, flags, psind);
- PMAP_UNLOCK(pmap);
- rw_wunlock(&pvh_global_lock);
- return (error);
- }
- static int
- mmu_booke_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
- vm_prot_t prot, u_int pmap_flags, int8_t psind __unused)
- {
- pte_t *pte;
- vm_paddr_t pa;
- pte_t flags;
- int error, su, sync;
- pa = VM_PAGE_TO_PHYS(m);
- su = (pmap == kernel_pmap);
- sync = 0;
- //debugf("mmu_booke_enter_locked: s (pmap=0x%08x su=%d tid=%d m=0x%08x va=0x%08x "
- // "pa=0x%08x prot=0x%08x flags=%#x)\n",
- // (u_int32_t)pmap, su, pmap->pm_tid,
- // (u_int32_t)m, va, pa, prot, flags);
- if (su) {
- KASSERT(((va >= virtual_avail) &&
- (va <= VM_MAX_KERNEL_ADDRESS)),
- ("mmu_booke_enter_locked: kernel pmap, non kernel va"));
- } else {
- KASSERT((va <= VM_MAXUSER_ADDRESS),
- ("mmu_booke_enter_locked: user pmap, non user va"));
- }
- if ((m->oflags & VPO_UNMANAGED) == 0) {
- if ((pmap_flags & PMAP_ENTER_QUICK_LOCKED) == 0)
- VM_PAGE_OBJECT_BUSY_ASSERT(m);
- else
- VM_OBJECT_ASSERT_LOCKED(m->object);
- }
- PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- /*
- * If there is an existing mapping, and the physical address has not
- * changed, must be protection or wiring change.
- */
- if (((pte = pte_find(pmap, va)) != NULL) &&
- (PTE_ISVALID(pte)) && (PTE_PA(pte) == pa)) {
-
- /*
- * Before actually updating pte->flags we calculate and
- * prepare its new value in a helper var.
- */
- flags = *pte;
- flags &= ~(PTE_UW | PTE_UX | PTE_SW | PTE_SX | PTE_MODIFIED);
- /* Wiring change, just update stats. */
- if ((pmap_flags & PMAP_ENTER_WIRED) != 0) {
- if (!PTE_ISWIRED(pte)) {
- flags |= PTE_WIRED;
- pmap->pm_stats.wired_count++;
- }
- } else {
- if (PTE_ISWIRED(pte)) {
- flags &= ~PTE_WIRED;
- pmap->pm_stats.wired_count--;
- }
- }
- if (prot & VM_PROT_WRITE) {
- /* Add write permissions. */
- flags |= PTE_SW;
- if (!su)
- flags |= PTE_UW;
- if ((flags & PTE_MANAGED) != 0)
- vm_page_aflag_set(m, PGA_WRITEABLE);
- } else {
- /* Handle modified pages, sense modify status. */
- /*
- * The PTE_MODIFIED flag could be set by underlying
- * TLB misses since we last read it (above), possibly
- * other CPUs could update it so we check in the PTE
- * directly rather than rely on that saved local flags
- * copy.
- */
- if (PTE_ISMODIFIED(pte))
- vm_page_dirty(m);
- }
- if (prot & VM_PROT_EXECUTE) {
- flags |= PTE_SX;
- if (!su)
- flags |= PTE_UX;
- /*
- * Check existing flags for execute permissions: if we
- * are turning execute permissions on, icache should
- * be flushed.
- */
- if ((*pte & (PTE_UX | PTE_SX)) == 0)
- sync++;
- }
- flags &= ~PTE_REFERENCED;
- /*
- * The new flags value is all calculated -- only now actually
- * update the PTE.
- */
- mtx_lock_spin(&tlbivax_mutex);
- tlb_miss_lock();
- tlb0_flush_entry(va);
- *pte &= ~PTE_FLAGS_MASK;
- *pte |= flags;
- tlb_miss_unlock();
- mtx_unlock_spin(&tlbivax_mutex);
- } else {
- /*
- * If there is an existing mapping, but it's for a different
- * physical address, pte_enter() will delete the old mapping.
- */
- //if ((pte != NULL) && PTE_ISVALID(pte))
- // debugf("mmu_booke_enter_locked: replace\n");
- //else
- // debugf("mmu_booke_enter_locked: new\n");
- /* Now set up the flags and install the new mapping. */
- flags = (PTE_SR | PTE_VALID);
- flags |= PTE_M;
- if (!su)
- flags |= PTE_UR;
- if (prot & VM_PROT_WRITE) {
- flags |= PTE_SW;
- if (!su)
- flags |= PTE_UW;
- if ((m->oflags & VPO_UNMANAGED) == 0)
- vm_page_aflag_set(m, PGA_WRITEABLE);
- }
- if (prot & VM_PROT_EXECUTE) {
- flags |= PTE_SX;
- if (!su)
- flags |= PTE_UX;
- }
- /* If its wired update stats. */
- if ((pmap_flags & PMAP_ENTER_WIRED) != 0)
- flags |= PTE_WIRED;
- error = pte_enter(pmap, m, va, flags,
- (pmap_flags & PMAP_ENTER_NOSLEEP) != 0);
- if (error != 0)
- return (KERN_RESOURCE_SHORTAGE);
- if ((flags & PMAP_ENTER_WIRED) != 0)
- pmap->pm_stats.wired_count++;
- /* Flush the real memory from the instruction cache. */
- if (prot & VM_PROT_EXECUTE)
- sync++;
- }
- if (sync && (su || pmap == PCPU_GET(curpmap))) {
- __syncicache((void *)va, PAGE_SIZE);
- sync = 0;
- }
- return (KERN_SUCCESS);
- }
- /*
- * Maps a sequence of resident pages belonging to the same object.
- * The sequence begins with the given page m_start. This page is
- * mapped at the given virtual address start. Each subsequent page is
- * mapped at a virtual address that is offset from start by the same
- * amount as the page is offset from m_start within the object. The
- * last page in the sequence is the page with the largest offset from
- * m_start that can be mapped at a virtual address less than the given
- * virtual address end. Not every virtual page between start and end
- * is mapped; only those for which a resident page exists with the
- * corresponding offset from m_start are mapped.
- */
- static void
- mmu_booke_enter_object(pmap_t pmap, vm_offset_t start,
- vm_offset_t end, vm_page_t m_start, vm_prot_t prot)
- {
- vm_page_t m;
- vm_pindex_t diff, psize;
- VM_OBJECT_ASSERT_LOCKED(m_start->object);
- psize = atop(end - start);
- m = m_start;
- rw_wlock(&pvh_global_lock);
- PMAP_LOCK(pmap);
- while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
- mmu_booke_enter_locked(pmap, start + ptoa(diff), m,
- prot & (VM_PROT_READ | VM_PROT_EXECUTE),
- PMAP_ENTER_NOSLEEP | PMAP_ENTER_QUICK_LOCKED, 0);
- m = TAILQ_NEXT(m, listq);
- }
- PMAP_UNLOCK(pmap);
- rw_wunlock(&pvh_global_lock);
- }
- static void
- mmu_booke_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m,
- vm_prot_t prot)
- {
- rw_wlock(&pvh_global_lock);
- PMAP_LOCK(pmap);
- mmu_booke_enter_locked(pmap, va, m,
- prot & (VM_PROT_READ | VM_PROT_EXECUTE), PMAP_ENTER_NOSLEEP |
- PMAP_ENTER_QUICK_LOCKED, 0);
- PMAP_UNLOCK(pmap);
- rw_wunlock(&pvh_global_lock);
- }
- /*
- * Remove the given range of addresses from the specified map.
- *
- * It is assumed that the start and end are properly rounded to the page size.
- */
- static void
- mmu_booke_remove(pmap_t pmap, vm_offset_t va, vm_offset_t endva)
- {
- pte_t *pte;
- uint8_t hold_flag;
- int su = (pmap == kernel_pmap);
- //debugf("mmu_booke_remove: s (su = %d pmap=0x%08x tid=%d va=0x%08x endva=0x%08x)\n",
- // su, (u_int32_t)pmap, pmap->pm_tid, va, endva);
- if (su) {
- KASSERT(((va >= virtual_avail) &&
- (va <= VM_MAX_KERNEL_ADDRESS)),
- ("mmu_booke_remove: kernel pmap, non kernel va"));
- } else {
- KASSERT((va <= VM_MAXUSER_ADDRESS),
- ("mmu_booke_remove: user pmap, non user va"));
- }
- if (PMAP_REMOVE_DONE(pmap)) {
- //debugf("mmu_booke_remove: e (empty)\n");
- return;
- }
- hold_flag = PTBL_HOLD_FLAG(pmap);
- //debugf("mmu_booke_remove: hold_flag = %d\n", hold_flag);
- rw_wlock(&pvh_global_lock);
- PMAP_LOCK(pmap);
- for (; va < endva; va += PAGE_SIZE) {
- pte = pte_find_next(pmap, &va);
- if ((pte == NULL) || !PTE_ISVALID(pte))
- break;
- if (va >= endva)
- break;
- pte_remove(pmap, va, hold_flag);
- }
- PMAP_UNLOCK(pmap);
- rw_wunlock(&pvh_global_lock);
- //debugf("mmu_booke_remove: e\n");
- }
- /*
- * Remove physical page from all pmaps in which it resides.
- */
- static void
- mmu_booke_remove_all(vm_page_t m)
- {
- pv_entry_t pv, pvn;
- uint8_t hold_flag;
- rw_wlock(&pvh_global_lock);
- TAILQ_FOREACH_SAFE(pv, &m->md.pv_list, pv_link, pvn) {
- PMAP_LOCK(pv->pv_pmap);
- hold_flag = PTBL_HOLD_FLAG(pv->pv_pmap);
- pte_remove(pv->pv_pmap, pv->pv_va, hold_flag);
- PMAP_UNLOCK(pv->pv_pmap);
- }
- vm_page_aflag_clear(m, PGA_WRITEABLE);
- rw_wunlock(&pvh_global_lock);
- }
- /*
- * Map a range of physical addresses into kernel virtual address space.
- */
- static vm_offset_t
- mmu_booke_map(vm_offset_t *virt, vm_paddr_t pa_start,
- vm_paddr_t pa_end, int prot)
- {
- vm_offset_t sva = *virt;
- vm_offset_t va = sva;
- #ifdef __powerpc64__
- /* XXX: Handle memory not starting at 0x0. */
- if (pa_end < ctob(Maxmem))
- return (PHYS_TO_DMAP(pa_start));
- #endif
- while (pa_start < pa_end) {
- mmu_booke_kenter(va, pa_start);
- va += PAGE_SIZE;
- pa_start += PAGE_SIZE;
- }
- *virt = va;
- return (sva);
- }
- /*
- * The pmap must be activated before it's address space can be accessed in any
- * way.
- */
- static void
- mmu_booke_activate(struct thread *td)
- {
- pmap_t pmap;
- u_int cpuid;
- pmap = &td->td_proc->p_vmspace->vm_pmap;
- CTR5(KTR_PMAP, "%s: s (td = %p, proc = '%s', id = %d, pmap = 0x%"PRI0ptrX")",
- __func__, td, td->td_proc->p_comm, td->td_proc->p_pid, pmap);
- KASSERT((pmap != kernel_pmap), ("mmu_booke_activate: kernel_pmap!"));
- sched_pin();
- cpuid = PCPU_GET(cpuid);
- CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
- PCPU_SET(curpmap, pmap);
- if (pmap->pm_tid[cpuid] == TID_NONE)
- tid_alloc(pmap);
- /* Load PID0 register with pmap tid value. */
- mtspr(SPR_PID0, pmap->pm_tid[cpuid]);
- __asm __volatile("isync");
- mtspr(SPR_DBCR0, td->td_pcb->pcb_cpu.booke.dbcr0);
- sched_unpin();
- CTR3(KTR_PMAP, "%s: e (tid = %d for '%s')", __func__,
- pmap->pm_tid[PCPU_GET(cpuid)], td->td_proc->p_comm);
- }
- /*
- * Deactivate the specified process's address space.
- */
- static void
- mmu_booke_deactivate(struct thread *td)
- {
- pmap_t pmap;
- pmap = &td->td_proc->p_vmspace->vm_pmap;
- CTR5(KTR_PMAP, "%s: td=%p, proc = '%s', id = %d, pmap = 0x%"PRI0ptrX,
- __func__, td, td->td_proc->p_comm, td->td_proc->p_pid, pmap);
- td->td_pcb->pcb_cpu.booke.dbcr0 = mfspr(SPR_DBCR0);
- CPU_CLR_ATOMIC(PCPU_GET(cpuid), &pmap->pm_active);
- PCPU_SET(curpmap, NULL);
- }
- /*
- * Copy the range specified by src_addr/len
- * from the source map to the range dst_addr/len
- * in the destination map.
- *
- * This routine is only advisory and need not do anything.
- */
- static void
- mmu_booke_copy(pmap_t dst_pmap, pmap_t src_pmap,
- vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr)
- {
- }
- /*
- * Set the physical protection on the specified range of this map as requested.
- */
- static void
- mmu_booke_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
- vm_prot_t prot)
- {
- vm_offset_t va;
- vm_page_t m;
- pte_t *pte;
- if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
- mmu_booke_remove(pmap, sva, eva);
- return;
- }
- if (prot & VM_PROT_WRITE)
- return;
- PMAP_LOCK(pmap);
- for (va = sva; va < eva; va += PAGE_SIZE) {
- if ((pte = pte_find(pmap, va)) != NULL) {
- if (PTE_ISVALID(pte)) {
- m = PHYS_TO_VM_PAGE(PTE_PA(pte));
- mtx_lock_spin(&tlbivax_mutex);
- tlb_miss_lock();
- /* Handle modified pages. */
- if (PTE_ISMODIFIED(pte) && PTE_ISMANAGED(pte))
- vm_page_dirty(m);
- tlb0_flush_entry(va);
- *pte &= ~(PTE_UW | PTE_SW | PTE_MODIFIED);
- tlb_miss_unlock();
- mtx_unlock_spin(&tlbivax_mutex);
- }
- }
- }
- PMAP_UNLOCK(pmap);
- }
- /*
- * Clear the write and modified bits in each of the given page's mappings.
- */
- static void
- mmu_booke_remove_write(vm_page_t m)
- {
- pv_entry_t pv;
- pte_t *pte;
- KASSERT((m->oflags & VPO_UNMANAGED) == 0,
- ("mmu_booke_remove_write: page %p is not managed", m));
- vm_page_assert_busied(m);
- if (!pmap_page_is_write_mapped(m))
- return;
- rw_wlock(&pvh_global_lock);
- TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
- PMAP_LOCK(pv->pv_pmap);
- if ((pte = pte_find(pv->pv_pmap, pv->pv_va)) != NULL) {
- if (PTE_ISVALID(pte)) {
- m = PHYS_TO_VM_PAGE(PTE_PA(pte));
- mtx_lock_spin(&tlbivax_mutex);
- tlb_miss_lock();
- /* Handle modified pages. */
- if (PTE_ISMODIFIED(pte))
- vm_page_dirty(m);
- /* Flush mapping from TLB0. */
- *pte &= ~(PTE_UW | PTE_SW | PTE_MODIFIED);
- tlb_miss_unlock();
- mtx_unlock_spin(&tlbivax_mutex);
- }
- }
- PMAP_UNLOCK(pv->pv_pmap);
- }
- vm_page_aflag_clear(m, PGA_WRITEABLE);
- rw_wunlock(&pvh_global_lock);
- }
- /*
- * Atomically extract and hold the physical page with the given
- * pmap and virtual address pair if that mapping permits the given
- * protection.
- */
- static vm_page_t
- mmu_booke_extract_and_hold(pmap_t pmap, vm_offset_t va,
- vm_prot_t prot)
- {
- pte_t *pte;
- vm_page_t m;
- uint32_t pte_wbit;
- m = NULL;
- PMAP_LOCK(pmap);
- pte = pte_find(pmap, va);
- if ((pte != NULL) && PTE_ISVALID(pte)) {
- if (pmap == kernel_pmap)
- pte_wbit = PTE_SW;
- else
- pte_wbit = PTE_UW;
- if ((*pte & pte_wbit) != 0 || (prot & VM_PROT_WRITE) == 0) {
- m = PHYS_TO_VM_PAGE(PTE_PA(pte));
- if (!vm_page_wire_mapped(m))
- m = NULL;
- }
- }
- PMAP_UNLOCK(pmap);
- return (m);
- }
- /*
- * Initialize a vm_page's machine-dependent fields.
- */
- static void
- mmu_booke_page_init(vm_page_t m)
- {
- m->md.pv_tracked = 0;
- TAILQ_INIT(&m->md.pv_list);
- }
- /*
- * Return whether or not the specified physical page was modified
- * in any of physical maps.
- */
- static boolean_t
- mmu_booke_is_modified(vm_page_t m)
- {
- pte_t *pte;
- pv_entry_t pv;
- boolean_t rv;
- KASSERT((m->oflags & VPO_UNMANAGED) == 0,
- ("mmu_booke_is_modified: page %p is not managed", m));
- rv = FALSE;
- /*
- * If the page is not busied then this check is racy.
- */
- if (!pmap_page_is_write_mapped(m))
- return (FALSE);
- rw_wlock(&pvh_global_lock);
- TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
- PMAP_LOCK(pv->pv_pmap);
- if ((pte = pte_find(pv->pv_pmap, pv->pv_va)) != NULL &&
- PTE_ISVALID(pte)) {
- if (PTE_ISMODIFIED(pte))
- rv = TRUE;
- }
- PMAP_UNLOCK(pv->pv_pmap);
- if (rv)
- break;
- }
- rw_wunlock(&pvh_global_lock);
- return (rv);
- }
- /*
- * Return whether or not the specified virtual address is eligible
- * for prefault.
- */
- static boolean_t
- mmu_booke_is_prefaultable(pmap_t pmap, vm_offset_t addr)
- {
- return (FALSE);
- }
- /*
- * Return whether or not the specified physical page was referenced
- * in any physical maps.
- */
- static boolean_t
- mmu_booke_is_referenced(vm_page_t m)
- {
- pte_t *pte;
- pv_entry_t pv;
- boolean_t rv;
- KASSERT((m->oflags & VPO_UNMANAGED) == 0,
- ("mmu_booke_is_referenced: page %p is not managed", m));
- rv = FALSE;
- rw_wlock(&pvh_global_lock);
- TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
- PMAP_LOCK(pv->pv_pmap);
- if ((pte = pte_find(pv->pv_pmap, pv->pv_va)) != NULL &&
- PTE_ISVALID(pte)) {
- if (PTE_ISREFERENCED(pte))
- rv = TRUE;
- }
- PMAP_UNLOCK(pv->pv_pmap);
- if (rv)
- break;
- }
- rw_wunlock(&pvh_global_lock);
- return (rv);
- }
- /*
- * Clear the modify bits on the specified physical page.
- */
- static void
- mmu_booke_clear_modify(vm_page_t m)
- {
- pte_t *pte;
- pv_entry_t pv;
- KASSERT((m->oflags & VPO_UNMANAGED) == 0,
- ("mmu_booke_clear_modify: page %p is not managed", m));
- vm_page_assert_busied(m);
- if (!pmap_page_is_write_mapped(m))
- return;
- rw_wlock(&pvh_global_lock);
- TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
- PMAP_LOCK(pv->pv_pmap);
- if ((pte = pte_find(pv->pv_pmap, pv->pv_va)) != NULL &&
- PTE_ISVALID(pte)) {
- mtx_lock_spin(&tlbivax_mutex);
- tlb_miss_lock();
-
- if (*pte & (PTE_SW | PTE_UW | PTE_MODIFIED)) {
- tlb0_flush_entry(pv->pv_va);
- *pte &= ~(PTE_SW | PTE_UW | PTE_MODIFIED |
- PTE_REFERENCED);
- }
- tlb_miss_unlock();
- mtx_unlock_spin(&tlbivax_mutex);
- }
- PMAP_UNLOCK(pv->pv_pmap);
- }
- rw_wunlock(&pvh_global_lock);
- }
- /*
- * Return a count of reference bits for a page, clearing those bits.
- * It is not necessary for every reference bit to be cleared, but it
- * is necessary that 0 only be returned when there are truly no
- * reference bits set.
- *
- * As an optimization, update the page's dirty field if a modified bit is
- * found while counting reference bits. This opportunistic update can be
- * performed at low cost and can eliminate the need for some future calls
- * to pmap_is_modified(). However, since this function stops after
- * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some
- * dirty pages. Those dirty pages will only be detected by a future call
- * to pmap_is_modified().
- */
- static int
- mmu_booke_ts_referenced(vm_page_t m)
- {
- pte_t *pte;
- pv_entry_t pv;
- int count;
- KASSERT((m->oflags & VPO_UNMANAGED) == 0,
- ("mmu_booke_ts_referenced: page %p is not managed", m));
- count = 0;
- rw_wlock(&pvh_global_lock);
- TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
- PMAP_LOCK(pv->pv_pmap);
- if ((pte = pte_find(pv->pv_pmap, pv->pv_va)) != NULL &&
- PTE_ISVALID(pte)) {
- if (PTE_ISMODIFIED(pte))
- vm_page_dirty(m);
- if (PTE_ISREFERENCED(pte)) {
- mtx_lock_spin(&tlbivax_mutex);
- tlb_miss_lock();
- tlb0_flush_entry(pv->pv_va);
- *pte &= ~PTE_REFERENCED;
- tlb_miss_unlock();
- mtx_unlock_spin(&tlbivax_mutex);
- if (++count >= PMAP_TS_REFERENCED_MAX) {
- PMAP_UNLOCK(pv->pv_pmap);
- break;
- }
- }
- }
- PMAP_UNLOCK(pv->pv_pmap);
- }
- rw_wunlock(&pvh_global_lock);
- return (count);
- }
- /*
- * Clear the wired attribute from the mappings for the specified range of
- * addresses in the given pmap. Every valid mapping within that range must
- * have the wired attribute set. In contrast, invalid mappings cannot have
- * the wired attribute set, so they are ignored.
- *
- * The wired attribute of the page table entry is not a hardware feature, so
- * there is no need to invalidate any TLB entries.
- */
- static void
- mmu_booke_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
- {
- vm_offset_t va;
- pte_t *pte;
- PMAP_LOCK(pmap);
- for (va = sva; va < eva; va += PAGE_SIZE) {
- if ((pte = pte_find(pmap, va)) != NULL &&
- PTE_ISVALID(pte)) {
- if (!PTE_ISWIRED(pte))
- panic("mmu_booke_unwire: pte %p isn't wired",
- pte);
- *pte &= ~PTE_WIRED;
- pmap->pm_stats.wired_count--;
- }
- }
- PMAP_UNLOCK(pmap);
- }
- /*
- * Return true if the pmap's pv is one of the first 16 pvs linked to from this
- * page. This count may be changed upwards or downwards in the future; it is
- * only necessary that true be returned for a small subset of pmaps for proper
- * page aging.
- */
- static boolean_t
- mmu_booke_page_exists_quick(pmap_t pmap, vm_page_t m)
- {
- pv_entry_t pv;
- int loops;
- boolean_t rv;
- KASSERT((m->oflags & VPO_UNMANAGED) == 0,
- ("mmu_booke_page_exists_quick: page %p is not managed", m));
- loops = 0;
- rv = FALSE;
- rw_wlock(&pvh_global_lock);
- TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
- if (pv->pv_pmap == pmap) {
- rv = TRUE;
- break;
- }
- if (++loops >= 16)
- break;
- }
- rw_wunlock(&pvh_global_lock);
- return (rv);
- }
- /*
- * Return the number of managed mappings to the given physical page that are
- * wired.
- */
- static int
- mmu_booke_page_wired_mappings(vm_page_t m)
- {
- pv_entry_t pv;
- pte_t *pte;
- int count = 0;
- if ((m->oflags & VPO_UNMANAGED) != 0)
- return (count);
- rw_wlock(&pvh_global_lock);
- TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
- PMAP_LOCK(pv->pv_pmap);
- if ((pte = pte_find(pv->pv_pmap, pv->pv_va)) != NULL)
- if (PTE_ISVALID(pte) && PTE_ISWIRED(pte))
- count++;
- PMAP_UNLOCK(pv->pv_pmap);
- }
- rw_wunlock(&pvh_global_lock);
- return (count);
- }
- static int
- mmu_booke_dev_direct_mapped(vm_paddr_t pa, vm_size_t size)
- {
- int i;
- vm_offset_t va;
- /*
- * This currently does not work for entries that
- * overlap TLB1 entries.
- */
- for (i = 0; i < TLB1_ENTRIES; i ++) {
- if (tlb1_iomapped(i, pa, size, &va) == 0)
- return (0);
- }
- return (EFAULT);
- }
- void
- mmu_booke_dumpsys_map(vm_paddr_t pa, size_t sz, void **va)
- {
- vm_paddr_t ppa;
- vm_offset_t ofs;
- vm_size_t gran;
- /* Minidumps are based on virtual memory addresses. */
- if (do_minidump) {
- *va = (void *)(vm_offset_t)pa;
- return;
- }
- /* Raw physical memory dumps don't have a virtual address. */
- /* We always map a 256MB page at 256M. */
- gran = 256 * 1024 * 1024;
- ppa = rounddown2(pa, gran);
- ofs = pa - ppa;
- *va = (void *)gran;
- tlb1_set_entry((vm_offset_t)va, ppa, gran, _TLB_ENTRY_IO);
- if (sz > (gran - ofs))
- tlb1_set_entry((vm_offset_t)(va + gran), ppa + gran, gran,
- _TLB_ENTRY_IO);
- }
- void
- mmu_booke_dumpsys_unmap(vm_paddr_t pa, size_t sz, void *va)
- {
- vm_paddr_t ppa;
- vm_offset_t ofs;
- vm_size_t gran;
- tlb_entry_t e;
- int i;
- /* Minidumps are based on virtual memory addresses. */
- /* Nothing to do... */
- if (do_minidump)
- return;
- for (i = 0; i < TLB1_ENTRIES; i++) {
- tlb1_read_entry(&e, i);
- if (!(e.mas1 & MAS1_VALID))
- break;
- }
- /* Raw physical memory dumps don't have a virtual address. */
- i--;
- e.mas1 = 0;
- e.mas2 = 0;
- e.mas3 = 0;
- tlb1_write_entry(&e, i);
- gran = 256 * 1024 * 1024;
- ppa = rounddown2(pa, gran);
- ofs = pa - ppa;
- if (sz > (gran - ofs)) {
- i--;
- e.mas1 = 0;
- e.mas2 = 0;
- e.mas3 = 0;
- tlb1_write_entry(&e, i);
- }
- }
- extern struct dump_pa dump_map[PHYS_AVAIL_SZ + 1];
- void
- mmu_booke_scan_init()
- {
- vm_offset_t va;
- pte_t *pte;
- int i;
- if (!do_minidump) {
- /* Initialize phys. segments for dumpsys(). */
- memset(&dump_map, 0, sizeof(dump_map));
- mem_regions(&physmem_regions, &physmem_regions_sz, &availmem_regions,
- &availmem_regions_sz);
- for (i = 0; i < physmem_regions_sz; i++) {
- dump_map[i].pa_start = physmem_regions[i].mr_start;
- dump_map[i].pa_size = physmem_regions[i].mr_size;
- }
- return;
- }
- /* Virtual segments for minidumps: */
- memset(&dump_map, 0, sizeof(dump_map));
- /* 1st: kernel .data and .bss. */
- dump_map[0].pa_start = trunc_page((uintptr_t)_etext);
- dump_map[0].pa_size =
- round_page((uintptr_t)_end) - dump_map[0].pa_start;
- /* 2nd: msgbuf and tables (see pmap_bootstrap()). */
- dump_map[1].pa_start = data_start;
- dump_map[1].pa_size = data_end - data_start;
- /* 3rd: kernel VM. */
- va = dump_map[1].pa_start + dump_map[1].pa_size;
- /* Find start of next chunk (from va). */
- while (va < virtual_end) {
- /* Don't dump the buffer cache. */
- if (va >= kmi.buffer_sva && va < kmi.buffer_eva) {
- va = kmi.buffer_eva;
- continue;
- }
- pte = pte_find(kernel_pmap, va);
- if (pte != NULL && PTE_ISVALID(pte))
- break;
- va += PAGE_SIZE;
- }
- if (va < virtual_end) {
- dump_map[2].pa_start = va;
- va += PAGE_SIZE;
- /* Find last page in chunk. */
- while (va < virtual_end) {
- /* Don't run into the buffer cache. */
- if (va == kmi.buffer_sva)
- break;
- pte = pte_find(kernel_pmap, va);
- if (pte == NULL || !PTE_ISVALID(pte))
- break;
- va += PAGE_SIZE;
- }
- dump_map[2].pa_size = va - dump_map[2].pa_start;
- }
- }
- /*
- * Map a set of physical memory pages into the kernel virtual address space.
- * Return a pointer to where it is mapped. This routine is intended to be used
- * for mapping device memory, NOT real memory.
- */
- static void *
- mmu_booke_mapdev(vm_paddr_t pa, vm_size_t size)
- {
- return (mmu_booke_mapdev_attr(pa, size, VM_MEMATTR_DEFAULT));
- }
- static int
- tlb1_find_pa(vm_paddr_t pa, tlb_entry_t *e)
- {
- int i;
- for (i = 0; i < TLB1_ENTRIES; i++) {
- tlb1_read_entry(e, i);
- if ((e->mas1 & MAS1_VALID) == 0)
- continue;
- if (e->phys == pa)
- return (i);
- }
- return (-1);
- }
- static void *
- mmu_booke_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t ma)
- {
- tlb_entry_t e;
- vm_paddr_t tmppa;
- #ifndef __powerpc64__
- uintptr_t tmpva;
- #endif
- uintptr_t va, retva;
- vm_size_t sz;
- int i;
- int wimge;
- /*
- * Check if this is premapped in TLB1.
- */
- sz = size;
- tmppa = pa;
- va = ~0;
- wimge = tlb_calc_wimg(pa, ma);
- for (i = 0; i < TLB1_ENTRIES; i++) {
- tlb1_read_entry(&e, i);
- if (!(e.mas1 & MAS1_VALID))
- continue;
- if (wimge != (e.mas2 & (MAS2_WIMGE_MASK & ~_TLB_ENTRY_SHARED)))
- continue;
- if (tmppa >= e.phys && tmppa < e.phys + e.size) {
- va = e.virt + (pa - e.phys);
- tmppa = e.phys + e.size;
- sz -= MIN(sz, e.size - (pa - e.phys));
- while (sz > 0 && (i = tlb1_find_pa(tmppa, &e)) != -1) {
- if (wimge != (e.mas2 & (MAS2_WIMGE_MASK & ~_TLB_ENTRY_SHARED)))
- break;
- sz -= MIN(sz, e.size);
- tmppa = e.phys + e.size;
- }
- if (sz != 0)
- break;
- return ((void *)va);
- }
- }
- size = roundup(size, PAGE_SIZE);
- #ifdef __powerpc64__
- KASSERT(pa < VM_MAPDEV_PA_MAX,
- ("Unsupported physical address! %lx", pa));
- va = VM_MAPDEV_BASE + pa;
- retva = va;
- #ifdef POW2_MAPPINGS
- /*
- * Align the mapping to a power of 2 size, taking into account that we
- * may need to increase the size multiple times to satisfy the size and
- * alignment requirements.
- *
- * This works in the general case because it's very rare (near never?)
- * to have different access properties (WIMG) within a single
- * power-of-two region. If a design does call for that, POW2_MAPPINGS
- * can be undefined, and exact mappings will be used instead.
- */
- sz = size;
- size = roundup2(size, 1 << ilog2(size));
- while (rounddown2(va, size) + size < va + sz)
- size <<= 1;
- va = rounddown2(va, size);
- pa = rounddown2(pa, size);
- #endif
- #else
- /*
- * The device mapping area is between VM_MAXUSER_ADDRESS and
- * VM_MIN_KERNEL_ADDRESS. This gives 1GB of device addressing.
- */
- #ifdef SPARSE_MAPDEV
- /*
- * With a sparse mapdev, align to the largest starting region. This
- * could feasibly be optimized for a 'best-fit' alignment, but that
- * calculation could be very costly.
- * Align to the smaller of:
- * - first set bit in overlap of (pa & size mask)
- * - largest size envelope
- *
- * It's possible the device mapping may start at a PA that's not larger
- * than the size mask, so we need to offset in to maximize the TLB entry
- * range and minimize the number of used TLB entries.
- */
- do {
- tmpva = tlb1_map_base;
- sz = ffsl((~((1 << flsl(size-1)) - 1)) & pa);
- sz = sz ? min(roundup(sz + 3, 4), flsl(size) - 1) : flsl(size) - 1;
- va = roundup(tlb1_map_base, 1 << sz) | (((1 << sz) - 1) & pa);
- } while (!atomic_cmpset_int(&tlb1_map_base, tmpva, va + size));
- #endif
- va = atomic_fetchadd_int(&tlb1_map_base, size);
- retva = va;
- #endif
- if (tlb1_mapin_region(va, pa, size, tlb_calc_wimg(pa, ma)) != size)
- return (NULL);
- return ((void *)retva);
- }
- /*
- * 'Unmap' a range mapped by mmu_booke_mapdev().
- */
- static void
- mmu_booke_unmapdev(vm_offset_t va, vm_size_t size)
- {
- #ifdef SUPPORTS_SHRINKING_TLB1
- vm_offset_t base, offset;
- /*
- * Unmap only if this is inside kernel virtual space.
- */
- if ((va >= VM_MIN_KERNEL_ADDRESS) && (va <= VM_MAX_KERNEL_ADDRESS)) {
- base = trunc_page(va);
- offset = va & PAGE_MASK;
- size = roundup(offset + size, PAGE_SIZE);
- mmu_booke_qremove(base, atop(size));
- kva_free(base, size);
- }
- #endif
- }
- /*
- * mmu_booke_object_init_pt preloads the ptes for a given object into the
- * specified pmap. This eliminates the blast of soft faults on process startup
- * and immediately after an mmap.
- */
- static void
- mmu_booke_object_init_pt(pmap_t pmap, vm_offset_t addr,
- vm_object_t object, vm_pindex_t pindex, vm_size_t size)
- {
- VM_OBJECT_ASSERT_WLOCKED(object);
- KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
- ("mmu_booke_object_init_pt: non-device object"));
- }
- /*
- * Perform the pmap work for mincore.
- */
- static int
- mmu_booke_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)
- {
- /* XXX: this should be implemented at some point */
- return (0);
- }
- static int
- mmu_booke_change_attr(vm_offset_t addr, vm_size_t sz, vm_memattr_t mode)
- {
- vm_offset_t va;
- pte_t *pte;
- int i, j;
- tlb_entry_t e;
- addr = trunc_page(addr);
- /* Only allow changes to mapped kernel addresses. This includes:
- * - KVA
- * - DMAP (powerpc64)
- * - Device mappings
- */
- if (addr <= VM_MAXUSER_ADDRESS ||
- #ifdef __powerpc64__
- (addr >= tlb1_map_base && addr < DMAP_BASE_ADDRESS) ||
- (addr > DMAP_MAX_ADDRESS && addr < VM_MIN_KERNEL_ADDRESS) ||
- #else
- (addr >= tlb1_map_base && addr < VM_MIN_KERNEL_ADDRESS) ||
- #endif
- (addr > VM_MAX_KERNEL_ADDRESS))
- return (EINVAL);
- /* Check TLB1 mappings */
- for (i = 0; i < TLB1_ENTRIES; i++) {
- tlb1_read_entry(&e, i);
- if (!(e.mas1 & MAS1_VALID))
- continue;
- if (addr >= e.virt && addr < e.virt + e.size)
- break;
- }
- if (i < TLB1_ENTRIES) {
- /* Only allow full mappings to be modified for now. */
- /* Validate the range. */
- for (j = i, va = addr; va < addr + sz; va += e.size, j++) {
- tlb1_read_entry(&e, j);
- if (va != e.virt || (sz - (va - addr) < e.size))
- return (EINVAL);
- }
- for (va = addr; va < addr + sz; va += e.size, i++) {
- tlb1_read_entry(&e, i);
- e.mas2 &= ~MAS2_WIMGE_MASK;
- e.mas2 |= tlb_calc_wimg(e.phys, mode);
- /*
- * Write it out to the TLB. Should really re-sync with other
- * cores.
- */
- tlb1_write_entry(&e, i);
- }
- return (0);
- }
- /* Not in TLB1, try through pmap */
- /* First validate the range. */
- for (va = addr; va < addr + sz; va += PAGE_SIZE) {
- pte = pte_find(kernel_pmap, va);
- if (pte == NULL || !PTE_ISVALID(pte))
- return (EINVAL);
- }
- mtx_lock_spin(&tlbivax_mutex);
- tlb_miss_lock();
- for (va = addr; va < addr + sz; va += PAGE_SIZE) {
- pte = pte_find(kernel_pmap, va);
- *pte &= ~(PTE_MAS2_MASK << PTE_MAS2_SHIFT);
- *pte |= tlb_calc_wimg(PTE_PA(pte), mode) << PTE_MAS2_SHIFT;
- tlb0_flush_entry(va);
- }
- tlb_miss_unlock();
- mtx_unlock_spin(&tlbivax_mutex);
- return (0);
- }
- static void
- mmu_booke_page_array_startup(long pages)
- {
- vm_page_array_size = pages;
- }
- /**************************************************************************/
- /* TID handling */
- /**************************************************************************/
- /*
- * Allocate a TID. If necessary, steal one from someone else.
- * The new TID is flushed from the TLB before returning.
- */
- static tlbtid_t
- tid_alloc(pmap_t pmap)
- {
- tlbtid_t tid;
- int thiscpu;
- KASSERT((pmap != kernel_pmap), ("tid_alloc: kernel pmap"));
- CTR2(KTR_PMAP, "%s: s (pmap = %p)", __func__, pmap);
- thiscpu = PCPU_GET(cpuid);
- tid = PCPU_GET(booke.tid_next);
- if (tid > TID_MAX)
- tid = TID_MIN;
- PCPU_SET(booke.tid_next, tid + 1);
- /* If we are stealing TID then clear the relevant pmap's field */
- if (tidbusy[thiscpu][tid] != NULL) {
- CTR2(KTR_PMAP, "%s: warning: stealing tid %d", __func__, tid);
-
- tidbusy[thiscpu][tid]->pm_tid[thiscpu] = TID_NONE;
- /* Flush all entries from TLB0 matching this TID. */
- tid_flush(tid);
- }
- tidbusy[thiscpu][tid] = pmap;
- pmap->pm_tid[thiscpu] = tid;
- __asm __volatile("msync; isync");
- CTR3(KTR_PMAP, "%s: e (%02d next = %02d)", __func__, tid,
- PCPU_GET(booke.tid_next));
- return (tid);
- }
- /**************************************************************************/
- /* TLB0 handling */
- /**************************************************************************/
- /* Convert TLB0 va and way number to tlb0[] table index. */
- static inline unsigned int
- tlb0_tableidx(vm_offset_t va, unsigned int way)
- {
- unsigned int idx;
- idx = (way * TLB0_ENTRIES_PER_WAY);
- idx += (va & MAS2_TLB0_ENTRY_IDX_MASK) >> MAS2_TLB0_ENTRY_IDX_SHIFT;
- return (idx);
- }
- /*
- * Invalidate TLB0 entry.
- */
- static inline void
- tlb0_flush_entry(vm_offset_t va)
- {
- CTR2(KTR_PMAP, "%s: s va=0x%08x", __func__, va);
- mtx_assert(&tlbivax_mutex, MA_OWNED);
- __asm __volatile("tlbivax 0, %0" :: "r"(va & MAS2_EPN_MASK));
- __asm __volatile("isync; msync");
- __asm __volatile("tlbsync; msync");
- CTR1(KTR_PMAP, "%s: e", __func__);
- }
- /**************************************************************************/
- /* TLB1 handling */
- /**************************************************************************/
- /*
- * TLB1 mapping notes:
- *
- * TLB1[0] Kernel text and data.
- * TLB1[1-15] Additional kernel text and data mappings (if required), PCI
- * windows, other devices mappings.
- */
- /*
- * Read an entry from given TLB1 slot.
- */
- void
- tlb1_read_entry(tlb_entry_t *entry, unsigned int slot)
- {
- register_t msr;
- uint32_t mas0;
- KASSERT((entry != NULL), ("%s(): Entry is NULL!", __func__));
- msr = mfmsr();
- __asm __volatile("wrteei 0");
- mas0 = MAS0_TLBSEL(1) | MAS0_ESEL(slot);
- mtspr(SPR_MAS0, mas0);
- __asm __volatile("isync; tlbre");
- entry->mas1 = mfspr(SPR_MAS1);
- entry->mas2 = mfspr(SPR_MAS2);
- entry->mas3 = mfspr(SPR_MAS3);
- switch ((mfpvr() >> 16) & 0xFFFF) {
- case FSL_E500v2:
- case FSL_E500mc:
- case FSL_E5500:
- case FSL_E6500:
- entry->mas7 = mfspr(SPR_MAS7);
- break;
- default:
- entry->mas7 = 0;
- break;
- }
- __asm __volatile("wrtee %0" :: "r"(msr));
- entry->virt = entry->mas2 & MAS2_EPN_MASK;
- entry->phys = ((vm_paddr_t)(entry->mas7 & MAS7_RPN) << 32) |
- (entry->mas3 & MAS3_RPN);
- entry->size =
- tsize2size((entry->mas1 & MAS1_TSIZE_MASK) >> MAS1_TSIZE_SHIFT);
- }
- struct tlbwrite_args {
- tlb_entry_t *e;
- unsigned int idx;
- };
- static uint32_t
- tlb1_find_free(void)
- {
- tlb_entry_t e;
- int i;
- for (i = 0; i < TLB1_ENTRIES; i++) {
- tlb1_read_entry(&e, i);
- if ((e.mas1 & MAS1_VALID) == 0)
- return (i);
- }
- return (-1);
- }
- static void
- tlb1_purge_va_range(vm_offset_t va, vm_size_t size)
- {
- tlb_entry_t e;
- int i;
- for (i = 0; i < TLB1_ENTRIES; i++) {
- tlb1_read_entry(&e, i);
- if ((e.mas1 & MAS1_VALID) == 0)
- continue;
- if ((e.mas2 & MAS2_EPN_MASK) >= va &&
- (e.mas2 & MAS2_EPN_MASK) < va + size) {
- mtspr(SPR_MAS1, e.mas1 & ~MAS1_VALID);
- __asm __volatile("isync; tlbwe; isync; msync");
- }
- }
- }
- static void
- tlb1_write_entry_int(void *arg)
- {
- struct tlbwrite_args *args = arg;
- uint32_t idx, mas0;
- idx = args->idx;
- if (idx == -1) {
- tlb1_purge_va_range(args->e->virt, args->e->size);
- idx = tlb1_find_free();
- if (idx == -1)
- panic("No free TLB1 entries!\n");
- }
- /* Select entry */
- mas0 = MAS0_TLBSEL(1) | MAS0_ESEL(idx);
- mtspr(SPR_MAS0, mas0);
- mtspr(SPR_MAS1, args->e->mas1);
- mtspr(SPR_MAS2, args->e->mas2);
- mtspr(SPR_MAS3, args->e->mas3);
- switch ((mfpvr() >> 16) & 0xFFFF) {
- case FSL_E500mc:
- case FSL_E5500:
- case FSL_E6500:
- mtspr(SPR_MAS8, 0);
- /* FALLTHROUGH */
- case FSL_E500v2:
- mtspr(SPR_MAS7, args->e->mas7);
- break;
- default:
- break;
- }
- __asm __volatile("isync; tlbwe; isync; msync");
- }
- static void
- tlb1_write_entry_sync(void *arg)
- {
- /* Empty synchronization point for smp_rendezvous(). */
- }
- /*
- * Write given entry to TLB1 hardware.
- */
- static void
- tlb1_write_entry(tlb_entry_t *e, unsigned int idx)
- {
- struct tlbwrite_args args;
- args.e = e;
- args.idx = idx;
- #ifdef SMP
- if ((e->mas2 & _TLB_ENTRY_SHARED) && smp_started) {
- mb();
- smp_rendezvous(tlb1_write_entry_sync,
- tlb1_write_entry_int,
- tlb1_write_entry_sync, &args);
- } else
- #endif
- {
- register_t msr;
- msr = mfmsr();
- __asm __volatile("wrteei 0");
- tlb1_write_entry_int(&args);
- __asm __volatile("wrtee %0" :: "r"(msr));
- }
- }
- /*
- * Convert TLB TSIZE value to mapped region size.
- */
- static vm_size_t
- tsize2size(unsigned int tsize)
- {
- /*
- * size = 4^tsize KB
- * size = 4^tsize * 2^10 = 2^(2 * tsize - 10)
- */
- return ((1 << (2 * tsize)) * 1024);
- }
- /*
- * Convert region size (must be power of 4) to TLB TSIZE value.
- */
- static unsigned int
- size2tsize(vm_size_t size)
- {
- return (ilog2(size) / 2 - 5);
- }
- /*
- * Register permanent kernel mapping in TLB1.
- *
- * Entries are created starting from index 0 (current free entry is
- * kept in tlb1_idx) and are not supposed to be invalidated.
- */
- int
- tlb1_set_entry(vm_offset_t va, vm_paddr_t pa, vm_size_t size,
- uint32_t flags)
- {
- tlb_entry_t e;
- uint32_t ts, tid;
- int tsize, index;
- /* First try to update an existing entry. */
- for (index = 0; index < TLB1_ENTRIES; index++) {
- tlb1_read_entry(&e, index);
- /* Check if we're just updating the flags, and update them. */
- if (e.phys == pa && e.virt == va && e.size == size) {
- e.mas2 = (va & MAS2_EPN_MASK) | flags;
- tlb1_write_entry(&e, index);
- return (0);
- }
- }
- /* Convert size to TSIZE */
- tsize = size2tsize(size);
- tid = (TID_KERNEL << MAS1_TID_SHIFT) & MAS1_TID_MASK;
- /* XXX TS is hard coded to 0 for now as we only use single address space */
- ts = (0 << MAS1_TS_SHIFT) & MAS1_TS_MASK;
- e.phys = pa;
- e.virt = va;
- e.size = size;
- e.mas1 = MAS1_VALID | MAS1_IPROT | ts | tid;
- e.mas1 |= ((tsize << MAS1_TSIZE_SHIFT) & MAS1_TSIZE_MASK);
- e.mas2 = (va & MAS2_EPN_MASK) | flags;
- /* Set supervisor RWX permission bits */
- e.mas3 = (pa & MAS3_RPN) | MAS3_SR | MAS3_SW | MAS3_SX;
- e.mas7 = (pa >> 32) & MAS7_RPN;
- tlb1_write_entry(&e, -1);
- return (0);
- }
- /*
- * Map in contiguous RAM region into the TLB1.
- */
- static vm_size_t
- tlb1_mapin_region(vm_offset_t va, vm_paddr_t pa, vm_size_t size, int wimge)
- {
- vm_offset_t base;
- vm_size_t mapped, sz, ssize;
- mapped = 0;
- base = va;
- ssize = size;
- while (size > 0) {
- sz = 1UL << (ilog2(size) & ~1);
- /* Align size to PA */
- if (pa % sz != 0) {
- do {
- sz >>= 2;
- } while (pa % sz != 0);
- }
- /* Now align from there to VA */
- if (va % sz != 0) {
- do {
- sz >>= 2;
- } while (va % sz != 0);
- }
- #ifdef __powerpc64__
- /*
- * Clamp TLB1 entries to 4G.
- *
- * While the e6500 supports up to 1TB mappings, the e5500
- * only supports up to 4G mappings. (0b1011)
- *
- * If any e6500 machines capable of supporting a very
- * large amount of memory appear in the future, we can
- * revisit this.
- *
- * For now, though, since we have plenty of space in TLB1,
- * always avoid creating entries larger than 4GB.
- */
- sz = MIN(sz, 1UL << 32);
- #endif
- if (bootverbose)
- printf("Wiring VA=%p to PA=%jx (size=%lx)\n",
- (void *)va, (uintmax_t)pa, (long)sz);
- if (tlb1_set_entry(va, pa, sz,
- _TLB_ENTRY_SHARED | wimge) < 0)
- return (mapped);
- size -= sz;
- pa += sz;
- va += sz;
- }
- mapped = (va - base);
- if (bootverbose)
- printf("mapped size 0x%"PRIxPTR" (wasted space 0x%"PRIxPTR")\n",
- mapped, mapped - ssize);
- return (mapped);
- }
- /*
- * TLB1 initialization routine, to be called after the very first
- * assembler level setup done in locore.S.
- */
- void
- tlb1_init()
- {
- vm_offset_t mas2;
- uint32_t mas0, mas1, mas3, mas7;
- uint32_t tsz;
- tlb1_get_tlbconf();
- mas0 = MAS0_TLBSEL(1) | MAS0_ESEL(0);
- mtspr(SPR_MAS0, mas0);
- __asm __volatile("isync; tlbre");
- mas1 = mfspr(SPR_MAS1);
- mas2 = mfspr(SPR_MAS2);
- mas3 = mfspr(SPR_MAS3);
- mas7 = mfspr(SPR_MAS7);
- kernload = ((vm_paddr_t)(mas7 & MAS7_RPN) << 32) |
- (mas3 & MAS3_RPN);
- tsz = (mas1 & MAS1_TSIZE_MASK) >> MAS1_TSIZE_SHIFT;
- kernsize += (tsz > 0) ? tsize2size(tsz) : 0;
- kernstart = trunc_page(mas2);
- /* Setup TLB miss defaults */
- set_mas4_defaults();
- }
- /*
- * pmap_early_io_unmap() should be used in short conjunction with
- * pmap_early_io_map(), as in the following snippet:
- *
- * x = pmap_early_io_map(...);
- * <do something with x>
- * pmap_early_io_unmap(x, size);
- *
- * And avoiding more allocations between.
- */
- void
- pmap_early_io_unmap(vm_offset_t va, vm_size_t size)
- {
- int i;
- tlb_entry_t e;
- vm_size_t isize;
- size = roundup(size, PAGE_SIZE);
- isize = size;
- for (i = 0; i < TLB1_ENTRIES && size > 0; i++) {
- tlb1_read_entry(&e, i);
- if (!(e.mas1 & MAS1_VALID))
- continue;
- if (va <= e.virt && (va + isize) >= (e.virt + e.size)) {
- size -= e.size;
- e.mas1 &= ~MAS1_VALID;
- tlb1_write_entry(&e, i);
- }
- }
- if (tlb1_map_base == va + isize)
- tlb1_map_base -= isize;
- }
-
- vm_offset_t
- pmap_early_io_map(vm_paddr_t pa, vm_size_t size)
- {
- vm_paddr_t pa_base;
- vm_offset_t va, sz;
- int i;
- tlb_entry_t e;
- KASSERT(!pmap_bootstrapped, ("Do not use after PMAP is up!"));
- for (i = 0; i < TLB1_ENTRIES; i++) {
- tlb1_read_entry(&e, i);
- if (!(e.mas1 & MAS1_VALID))
- continue;
- if (pa >= e.phys && (pa + size) <=
- (e.phys + e.size))
- return (e.virt + (pa - e.phys));
- }
- pa_base = rounddown(pa, PAGE_SIZE);
- size = roundup(size + (pa - pa_base), PAGE_SIZE);
- tlb1_map_base = roundup2(tlb1_map_base, 1 << (ilog2(size) & ~1));
- va = tlb1_map_base + (pa - pa_base);
- do {
- sz = 1 << (ilog2(size) & ~1);
- tlb1_set_entry(tlb1_map_base, pa_base, sz,
- _TLB_ENTRY_SHARED | _TLB_ENTRY_IO);
- size -= sz;
- pa_base += sz;
- tlb1_map_base += sz;
- } while (size > 0);
- return (va);
- }
- void
- pmap_track_page(pmap_t pmap, vm_offset_t va)
- {
- vm_paddr_t pa;
- vm_page_t page;
- struct pv_entry *pve;
- va = trunc_page(va);
- pa = pmap_kextract(va);
- page = PHYS_TO_VM_PAGE(pa);
- rw_wlock(&pvh_global_lock);
- PMAP_LOCK(pmap);
- TAILQ_FOREACH(pve, &page->md.pv_list, pv_link) {
- if ((pmap == pve->pv_pmap) && (va == pve->pv_va)) {
- goto out;
- }
- }
- page->md.pv_tracked = true;
- pv_insert(pmap, va, page);
- out:
- PMAP_UNLOCK(pmap);
- rw_wunlock(&pvh_global_lock);
- }
- /*
- * Setup MAS4 defaults.
- * These values are loaded to MAS0-2 on a TLB miss.
- */
- static void
- set_mas4_defaults(void)
- {
- uint32_t mas4;
- /* Defaults: TLB0, PID0, TSIZED=4K */
- mas4 = MAS4_TLBSELD0;
- mas4 |= (TLB_SIZE_4K << MAS4_TSIZED_SHIFT) & MAS4_TSIZED_MASK;
- #ifdef SMP
- mas4 |= MAS4_MD;
- #endif
- mtspr(SPR_MAS4, mas4);
- __asm __volatile("isync");
- }
- /*
- * Return 0 if the physical IO range is encompassed by one of the
- * the TLB1 entries, otherwise return related error code.
- */
- static int
- tlb1_iomapped(int i, vm_paddr_t pa, vm_size_t size, vm_offset_t *va)
- {
- uint32_t prot;
- vm_paddr_t pa_start;
- vm_paddr_t pa_end;
- unsigned int entry_tsize;
- vm_size_t entry_size;
- tlb_entry_t e;
- *va = (vm_offset_t)NULL;
- tlb1_read_entry(&e, i);
- /* Skip invalid entries */
- if (!(e.mas1 & MAS1_VALID))
- return (EINVAL);
- /*
- * The entry must be cache-inhibited, guarded, and r/w
- * so it can function as an i/o page
- */
- prot = e.mas2 & (MAS2_I | MAS2_G);
- if (prot != (MAS2_I | MAS2_G))
- return (EPERM);
- prot = e.mas3 & (MAS3_SR | MAS3_SW);
- if (prot != (MAS3_SR | MAS3_SW))
- return (EPERM);
- /* The address should be within the entry range. */
- entry_tsize = (e.mas1 & MAS1_TSIZE_MASK) >> MAS1_TSIZE_SHIFT;
- KASSERT((entry_tsize), ("tlb1_iomapped: invalid entry tsize"));
- entry_size = tsize2size(entry_tsize);
- pa_start = (((vm_paddr_t)e.mas7 & MAS7_RPN) << 32) |
- (e.mas3 & MAS3_RPN);
- pa_end = pa_start + entry_size;
- if ((pa < pa_start) || ((pa + size) > pa_end))
- return (ERANGE);
- /* Return virtual address of this mapping. */
- *va = (e.mas2 & MAS2_EPN_MASK) + (pa - pa_start);
- return (0);
- }
- #ifdef DDB
- /* Print out contents of the MAS registers for each TLB0 entry */
- static void
- #ifdef __powerpc64__
- tlb_print_entry(int i, uint32_t mas1, uint64_t mas2, uint32_t mas3,
- #else
- tlb_print_entry(int i, uint32_t mas1, uint32_t mas2, uint32_t mas3,
- #endif
- uint32_t mas7)
- {
- int as;
- char desc[3];
- tlbtid_t tid;
- vm_size_t size;
- unsigned int tsize;
- desc[2] = '\0';
- if (mas1 & MAS1_VALID)
- desc[0] = 'V';
- else
- desc[0] = ' ';
- if (mas1 & MAS1_IPROT)
- desc[1] = 'P';
- else
- desc[1] = ' ';
- as = (mas1 & MAS1_TS_MASK) ? 1 : 0;
- tid = MAS1_GETTID(mas1);
- tsize = (mas1 & MAS1_TSIZE_MASK) >> MAS1_TSIZE_SHIFT;
- size = 0;
- if (tsize)
- size = tsize2size(tsize);
- printf("%3d: (%s) [AS=%d] "
- "sz = 0x%jx tsz = %d tid = %d mas1 = 0x%08x "
- "mas2(va) = 0x%"PRI0ptrX" mas3(pa) = 0x%08x mas7 = 0x%08x\n",
- i, desc, as, (uintmax_t)size, tsize, tid, mas1, mas2, mas3, mas7);
- }
- DB_SHOW_COMMAND(tlb0, tlb0_print_tlbentries)
- {
- uint32_t mas0, mas1, mas3, mas7;
- #ifdef __powerpc64__
- uint64_t mas2;
- #else
- uint32_t mas2;
- #endif
- int entryidx, way, idx;
- printf("TLB0 entries:\n");
- for (way = 0; way < TLB0_WAYS; way ++)
- for (entryidx = 0; entryidx < TLB0_ENTRIES_PER_WAY; entryidx++) {
- mas0 = MAS0_TLBSEL(0) | MAS0_ESEL(way);
- mtspr(SPR_MAS0, mas0);
- mas2 = entryidx << MAS2_TLB0_ENTRY_IDX_SHIFT;
- mtspr(SPR_MAS2, mas2);
- __asm __volatile("isync; tlbre");
- mas1 = mfspr(SPR_MAS1);
- mas2 = mfspr(SPR_MAS2);
- mas3 = mfspr(SPR_MAS3);
- mas7 = mfspr(SPR_MAS7);
- idx = tlb0_tableidx(mas2, way);
- tlb_print_entry(idx, mas1, mas2, mas3, mas7);
- }
- }
- /*
- * Print out contents of the MAS registers for each TLB1 entry
- */
- DB_SHOW_COMMAND(tlb1, tlb1_print_tlbentries)
- {
- uint32_t mas0, mas1, mas3, mas7;
- #ifdef __powerpc64__
- uint64_t mas2;
- #else
- uint32_t mas2;
- #endif
- int i;
- printf("TLB1 entries:\n");
- for (i = 0; i < TLB1_ENTRIES; i++) {
- mas0 = MAS0_TLBSEL(1) | MAS0_ESEL(i);
- mtspr(SPR_MAS0, mas0);
- __asm __volatile("isync; tlbre");
- mas1 = mfspr(SPR_MAS1);
- mas2 = mfspr(SPR_MAS2);
- mas3 = mfspr(SPR_MAS3);
- mas7 = mfspr(SPR_MAS7);
- tlb_print_entry(i, mas1, mas2, mas3, mas7);
- }
- }
- #endif
|