kvm_main.c 81 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * This module enables machines with Intel VT-x extensions to run virtual
  5. * machines without emulation or binary translation.
  6. *
  7. * Copyright (C) 2006 Qumranet, Inc.
  8. * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  9. *
  10. * Authors:
  11. * Avi Kivity <avi@qumranet.com>
  12. * Yaniv Kamay <yaniv@qumranet.com>
  13. *
  14. * This work is licensed under the terms of the GNU GPL, version 2. See
  15. * the COPYING file in the top-level directory.
  16. *
  17. */
  18. #include <kvm/iodev.h>
  19. #include <linux/kvm_host.h>
  20. #include <linux/kvm.h>
  21. #include <linux/module.h>
  22. #include <linux/errno.h>
  23. #include <linux/percpu.h>
  24. #include <linux/mm.h>
  25. #include <linux/miscdevice.h>
  26. #include <linux/vmalloc.h>
  27. #include <linux/reboot.h>
  28. #include <linux/debugfs.h>
  29. #include <linux/highmem.h>
  30. #include <linux/file.h>
  31. #include <linux/syscore_ops.h>
  32. #include <linux/cpu.h>
  33. #include <linux/sched.h>
  34. #include <linux/cpumask.h>
  35. #include <linux/smp.h>
  36. #include <linux/anon_inodes.h>
  37. #include <linux/profile.h>
  38. #include <linux/kvm_para.h>
  39. #include <linux/pagemap.h>
  40. #include <linux/mman.h>
  41. #include <linux/swap.h>
  42. #include <linux/bitops.h>
  43. #include <linux/spinlock.h>
  44. #include <linux/compat.h>
  45. #include <linux/srcu.h>
  46. #include <linux/hugetlb.h>
  47. #include <linux/slab.h>
  48. #include <linux/sort.h>
  49. #include <linux/bsearch.h>
  50. #include <asm/processor.h>
  51. #include <asm/io.h>
  52. #include <asm/ioctl.h>
  53. #include <asm/uaccess.h>
  54. #include <asm/pgtable.h>
  55. #include "coalesced_mmio.h"
  56. #include "async_pf.h"
  57. #include "vfio.h"
  58. #define CREATE_TRACE_POINTS
  59. #include <trace/events/kvm.h>
  60. MODULE_AUTHOR("Qumranet");
  61. MODULE_LICENSE("GPL");
  62. static unsigned int halt_poll_ns;
  63. module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR);
  64. /*
  65. * Ordering of locks:
  66. *
  67. * kvm->lock --> kvm->slots_lock --> kvm->irq_lock
  68. */
  69. DEFINE_SPINLOCK(kvm_lock);
  70. static DEFINE_RAW_SPINLOCK(kvm_count_lock);
  71. LIST_HEAD(vm_list);
  72. static cpumask_var_t cpus_hardware_enabled;
  73. static int kvm_usage_count;
  74. static atomic_t hardware_enable_failed;
  75. struct kmem_cache *kvm_vcpu_cache;
  76. EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
  77. static __read_mostly struct preempt_ops kvm_preempt_ops;
  78. struct dentry *kvm_debugfs_dir;
  79. EXPORT_SYMBOL_GPL(kvm_debugfs_dir);
  80. static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
  81. unsigned long arg);
  82. #ifdef CONFIG_KVM_COMPAT
  83. static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
  84. unsigned long arg);
  85. #endif
  86. static int hardware_enable_all(void);
  87. static void hardware_disable_all(void);
  88. static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
  89. static void kvm_release_pfn_dirty(pfn_t pfn);
  90. static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn);
  91. __visible bool kvm_rebooting;
  92. EXPORT_SYMBOL_GPL(kvm_rebooting);
  93. static bool largepages_enabled = true;
  94. bool kvm_is_reserved_pfn(pfn_t pfn)
  95. {
  96. if (pfn_valid(pfn))
  97. return PageReserved(pfn_to_page(pfn));
  98. return true;
  99. }
  100. /*
  101. * Switches to specified vcpu, until a matching vcpu_put()
  102. */
  103. int vcpu_load(struct kvm_vcpu *vcpu)
  104. {
  105. int cpu;
  106. if (mutex_lock_killable(&vcpu->mutex))
  107. return -EINTR;
  108. cpu = get_cpu();
  109. preempt_notifier_register(&vcpu->preempt_notifier);
  110. kvm_arch_vcpu_load(vcpu, cpu);
  111. put_cpu();
  112. return 0;
  113. }
  114. void vcpu_put(struct kvm_vcpu *vcpu)
  115. {
  116. preempt_disable();
  117. kvm_arch_vcpu_put(vcpu);
  118. preempt_notifier_unregister(&vcpu->preempt_notifier);
  119. preempt_enable();
  120. mutex_unlock(&vcpu->mutex);
  121. }
  122. static void ack_flush(void *_completed)
  123. {
  124. }
  125. bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
  126. {
  127. int i, cpu, me;
  128. cpumask_var_t cpus;
  129. bool called = true;
  130. struct kvm_vcpu *vcpu;
  131. zalloc_cpumask_var(&cpus, GFP_ATOMIC);
  132. me = get_cpu();
  133. kvm_for_each_vcpu(i, vcpu, kvm) {
  134. kvm_make_request(req, vcpu);
  135. cpu = vcpu->cpu;
  136. /* Set ->requests bit before we read ->mode */
  137. smp_mb();
  138. if (cpus != NULL && cpu != -1 && cpu != me &&
  139. kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
  140. cpumask_set_cpu(cpu, cpus);
  141. }
  142. if (unlikely(cpus == NULL))
  143. smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
  144. else if (!cpumask_empty(cpus))
  145. smp_call_function_many(cpus, ack_flush, NULL, 1);
  146. else
  147. called = false;
  148. put_cpu();
  149. free_cpumask_var(cpus);
  150. return called;
  151. }
  152. #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
  153. void kvm_flush_remote_tlbs(struct kvm *kvm)
  154. {
  155. long dirty_count = kvm->tlbs_dirty;
  156. smp_mb();
  157. if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
  158. ++kvm->stat.remote_tlb_flush;
  159. cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
  160. }
  161. EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
  162. #endif
  163. void kvm_reload_remote_mmus(struct kvm *kvm)
  164. {
  165. kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
  166. }
  167. void kvm_make_mclock_inprogress_request(struct kvm *kvm)
  168. {
  169. kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
  170. }
  171. void kvm_make_scan_ioapic_request(struct kvm *kvm)
  172. {
  173. kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
  174. }
  175. int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
  176. {
  177. struct page *page;
  178. int r;
  179. mutex_init(&vcpu->mutex);
  180. vcpu->cpu = -1;
  181. vcpu->kvm = kvm;
  182. vcpu->vcpu_id = id;
  183. vcpu->pid = NULL;
  184. init_waitqueue_head(&vcpu->wq);
  185. kvm_async_pf_vcpu_init(vcpu);
  186. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  187. if (!page) {
  188. r = -ENOMEM;
  189. goto fail;
  190. }
  191. vcpu->run = page_address(page);
  192. kvm_vcpu_set_in_spin_loop(vcpu, false);
  193. kvm_vcpu_set_dy_eligible(vcpu, false);
  194. vcpu->preempted = false;
  195. r = kvm_arch_vcpu_init(vcpu);
  196. if (r < 0)
  197. goto fail_free_run;
  198. return 0;
  199. fail_free_run:
  200. free_page((unsigned long)vcpu->run);
  201. fail:
  202. return r;
  203. }
  204. EXPORT_SYMBOL_GPL(kvm_vcpu_init);
  205. void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
  206. {
  207. put_pid(vcpu->pid);
  208. kvm_arch_vcpu_uninit(vcpu);
  209. free_page((unsigned long)vcpu->run);
  210. }
  211. EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
  212. #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
  213. static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
  214. {
  215. return container_of(mn, struct kvm, mmu_notifier);
  216. }
  217. static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
  218. struct mm_struct *mm,
  219. unsigned long address)
  220. {
  221. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  222. int need_tlb_flush, idx;
  223. /*
  224. * When ->invalidate_page runs, the linux pte has been zapped
  225. * already but the page is still allocated until
  226. * ->invalidate_page returns. So if we increase the sequence
  227. * here the kvm page fault will notice if the spte can't be
  228. * established because the page is going to be freed. If
  229. * instead the kvm page fault establishes the spte before
  230. * ->invalidate_page runs, kvm_unmap_hva will release it
  231. * before returning.
  232. *
  233. * The sequence increase only need to be seen at spin_unlock
  234. * time, and not at spin_lock time.
  235. *
  236. * Increasing the sequence after the spin_unlock would be
  237. * unsafe because the kvm page fault could then establish the
  238. * pte after kvm_unmap_hva returned, without noticing the page
  239. * is going to be freed.
  240. */
  241. idx = srcu_read_lock(&kvm->srcu);
  242. spin_lock(&kvm->mmu_lock);
  243. kvm->mmu_notifier_seq++;
  244. need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
  245. /* we've to flush the tlb before the pages can be freed */
  246. if (need_tlb_flush)
  247. kvm_flush_remote_tlbs(kvm);
  248. spin_unlock(&kvm->mmu_lock);
  249. kvm_arch_mmu_notifier_invalidate_page(kvm, address);
  250. srcu_read_unlock(&kvm->srcu, idx);
  251. }
  252. static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
  253. struct mm_struct *mm,
  254. unsigned long address,
  255. pte_t pte)
  256. {
  257. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  258. int idx;
  259. idx = srcu_read_lock(&kvm->srcu);
  260. spin_lock(&kvm->mmu_lock);
  261. kvm->mmu_notifier_seq++;
  262. kvm_set_spte_hva(kvm, address, pte);
  263. spin_unlock(&kvm->mmu_lock);
  264. srcu_read_unlock(&kvm->srcu, idx);
  265. }
  266. static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
  267. struct mm_struct *mm,
  268. unsigned long start,
  269. unsigned long end)
  270. {
  271. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  272. int need_tlb_flush = 0, idx;
  273. idx = srcu_read_lock(&kvm->srcu);
  274. spin_lock(&kvm->mmu_lock);
  275. /*
  276. * The count increase must become visible at unlock time as no
  277. * spte can be established without taking the mmu_lock and
  278. * count is also read inside the mmu_lock critical section.
  279. */
  280. kvm->mmu_notifier_count++;
  281. need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
  282. need_tlb_flush |= kvm->tlbs_dirty;
  283. /* we've to flush the tlb before the pages can be freed */
  284. if (need_tlb_flush)
  285. kvm_flush_remote_tlbs(kvm);
  286. spin_unlock(&kvm->mmu_lock);
  287. srcu_read_unlock(&kvm->srcu, idx);
  288. }
  289. static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
  290. struct mm_struct *mm,
  291. unsigned long start,
  292. unsigned long end)
  293. {
  294. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  295. spin_lock(&kvm->mmu_lock);
  296. /*
  297. * This sequence increase will notify the kvm page fault that
  298. * the page that is going to be mapped in the spte could have
  299. * been freed.
  300. */
  301. kvm->mmu_notifier_seq++;
  302. smp_wmb();
  303. /*
  304. * The above sequence increase must be visible before the
  305. * below count decrease, which is ensured by the smp_wmb above
  306. * in conjunction with the smp_rmb in mmu_notifier_retry().
  307. */
  308. kvm->mmu_notifier_count--;
  309. spin_unlock(&kvm->mmu_lock);
  310. BUG_ON(kvm->mmu_notifier_count < 0);
  311. }
  312. static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
  313. struct mm_struct *mm,
  314. unsigned long start,
  315. unsigned long end)
  316. {
  317. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  318. int young, idx;
  319. idx = srcu_read_lock(&kvm->srcu);
  320. spin_lock(&kvm->mmu_lock);
  321. young = kvm_age_hva(kvm, start, end);
  322. if (young)
  323. kvm_flush_remote_tlbs(kvm);
  324. spin_unlock(&kvm->mmu_lock);
  325. srcu_read_unlock(&kvm->srcu, idx);
  326. return young;
  327. }
  328. static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
  329. struct mm_struct *mm,
  330. unsigned long address)
  331. {
  332. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  333. int young, idx;
  334. idx = srcu_read_lock(&kvm->srcu);
  335. spin_lock(&kvm->mmu_lock);
  336. young = kvm_test_age_hva(kvm, address);
  337. spin_unlock(&kvm->mmu_lock);
  338. srcu_read_unlock(&kvm->srcu, idx);
  339. return young;
  340. }
  341. static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
  342. struct mm_struct *mm)
  343. {
  344. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  345. int idx;
  346. idx = srcu_read_lock(&kvm->srcu);
  347. kvm_arch_flush_shadow_all(kvm);
  348. srcu_read_unlock(&kvm->srcu, idx);
  349. }
  350. static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
  351. .invalidate_page = kvm_mmu_notifier_invalidate_page,
  352. .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
  353. .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end,
  354. .clear_flush_young = kvm_mmu_notifier_clear_flush_young,
  355. .test_young = kvm_mmu_notifier_test_young,
  356. .change_pte = kvm_mmu_notifier_change_pte,
  357. .release = kvm_mmu_notifier_release,
  358. };
  359. static int kvm_init_mmu_notifier(struct kvm *kvm)
  360. {
  361. kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
  362. return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
  363. }
  364. #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
  365. static int kvm_init_mmu_notifier(struct kvm *kvm)
  366. {
  367. return 0;
  368. }
  369. #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
  370. static struct kvm_memslots *kvm_alloc_memslots(void)
  371. {
  372. int i;
  373. struct kvm_memslots *slots;
  374. slots = kvm_kvzalloc(sizeof(struct kvm_memslots));
  375. if (!slots)
  376. return NULL;
  377. /*
  378. * Init kvm generation close to the maximum to easily test the
  379. * code of handling generation number wrap-around.
  380. */
  381. slots->generation = -150;
  382. for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
  383. slots->id_to_index[i] = slots->memslots[i].id = i;
  384. return slots;
  385. }
  386. static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
  387. {
  388. if (!memslot->dirty_bitmap)
  389. return;
  390. kvfree(memslot->dirty_bitmap);
  391. memslot->dirty_bitmap = NULL;
  392. }
  393. /*
  394. * Free any memory in @free but not in @dont.
  395. */
  396. static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
  397. struct kvm_memory_slot *dont)
  398. {
  399. if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
  400. kvm_destroy_dirty_bitmap(free);
  401. kvm_arch_free_memslot(kvm, free, dont);
  402. free->npages = 0;
  403. }
  404. static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
  405. {
  406. struct kvm_memory_slot *memslot;
  407. if (!slots)
  408. return;
  409. kvm_for_each_memslot(memslot, slots)
  410. kvm_free_memslot(kvm, memslot, NULL);
  411. kvfree(slots);
  412. }
  413. static struct kvm *kvm_create_vm(unsigned long type)
  414. {
  415. int r, i;
  416. struct kvm *kvm = kvm_arch_alloc_vm();
  417. if (!kvm)
  418. return ERR_PTR(-ENOMEM);
  419. r = kvm_arch_init_vm(kvm, type);
  420. if (r)
  421. goto out_err_no_disable;
  422. r = hardware_enable_all();
  423. if (r)
  424. goto out_err_no_disable;
  425. #ifdef CONFIG_HAVE_KVM_IRQFD
  426. INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
  427. #endif
  428. BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
  429. r = -ENOMEM;
  430. for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
  431. kvm->memslots[i] = kvm_alloc_memslots();
  432. if (!kvm->memslots[i])
  433. goto out_err_no_srcu;
  434. }
  435. if (init_srcu_struct(&kvm->srcu))
  436. goto out_err_no_srcu;
  437. if (init_srcu_struct(&kvm->irq_srcu))
  438. goto out_err_no_irq_srcu;
  439. for (i = 0; i < KVM_NR_BUSES; i++) {
  440. kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
  441. GFP_KERNEL);
  442. if (!kvm->buses[i])
  443. goto out_err;
  444. }
  445. spin_lock_init(&kvm->mmu_lock);
  446. kvm->mm = current->mm;
  447. atomic_inc(&kvm->mm->mm_count);
  448. kvm_eventfd_init(kvm);
  449. mutex_init(&kvm->lock);
  450. mutex_init(&kvm->irq_lock);
  451. mutex_init(&kvm->slots_lock);
  452. atomic_set(&kvm->users_count, 1);
  453. INIT_LIST_HEAD(&kvm->devices);
  454. r = kvm_init_mmu_notifier(kvm);
  455. if (r)
  456. goto out_err;
  457. spin_lock(&kvm_lock);
  458. list_add(&kvm->vm_list, &vm_list);
  459. spin_unlock(&kvm_lock);
  460. preempt_notifier_inc();
  461. return kvm;
  462. out_err:
  463. cleanup_srcu_struct(&kvm->irq_srcu);
  464. out_err_no_irq_srcu:
  465. cleanup_srcu_struct(&kvm->srcu);
  466. out_err_no_srcu:
  467. hardware_disable_all();
  468. out_err_no_disable:
  469. for (i = 0; i < KVM_NR_BUSES; i++)
  470. kfree(kvm->buses[i]);
  471. for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
  472. kvm_free_memslots(kvm, kvm->memslots[i]);
  473. kvm_arch_free_vm(kvm);
  474. return ERR_PTR(r);
  475. }
  476. /*
  477. * Avoid using vmalloc for a small buffer.
  478. * Should not be used when the size is statically known.
  479. */
  480. void *kvm_kvzalloc(unsigned long size)
  481. {
  482. if (size > PAGE_SIZE)
  483. return vzalloc(size);
  484. else
  485. return kzalloc(size, GFP_KERNEL);
  486. }
  487. static void kvm_destroy_devices(struct kvm *kvm)
  488. {
  489. struct list_head *node, *tmp;
  490. list_for_each_safe(node, tmp, &kvm->devices) {
  491. struct kvm_device *dev =
  492. list_entry(node, struct kvm_device, vm_node);
  493. list_del(node);
  494. dev->ops->destroy(dev);
  495. }
  496. }
  497. static void kvm_destroy_vm(struct kvm *kvm)
  498. {
  499. int i;
  500. struct mm_struct *mm = kvm->mm;
  501. kvm_arch_sync_events(kvm);
  502. spin_lock(&kvm_lock);
  503. list_del(&kvm->vm_list);
  504. spin_unlock(&kvm_lock);
  505. kvm_free_irq_routing(kvm);
  506. for (i = 0; i < KVM_NR_BUSES; i++)
  507. kvm_io_bus_destroy(kvm->buses[i]);
  508. kvm_coalesced_mmio_free(kvm);
  509. #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
  510. mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
  511. #else
  512. kvm_arch_flush_shadow_all(kvm);
  513. #endif
  514. kvm_arch_destroy_vm(kvm);
  515. kvm_destroy_devices(kvm);
  516. for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
  517. kvm_free_memslots(kvm, kvm->memslots[i]);
  518. cleanup_srcu_struct(&kvm->irq_srcu);
  519. cleanup_srcu_struct(&kvm->srcu);
  520. kvm_arch_free_vm(kvm);
  521. preempt_notifier_dec();
  522. hardware_disable_all();
  523. mmdrop(mm);
  524. }
  525. void kvm_get_kvm(struct kvm *kvm)
  526. {
  527. atomic_inc(&kvm->users_count);
  528. }
  529. EXPORT_SYMBOL_GPL(kvm_get_kvm);
  530. void kvm_put_kvm(struct kvm *kvm)
  531. {
  532. if (atomic_dec_and_test(&kvm->users_count))
  533. kvm_destroy_vm(kvm);
  534. }
  535. EXPORT_SYMBOL_GPL(kvm_put_kvm);
  536. static int kvm_vm_release(struct inode *inode, struct file *filp)
  537. {
  538. struct kvm *kvm = filp->private_data;
  539. kvm_irqfd_release(kvm);
  540. kvm_put_kvm(kvm);
  541. return 0;
  542. }
  543. /*
  544. * Allocation size is twice as large as the actual dirty bitmap size.
  545. * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
  546. */
  547. static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
  548. {
  549. unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
  550. memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes);
  551. if (!memslot->dirty_bitmap)
  552. return -ENOMEM;
  553. return 0;
  554. }
  555. /*
  556. * Insert memslot and re-sort memslots based on their GFN,
  557. * so binary search could be used to lookup GFN.
  558. * Sorting algorithm takes advantage of having initially
  559. * sorted array and known changed memslot position.
  560. */
  561. static void update_memslots(struct kvm_memslots *slots,
  562. struct kvm_memory_slot *new)
  563. {
  564. int id = new->id;
  565. int i = slots->id_to_index[id];
  566. struct kvm_memory_slot *mslots = slots->memslots;
  567. WARN_ON(mslots[i].id != id);
  568. if (!new->npages) {
  569. WARN_ON(!mslots[i].npages);
  570. if (mslots[i].npages)
  571. slots->used_slots--;
  572. } else {
  573. if (!mslots[i].npages)
  574. slots->used_slots++;
  575. }
  576. while (i < KVM_MEM_SLOTS_NUM - 1 &&
  577. new->base_gfn <= mslots[i + 1].base_gfn) {
  578. if (!mslots[i + 1].npages)
  579. break;
  580. mslots[i] = mslots[i + 1];
  581. slots->id_to_index[mslots[i].id] = i;
  582. i++;
  583. }
  584. /*
  585. * The ">=" is needed when creating a slot with base_gfn == 0,
  586. * so that it moves before all those with base_gfn == npages == 0.
  587. *
  588. * On the other hand, if new->npages is zero, the above loop has
  589. * already left i pointing to the beginning of the empty part of
  590. * mslots, and the ">=" would move the hole backwards in this
  591. * case---which is wrong. So skip the loop when deleting a slot.
  592. */
  593. if (new->npages) {
  594. while (i > 0 &&
  595. new->base_gfn >= mslots[i - 1].base_gfn) {
  596. mslots[i] = mslots[i - 1];
  597. slots->id_to_index[mslots[i].id] = i;
  598. i--;
  599. }
  600. } else
  601. WARN_ON_ONCE(i != slots->used_slots);
  602. mslots[i] = *new;
  603. slots->id_to_index[mslots[i].id] = i;
  604. }
  605. static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem)
  606. {
  607. u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
  608. #ifdef __KVM_HAVE_READONLY_MEM
  609. valid_flags |= KVM_MEM_READONLY;
  610. #endif
  611. if (mem->flags & ~valid_flags)
  612. return -EINVAL;
  613. return 0;
  614. }
  615. static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
  616. int as_id, struct kvm_memslots *slots)
  617. {
  618. struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id);
  619. /*
  620. * Set the low bit in the generation, which disables SPTE caching
  621. * until the end of synchronize_srcu_expedited.
  622. */
  623. WARN_ON(old_memslots->generation & 1);
  624. slots->generation = old_memslots->generation + 1;
  625. rcu_assign_pointer(kvm->memslots[as_id], slots);
  626. synchronize_srcu_expedited(&kvm->srcu);
  627. /*
  628. * Increment the new memslot generation a second time. This prevents
  629. * vm exits that race with memslot updates from caching a memslot
  630. * generation that will (potentially) be valid forever.
  631. */
  632. slots->generation++;
  633. kvm_arch_memslots_updated(kvm, slots);
  634. return old_memslots;
  635. }
  636. /*
  637. * Allocate some memory and give it an address in the guest physical address
  638. * space.
  639. *
  640. * Discontiguous memory is allowed, mostly for framebuffers.
  641. *
  642. * Must be called holding kvm->slots_lock for write.
  643. */
  644. int __kvm_set_memory_region(struct kvm *kvm,
  645. const struct kvm_userspace_memory_region *mem)
  646. {
  647. int r;
  648. gfn_t base_gfn;
  649. unsigned long npages;
  650. struct kvm_memory_slot *slot;
  651. struct kvm_memory_slot old, new;
  652. struct kvm_memslots *slots = NULL, *old_memslots;
  653. int as_id, id;
  654. enum kvm_mr_change change;
  655. r = check_memory_region_flags(mem);
  656. if (r)
  657. goto out;
  658. r = -EINVAL;
  659. as_id = mem->slot >> 16;
  660. id = (u16)mem->slot;
  661. /* General sanity checks */
  662. if (mem->memory_size & (PAGE_SIZE - 1))
  663. goto out;
  664. if (mem->guest_phys_addr & (PAGE_SIZE - 1))
  665. goto out;
  666. /* We can read the guest memory with __xxx_user() later on. */
  667. if ((id < KVM_USER_MEM_SLOTS) &&
  668. ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
  669. !access_ok(VERIFY_WRITE,
  670. (void __user *)(unsigned long)mem->userspace_addr,
  671. mem->memory_size)))
  672. goto out;
  673. if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM)
  674. goto out;
  675. if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
  676. goto out;
  677. slot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
  678. base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
  679. npages = mem->memory_size >> PAGE_SHIFT;
  680. if (npages > KVM_MEM_MAX_NR_PAGES)
  681. goto out;
  682. new = old = *slot;
  683. new.id = id;
  684. new.base_gfn = base_gfn;
  685. new.npages = npages;
  686. new.flags = mem->flags;
  687. if (npages) {
  688. if (!old.npages)
  689. change = KVM_MR_CREATE;
  690. else { /* Modify an existing slot. */
  691. if ((mem->userspace_addr != old.userspace_addr) ||
  692. (npages != old.npages) ||
  693. ((new.flags ^ old.flags) & KVM_MEM_READONLY))
  694. goto out;
  695. if (base_gfn != old.base_gfn)
  696. change = KVM_MR_MOVE;
  697. else if (new.flags != old.flags)
  698. change = KVM_MR_FLAGS_ONLY;
  699. else { /* Nothing to change. */
  700. r = 0;
  701. goto out;
  702. }
  703. }
  704. } else {
  705. if (!old.npages)
  706. goto out;
  707. change = KVM_MR_DELETE;
  708. new.base_gfn = 0;
  709. new.flags = 0;
  710. }
  711. if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
  712. /* Check for overlaps */
  713. r = -EEXIST;
  714. kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) {
  715. if ((slot->id >= KVM_USER_MEM_SLOTS) ||
  716. (slot->id == id))
  717. continue;
  718. if (!((base_gfn + npages <= slot->base_gfn) ||
  719. (base_gfn >= slot->base_gfn + slot->npages)))
  720. goto out;
  721. }
  722. }
  723. /* Free page dirty bitmap if unneeded */
  724. if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
  725. new.dirty_bitmap = NULL;
  726. r = -ENOMEM;
  727. if (change == KVM_MR_CREATE) {
  728. new.userspace_addr = mem->userspace_addr;
  729. if (kvm_arch_create_memslot(kvm, &new, npages))
  730. goto out_free;
  731. }
  732. /* Allocate page dirty bitmap if needed */
  733. if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
  734. if (kvm_create_dirty_bitmap(&new) < 0)
  735. goto out_free;
  736. }
  737. slots = kvm_kvzalloc(sizeof(struct kvm_memslots));
  738. if (!slots)
  739. goto out_free;
  740. memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots));
  741. if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
  742. slot = id_to_memslot(slots, id);
  743. slot->flags |= KVM_MEMSLOT_INVALID;
  744. old_memslots = install_new_memslots(kvm, as_id, slots);
  745. /* slot was deleted or moved, clear iommu mapping */
  746. kvm_iommu_unmap_pages(kvm, &old);
  747. /* From this point no new shadow pages pointing to a deleted,
  748. * or moved, memslot will be created.
  749. *
  750. * validation of sp->gfn happens in:
  751. * - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
  752. * - kvm_is_visible_gfn (mmu_check_roots)
  753. */
  754. kvm_arch_flush_shadow_memslot(kvm, slot);
  755. /*
  756. * We can re-use the old_memslots from above, the only difference
  757. * from the currently installed memslots is the invalid flag. This
  758. * will get overwritten by update_memslots anyway.
  759. */
  760. slots = old_memslots;
  761. }
  762. r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
  763. if (r)
  764. goto out_slots;
  765. /* actual memory is freed via old in kvm_free_memslot below */
  766. if (change == KVM_MR_DELETE) {
  767. new.dirty_bitmap = NULL;
  768. memset(&new.arch, 0, sizeof(new.arch));
  769. }
  770. update_memslots(slots, &new);
  771. old_memslots = install_new_memslots(kvm, as_id, slots);
  772. kvm_arch_commit_memory_region(kvm, mem, &old, &new, change);
  773. kvm_free_memslot(kvm, &old, &new);
  774. kvfree(old_memslots);
  775. /*
  776. * IOMMU mapping: New slots need to be mapped. Old slots need to be
  777. * un-mapped and re-mapped if their base changes. Since base change
  778. * unmapping is handled above with slot deletion, mapping alone is
  779. * needed here. Anything else the iommu might care about for existing
  780. * slots (size changes, userspace addr changes and read-only flag
  781. * changes) is disallowed above, so any other attribute changes getting
  782. * here can be skipped.
  783. */
  784. if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
  785. r = kvm_iommu_map_pages(kvm, &new);
  786. return r;
  787. }
  788. return 0;
  789. out_slots:
  790. kvfree(slots);
  791. out_free:
  792. kvm_free_memslot(kvm, &new, &old);
  793. out:
  794. return r;
  795. }
  796. EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
  797. int kvm_set_memory_region(struct kvm *kvm,
  798. const struct kvm_userspace_memory_region *mem)
  799. {
  800. int r;
  801. mutex_lock(&kvm->slots_lock);
  802. r = __kvm_set_memory_region(kvm, mem);
  803. mutex_unlock(&kvm->slots_lock);
  804. return r;
  805. }
  806. EXPORT_SYMBOL_GPL(kvm_set_memory_region);
  807. static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
  808. struct kvm_userspace_memory_region *mem)
  809. {
  810. if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
  811. return -EINVAL;
  812. return kvm_set_memory_region(kvm, mem);
  813. }
  814. int kvm_get_dirty_log(struct kvm *kvm,
  815. struct kvm_dirty_log *log, int *is_dirty)
  816. {
  817. struct kvm_memslots *slots;
  818. struct kvm_memory_slot *memslot;
  819. int r, i, as_id, id;
  820. unsigned long n;
  821. unsigned long any = 0;
  822. r = -EINVAL;
  823. as_id = log->slot >> 16;
  824. id = (u16)log->slot;
  825. if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
  826. goto out;
  827. slots = __kvm_memslots(kvm, as_id);
  828. memslot = id_to_memslot(slots, id);
  829. r = -ENOENT;
  830. if (!memslot->dirty_bitmap)
  831. goto out;
  832. n = kvm_dirty_bitmap_bytes(memslot);
  833. for (i = 0; !any && i < n/sizeof(long); ++i)
  834. any = memslot->dirty_bitmap[i];
  835. r = -EFAULT;
  836. if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
  837. goto out;
  838. if (any)
  839. *is_dirty = 1;
  840. r = 0;
  841. out:
  842. return r;
  843. }
  844. EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
  845. #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
  846. /**
  847. * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages
  848. * are dirty write protect them for next write.
  849. * @kvm: pointer to kvm instance
  850. * @log: slot id and address to which we copy the log
  851. * @is_dirty: flag set if any page is dirty
  852. *
  853. * We need to keep it in mind that VCPU threads can write to the bitmap
  854. * concurrently. So, to avoid losing track of dirty pages we keep the
  855. * following order:
  856. *
  857. * 1. Take a snapshot of the bit and clear it if needed.
  858. * 2. Write protect the corresponding page.
  859. * 3. Copy the snapshot to the userspace.
  860. * 4. Upon return caller flushes TLB's if needed.
  861. *
  862. * Between 2 and 4, the guest may write to the page using the remaining TLB
  863. * entry. This is not a problem because the page is reported dirty using
  864. * the snapshot taken before and step 4 ensures that writes done after
  865. * exiting to userspace will be logged for the next call.
  866. *
  867. */
  868. int kvm_get_dirty_log_protect(struct kvm *kvm,
  869. struct kvm_dirty_log *log, bool *is_dirty)
  870. {
  871. struct kvm_memslots *slots;
  872. struct kvm_memory_slot *memslot;
  873. int r, i, as_id, id;
  874. unsigned long n;
  875. unsigned long *dirty_bitmap;
  876. unsigned long *dirty_bitmap_buffer;
  877. r = -EINVAL;
  878. as_id = log->slot >> 16;
  879. id = (u16)log->slot;
  880. if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
  881. goto out;
  882. slots = __kvm_memslots(kvm, as_id);
  883. memslot = id_to_memslot(slots, id);
  884. dirty_bitmap = memslot->dirty_bitmap;
  885. r = -ENOENT;
  886. if (!dirty_bitmap)
  887. goto out;
  888. n = kvm_dirty_bitmap_bytes(memslot);
  889. dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long);
  890. memset(dirty_bitmap_buffer, 0, n);
  891. spin_lock(&kvm->mmu_lock);
  892. *is_dirty = false;
  893. for (i = 0; i < n / sizeof(long); i++) {
  894. unsigned long mask;
  895. gfn_t offset;
  896. if (!dirty_bitmap[i])
  897. continue;
  898. *is_dirty = true;
  899. mask = xchg(&dirty_bitmap[i], 0);
  900. dirty_bitmap_buffer[i] = mask;
  901. if (mask) {
  902. offset = i * BITS_PER_LONG;
  903. kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
  904. offset, mask);
  905. }
  906. }
  907. spin_unlock(&kvm->mmu_lock);
  908. r = -EFAULT;
  909. if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
  910. goto out;
  911. r = 0;
  912. out:
  913. return r;
  914. }
  915. EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect);
  916. #endif
  917. bool kvm_largepages_enabled(void)
  918. {
  919. return largepages_enabled;
  920. }
  921. void kvm_disable_largepages(void)
  922. {
  923. largepages_enabled = false;
  924. }
  925. EXPORT_SYMBOL_GPL(kvm_disable_largepages);
  926. struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
  927. {
  928. return __gfn_to_memslot(kvm_memslots(kvm), gfn);
  929. }
  930. EXPORT_SYMBOL_GPL(gfn_to_memslot);
  931. struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
  932. {
  933. return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn);
  934. }
  935. int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
  936. {
  937. struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
  938. if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
  939. memslot->flags & KVM_MEMSLOT_INVALID)
  940. return 0;
  941. return 1;
  942. }
  943. EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
  944. unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
  945. {
  946. struct vm_area_struct *vma;
  947. unsigned long addr, size;
  948. size = PAGE_SIZE;
  949. addr = gfn_to_hva(kvm, gfn);
  950. if (kvm_is_error_hva(addr))
  951. return PAGE_SIZE;
  952. down_read(&current->mm->mmap_sem);
  953. vma = find_vma(current->mm, addr);
  954. if (!vma)
  955. goto out;
  956. size = vma_kernel_pagesize(vma);
  957. out:
  958. up_read(&current->mm->mmap_sem);
  959. return size;
  960. }
  961. static bool memslot_is_readonly(struct kvm_memory_slot *slot)
  962. {
  963. return slot->flags & KVM_MEM_READONLY;
  964. }
  965. static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
  966. gfn_t *nr_pages, bool write)
  967. {
  968. if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
  969. return KVM_HVA_ERR_BAD;
  970. if (memslot_is_readonly(slot) && write)
  971. return KVM_HVA_ERR_RO_BAD;
  972. if (nr_pages)
  973. *nr_pages = slot->npages - (gfn - slot->base_gfn);
  974. return __gfn_to_hva_memslot(slot, gfn);
  975. }
  976. static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
  977. gfn_t *nr_pages)
  978. {
  979. return __gfn_to_hva_many(slot, gfn, nr_pages, true);
  980. }
  981. unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
  982. gfn_t gfn)
  983. {
  984. return gfn_to_hva_many(slot, gfn, NULL);
  985. }
  986. EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
  987. unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
  988. {
  989. return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
  990. }
  991. EXPORT_SYMBOL_GPL(gfn_to_hva);
  992. unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)
  993. {
  994. return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
  995. }
  996. EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva);
  997. /*
  998. * If writable is set to false, the hva returned by this function is only
  999. * allowed to be read.
  1000. */
  1001. unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
  1002. gfn_t gfn, bool *writable)
  1003. {
  1004. unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);
  1005. if (!kvm_is_error_hva(hva) && writable)
  1006. *writable = !memslot_is_readonly(slot);
  1007. return hva;
  1008. }
  1009. unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
  1010. {
  1011. struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
  1012. return gfn_to_hva_memslot_prot(slot, gfn, writable);
  1013. }
  1014. unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
  1015. {
  1016. struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
  1017. return gfn_to_hva_memslot_prot(slot, gfn, writable);
  1018. }
  1019. static int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
  1020. unsigned long start, int write, struct page **page)
  1021. {
  1022. int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;
  1023. if (write)
  1024. flags |= FOLL_WRITE;
  1025. return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
  1026. }
  1027. static inline int check_user_page_hwpoison(unsigned long addr)
  1028. {
  1029. int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;
  1030. rc = __get_user_pages(current, current->mm, addr, 1,
  1031. flags, NULL, NULL, NULL);
  1032. return rc == -EHWPOISON;
  1033. }
  1034. /*
  1035. * The atomic path to get the writable pfn which will be stored in @pfn,
  1036. * true indicates success, otherwise false is returned.
  1037. */
  1038. static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
  1039. bool write_fault, bool *writable, pfn_t *pfn)
  1040. {
  1041. struct page *page[1];
  1042. int npages;
  1043. if (!(async || atomic))
  1044. return false;
  1045. /*
  1046. * Fast pin a writable pfn only if it is a write fault request
  1047. * or the caller allows to map a writable pfn for a read fault
  1048. * request.
  1049. */
  1050. if (!(write_fault || writable))
  1051. return false;
  1052. npages = __get_user_pages_fast(addr, 1, 1, page);
  1053. if (npages == 1) {
  1054. *pfn = page_to_pfn(page[0]);
  1055. if (writable)
  1056. *writable = true;
  1057. return true;
  1058. }
  1059. return false;
  1060. }
  1061. /*
  1062. * The slow path to get the pfn of the specified host virtual address,
  1063. * 1 indicates success, -errno is returned if error is detected.
  1064. */
  1065. static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
  1066. bool *writable, pfn_t *pfn)
  1067. {
  1068. struct page *page[1];
  1069. int npages = 0;
  1070. might_sleep();
  1071. if (writable)
  1072. *writable = write_fault;
  1073. if (async) {
  1074. down_read(&current->mm->mmap_sem);
  1075. npages = get_user_page_nowait(current, current->mm,
  1076. addr, write_fault, page);
  1077. up_read(&current->mm->mmap_sem);
  1078. } else
  1079. npages = __get_user_pages_unlocked(current, current->mm, addr, 1,
  1080. write_fault, 0, page,
  1081. FOLL_TOUCH|FOLL_HWPOISON);
  1082. if (npages != 1)
  1083. return npages;
  1084. /* map read fault as writable if possible */
  1085. if (unlikely(!write_fault) && writable) {
  1086. struct page *wpage[1];
  1087. npages = __get_user_pages_fast(addr, 1, 1, wpage);
  1088. if (npages == 1) {
  1089. *writable = true;
  1090. put_page(page[0]);
  1091. page[0] = wpage[0];
  1092. }
  1093. npages = 1;
  1094. }
  1095. *pfn = page_to_pfn(page[0]);
  1096. return npages;
  1097. }
  1098. static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
  1099. {
  1100. if (unlikely(!(vma->vm_flags & VM_READ)))
  1101. return false;
  1102. if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
  1103. return false;
  1104. return true;
  1105. }
  1106. /*
  1107. * Pin guest page in memory and return its pfn.
  1108. * @addr: host virtual address which maps memory to the guest
  1109. * @atomic: whether this function can sleep
  1110. * @async: whether this function need to wait IO complete if the
  1111. * host page is not in the memory
  1112. * @write_fault: whether we should get a writable host page
  1113. * @writable: whether it allows to map a writable host page for !@write_fault
  1114. *
  1115. * The function will map a writable host page for these two cases:
  1116. * 1): @write_fault = true
  1117. * 2): @write_fault = false && @writable, @writable will tell the caller
  1118. * whether the mapping is writable.
  1119. */
  1120. static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
  1121. bool write_fault, bool *writable)
  1122. {
  1123. struct vm_area_struct *vma;
  1124. pfn_t pfn = 0;
  1125. int npages;
  1126. /* we can do it either atomically or asynchronously, not both */
  1127. BUG_ON(atomic && async);
  1128. if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
  1129. return pfn;
  1130. if (atomic)
  1131. return KVM_PFN_ERR_FAULT;
  1132. npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
  1133. if (npages == 1)
  1134. return pfn;
  1135. down_read(&current->mm->mmap_sem);
  1136. if (npages == -EHWPOISON ||
  1137. (!async && check_user_page_hwpoison(addr))) {
  1138. pfn = KVM_PFN_ERR_HWPOISON;
  1139. goto exit;
  1140. }
  1141. vma = find_vma_intersection(current->mm, addr, addr + 1);
  1142. if (vma == NULL)
  1143. pfn = KVM_PFN_ERR_FAULT;
  1144. else if ((vma->vm_flags & VM_PFNMAP)) {
  1145. pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
  1146. vma->vm_pgoff;
  1147. BUG_ON(!kvm_is_reserved_pfn(pfn));
  1148. } else {
  1149. if (async && vma_is_valid(vma, write_fault))
  1150. *async = true;
  1151. pfn = KVM_PFN_ERR_FAULT;
  1152. }
  1153. exit:
  1154. up_read(&current->mm->mmap_sem);
  1155. return pfn;
  1156. }
  1157. pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic,
  1158. bool *async, bool write_fault, bool *writable)
  1159. {
  1160. unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
  1161. if (addr == KVM_HVA_ERR_RO_BAD)
  1162. return KVM_PFN_ERR_RO_FAULT;
  1163. if (kvm_is_error_hva(addr))
  1164. return KVM_PFN_NOSLOT;
  1165. /* Do not map writable pfn in the readonly memslot. */
  1166. if (writable && memslot_is_readonly(slot)) {
  1167. *writable = false;
  1168. writable = NULL;
  1169. }
  1170. return hva_to_pfn(addr, atomic, async, write_fault,
  1171. writable);
  1172. }
  1173. EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot);
  1174. pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
  1175. bool *writable)
  1176. {
  1177. return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL,
  1178. write_fault, writable);
  1179. }
  1180. EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
  1181. pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
  1182. {
  1183. return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
  1184. }
  1185. EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot);
  1186. pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
  1187. {
  1188. return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
  1189. }
  1190. EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
  1191. pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
  1192. {
  1193. return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn);
  1194. }
  1195. EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
  1196. pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn)
  1197. {
  1198. return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
  1199. }
  1200. EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic);
  1201. pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
  1202. {
  1203. return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn);
  1204. }
  1205. EXPORT_SYMBOL_GPL(gfn_to_pfn);
  1206. pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
  1207. {
  1208. return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
  1209. }
  1210. EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn);
  1211. int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
  1212. struct page **pages, int nr_pages)
  1213. {
  1214. unsigned long addr;
  1215. gfn_t entry;
  1216. addr = gfn_to_hva_many(slot, gfn, &entry);
  1217. if (kvm_is_error_hva(addr))
  1218. return -1;
  1219. if (entry < nr_pages)
  1220. return 0;
  1221. return __get_user_pages_fast(addr, nr_pages, 1, pages);
  1222. }
  1223. EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
  1224. static struct page *kvm_pfn_to_page(pfn_t pfn)
  1225. {
  1226. if (is_error_noslot_pfn(pfn))
  1227. return KVM_ERR_PTR_BAD_PAGE;
  1228. if (kvm_is_reserved_pfn(pfn)) {
  1229. WARN_ON(1);
  1230. return KVM_ERR_PTR_BAD_PAGE;
  1231. }
  1232. return pfn_to_page(pfn);
  1233. }
  1234. struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
  1235. {
  1236. pfn_t pfn;
  1237. pfn = gfn_to_pfn(kvm, gfn);
  1238. return kvm_pfn_to_page(pfn);
  1239. }
  1240. EXPORT_SYMBOL_GPL(gfn_to_page);
  1241. struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn)
  1242. {
  1243. pfn_t pfn;
  1244. pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn);
  1245. return kvm_pfn_to_page(pfn);
  1246. }
  1247. EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page);
  1248. void kvm_release_page_clean(struct page *page)
  1249. {
  1250. WARN_ON(is_error_page(page));
  1251. kvm_release_pfn_clean(page_to_pfn(page));
  1252. }
  1253. EXPORT_SYMBOL_GPL(kvm_release_page_clean);
  1254. void kvm_release_pfn_clean(pfn_t pfn)
  1255. {
  1256. if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn))
  1257. put_page(pfn_to_page(pfn));
  1258. }
  1259. EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
  1260. void kvm_release_page_dirty(struct page *page)
  1261. {
  1262. WARN_ON(is_error_page(page));
  1263. kvm_release_pfn_dirty(page_to_pfn(page));
  1264. }
  1265. EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
  1266. static void kvm_release_pfn_dirty(pfn_t pfn)
  1267. {
  1268. kvm_set_pfn_dirty(pfn);
  1269. kvm_release_pfn_clean(pfn);
  1270. }
  1271. void kvm_set_pfn_dirty(pfn_t pfn)
  1272. {
  1273. if (!kvm_is_reserved_pfn(pfn)) {
  1274. struct page *page = pfn_to_page(pfn);
  1275. if (!PageReserved(page))
  1276. SetPageDirty(page);
  1277. }
  1278. }
  1279. EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
  1280. void kvm_set_pfn_accessed(pfn_t pfn)
  1281. {
  1282. if (!kvm_is_reserved_pfn(pfn))
  1283. mark_page_accessed(pfn_to_page(pfn));
  1284. }
  1285. EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
  1286. void kvm_get_pfn(pfn_t pfn)
  1287. {
  1288. if (!kvm_is_reserved_pfn(pfn))
  1289. get_page(pfn_to_page(pfn));
  1290. }
  1291. EXPORT_SYMBOL_GPL(kvm_get_pfn);
  1292. static int next_segment(unsigned long len, int offset)
  1293. {
  1294. if (len > PAGE_SIZE - offset)
  1295. return PAGE_SIZE - offset;
  1296. else
  1297. return len;
  1298. }
  1299. static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,
  1300. void *data, int offset, int len)
  1301. {
  1302. int r;
  1303. unsigned long addr;
  1304. addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
  1305. if (kvm_is_error_hva(addr))
  1306. return -EFAULT;
  1307. r = __copy_from_user(data, (void __user *)addr + offset, len);
  1308. if (r)
  1309. return -EFAULT;
  1310. return 0;
  1311. }
  1312. int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
  1313. int len)
  1314. {
  1315. struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
  1316. return __kvm_read_guest_page(slot, gfn, data, offset, len);
  1317. }
  1318. EXPORT_SYMBOL_GPL(kvm_read_guest_page);
  1319. int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,
  1320. int offset, int len)
  1321. {
  1322. struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
  1323. return __kvm_read_guest_page(slot, gfn, data, offset, len);
  1324. }
  1325. EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page);
  1326. int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
  1327. {
  1328. gfn_t gfn = gpa >> PAGE_SHIFT;
  1329. int seg;
  1330. int offset = offset_in_page(gpa);
  1331. int ret;
  1332. while ((seg = next_segment(len, offset)) != 0) {
  1333. ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
  1334. if (ret < 0)
  1335. return ret;
  1336. offset = 0;
  1337. len -= seg;
  1338. data += seg;
  1339. ++gfn;
  1340. }
  1341. return 0;
  1342. }
  1343. EXPORT_SYMBOL_GPL(kvm_read_guest);
  1344. int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
  1345. {
  1346. gfn_t gfn = gpa >> PAGE_SHIFT;
  1347. int seg;
  1348. int offset = offset_in_page(gpa);
  1349. int ret;
  1350. while ((seg = next_segment(len, offset)) != 0) {
  1351. ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
  1352. if (ret < 0)
  1353. return ret;
  1354. offset = 0;
  1355. len -= seg;
  1356. data += seg;
  1357. ++gfn;
  1358. }
  1359. return 0;
  1360. }
  1361. EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest);
  1362. static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
  1363. void *data, int offset, unsigned long len)
  1364. {
  1365. int r;
  1366. unsigned long addr;
  1367. addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
  1368. if (kvm_is_error_hva(addr))
  1369. return -EFAULT;
  1370. pagefault_disable();
  1371. r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
  1372. pagefault_enable();
  1373. if (r)
  1374. return -EFAULT;
  1375. return 0;
  1376. }
  1377. int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
  1378. unsigned long len)
  1379. {
  1380. gfn_t gfn = gpa >> PAGE_SHIFT;
  1381. struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
  1382. int offset = offset_in_page(gpa);
  1383. return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
  1384. }
  1385. EXPORT_SYMBOL_GPL(kvm_read_guest_atomic);
  1386. int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa,
  1387. void *data, unsigned long len)
  1388. {
  1389. gfn_t gfn = gpa >> PAGE_SHIFT;
  1390. struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
  1391. int offset = offset_in_page(gpa);
  1392. return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
  1393. }
  1394. EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic);
  1395. static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn,
  1396. const void *data, int offset, int len)
  1397. {
  1398. int r;
  1399. unsigned long addr;
  1400. addr = gfn_to_hva_memslot(memslot, gfn);
  1401. if (kvm_is_error_hva(addr))
  1402. return -EFAULT;
  1403. r = __copy_to_user((void __user *)addr + offset, data, len);
  1404. if (r)
  1405. return -EFAULT;
  1406. mark_page_dirty_in_slot(memslot, gfn);
  1407. return 0;
  1408. }
  1409. int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
  1410. const void *data, int offset, int len)
  1411. {
  1412. struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
  1413. return __kvm_write_guest_page(slot, gfn, data, offset, len);
  1414. }
  1415. EXPORT_SYMBOL_GPL(kvm_write_guest_page);
  1416. int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
  1417. const void *data, int offset, int len)
  1418. {
  1419. struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
  1420. return __kvm_write_guest_page(slot, gfn, data, offset, len);
  1421. }
  1422. EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page);
  1423. int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
  1424. unsigned long len)
  1425. {
  1426. gfn_t gfn = gpa >> PAGE_SHIFT;
  1427. int seg;
  1428. int offset = offset_in_page(gpa);
  1429. int ret;
  1430. while ((seg = next_segment(len, offset)) != 0) {
  1431. ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
  1432. if (ret < 0)
  1433. return ret;
  1434. offset = 0;
  1435. len -= seg;
  1436. data += seg;
  1437. ++gfn;
  1438. }
  1439. return 0;
  1440. }
  1441. EXPORT_SYMBOL_GPL(kvm_write_guest);
  1442. int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
  1443. unsigned long len)
  1444. {
  1445. gfn_t gfn = gpa >> PAGE_SHIFT;
  1446. int seg;
  1447. int offset = offset_in_page(gpa);
  1448. int ret;
  1449. while ((seg = next_segment(len, offset)) != 0) {
  1450. ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
  1451. if (ret < 0)
  1452. return ret;
  1453. offset = 0;
  1454. len -= seg;
  1455. data += seg;
  1456. ++gfn;
  1457. }
  1458. return 0;
  1459. }
  1460. EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest);
  1461. int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
  1462. gpa_t gpa, unsigned long len)
  1463. {
  1464. struct kvm_memslots *slots = kvm_memslots(kvm);
  1465. int offset = offset_in_page(gpa);
  1466. gfn_t start_gfn = gpa >> PAGE_SHIFT;
  1467. gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
  1468. gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
  1469. gfn_t nr_pages_avail;
  1470. ghc->gpa = gpa;
  1471. ghc->generation = slots->generation;
  1472. ghc->len = len;
  1473. ghc->memslot = gfn_to_memslot(kvm, start_gfn);
  1474. ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, NULL);
  1475. if (!kvm_is_error_hva(ghc->hva) && nr_pages_needed <= 1) {
  1476. ghc->hva += offset;
  1477. } else {
  1478. /*
  1479. * If the requested region crosses two memslots, we still
  1480. * verify that the entire region is valid here.
  1481. */
  1482. while (start_gfn <= end_gfn) {
  1483. ghc->memslot = gfn_to_memslot(kvm, start_gfn);
  1484. ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
  1485. &nr_pages_avail);
  1486. if (kvm_is_error_hva(ghc->hva))
  1487. return -EFAULT;
  1488. start_gfn += nr_pages_avail;
  1489. }
  1490. /* Use the slow path for cross page reads and writes. */
  1491. ghc->memslot = NULL;
  1492. }
  1493. return 0;
  1494. }
  1495. EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
  1496. int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
  1497. void *data, unsigned long len)
  1498. {
  1499. struct kvm_memslots *slots = kvm_memslots(kvm);
  1500. int r;
  1501. BUG_ON(len > ghc->len);
  1502. if (slots->generation != ghc->generation)
  1503. kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);
  1504. if (unlikely(!ghc->memslot))
  1505. return kvm_write_guest(kvm, ghc->gpa, data, len);
  1506. if (kvm_is_error_hva(ghc->hva))
  1507. return -EFAULT;
  1508. r = __copy_to_user((void __user *)ghc->hva, data, len);
  1509. if (r)
  1510. return -EFAULT;
  1511. mark_page_dirty_in_slot(ghc->memslot, ghc->gpa >> PAGE_SHIFT);
  1512. return 0;
  1513. }
  1514. EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
  1515. int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
  1516. void *data, unsigned long len)
  1517. {
  1518. struct kvm_memslots *slots = kvm_memslots(kvm);
  1519. int r;
  1520. BUG_ON(len > ghc->len);
  1521. if (slots->generation != ghc->generation)
  1522. kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);
  1523. if (unlikely(!ghc->memslot))
  1524. return kvm_read_guest(kvm, ghc->gpa, data, len);
  1525. if (kvm_is_error_hva(ghc->hva))
  1526. return -EFAULT;
  1527. r = __copy_from_user(data, (void __user *)ghc->hva, len);
  1528. if (r)
  1529. return -EFAULT;
  1530. return 0;
  1531. }
  1532. EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
  1533. int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
  1534. {
  1535. const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
  1536. return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
  1537. }
  1538. EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
  1539. int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
  1540. {
  1541. gfn_t gfn = gpa >> PAGE_SHIFT;
  1542. int seg;
  1543. int offset = offset_in_page(gpa);
  1544. int ret;
  1545. while ((seg = next_segment(len, offset)) != 0) {
  1546. ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
  1547. if (ret < 0)
  1548. return ret;
  1549. offset = 0;
  1550. len -= seg;
  1551. ++gfn;
  1552. }
  1553. return 0;
  1554. }
  1555. EXPORT_SYMBOL_GPL(kvm_clear_guest);
  1556. static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot,
  1557. gfn_t gfn)
  1558. {
  1559. if (memslot && memslot->dirty_bitmap) {
  1560. unsigned long rel_gfn = gfn - memslot->base_gfn;
  1561. set_bit_le(rel_gfn, memslot->dirty_bitmap);
  1562. }
  1563. }
  1564. void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
  1565. {
  1566. struct kvm_memory_slot *memslot;
  1567. memslot = gfn_to_memslot(kvm, gfn);
  1568. mark_page_dirty_in_slot(memslot, gfn);
  1569. }
  1570. EXPORT_SYMBOL_GPL(mark_page_dirty);
  1571. void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
  1572. {
  1573. struct kvm_memory_slot *memslot;
  1574. memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
  1575. mark_page_dirty_in_slot(memslot, gfn);
  1576. }
  1577. EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);
  1578. static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
  1579. {
  1580. if (kvm_arch_vcpu_runnable(vcpu)) {
  1581. kvm_make_request(KVM_REQ_UNHALT, vcpu);
  1582. return -EINTR;
  1583. }
  1584. if (kvm_cpu_has_pending_timer(vcpu))
  1585. return -EINTR;
  1586. if (signal_pending(current))
  1587. return -EINTR;
  1588. return 0;
  1589. }
  1590. /*
  1591. * The vCPU has executed a HLT instruction with in-kernel mode enabled.
  1592. */
  1593. void kvm_vcpu_block(struct kvm_vcpu *vcpu)
  1594. {
  1595. ktime_t start, cur;
  1596. DEFINE_WAIT(wait);
  1597. bool waited = false;
  1598. start = cur = ktime_get();
  1599. if (halt_poll_ns) {
  1600. ktime_t stop = ktime_add_ns(ktime_get(), halt_poll_ns);
  1601. do {
  1602. /*
  1603. * This sets KVM_REQ_UNHALT if an interrupt
  1604. * arrives.
  1605. */
  1606. if (kvm_vcpu_check_block(vcpu) < 0) {
  1607. ++vcpu->stat.halt_successful_poll;
  1608. goto out;
  1609. }
  1610. cur = ktime_get();
  1611. } while (single_task_running() && ktime_before(cur, stop));
  1612. }
  1613. for (;;) {
  1614. prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
  1615. if (kvm_vcpu_check_block(vcpu) < 0)
  1616. break;
  1617. waited = true;
  1618. schedule();
  1619. }
  1620. finish_wait(&vcpu->wq, &wait);
  1621. cur = ktime_get();
  1622. out:
  1623. trace_kvm_vcpu_wakeup(ktime_to_ns(cur) - ktime_to_ns(start), waited);
  1624. }
  1625. EXPORT_SYMBOL_GPL(kvm_vcpu_block);
  1626. #ifndef CONFIG_S390
  1627. /*
  1628. * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
  1629. */
  1630. void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
  1631. {
  1632. int me;
  1633. int cpu = vcpu->cpu;
  1634. wait_queue_head_t *wqp;
  1635. wqp = kvm_arch_vcpu_wq(vcpu);
  1636. if (waitqueue_active(wqp)) {
  1637. wake_up_interruptible(wqp);
  1638. ++vcpu->stat.halt_wakeup;
  1639. }
  1640. me = get_cpu();
  1641. if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
  1642. if (kvm_arch_vcpu_should_kick(vcpu))
  1643. smp_send_reschedule(cpu);
  1644. put_cpu();
  1645. }
  1646. EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
  1647. #endif /* !CONFIG_S390 */
  1648. int kvm_vcpu_yield_to(struct kvm_vcpu *target)
  1649. {
  1650. struct pid *pid;
  1651. struct task_struct *task = NULL;
  1652. int ret = 0;
  1653. rcu_read_lock();
  1654. pid = rcu_dereference(target->pid);
  1655. if (pid)
  1656. task = get_pid_task(pid, PIDTYPE_PID);
  1657. rcu_read_unlock();
  1658. if (!task)
  1659. return ret;
  1660. ret = yield_to(task, 1);
  1661. put_task_struct(task);
  1662. return ret;
  1663. }
  1664. EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
  1665. /*
  1666. * Helper that checks whether a VCPU is eligible for directed yield.
  1667. * Most eligible candidate to yield is decided by following heuristics:
  1668. *
  1669. * (a) VCPU which has not done pl-exit or cpu relax intercepted recently
  1670. * (preempted lock holder), indicated by @in_spin_loop.
  1671. * Set at the beiginning and cleared at the end of interception/PLE handler.
  1672. *
  1673. * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
  1674. * chance last time (mostly it has become eligible now since we have probably
  1675. * yielded to lockholder in last iteration. This is done by toggling
  1676. * @dy_eligible each time a VCPU checked for eligibility.)
  1677. *
  1678. * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
  1679. * to preempted lock-holder could result in wrong VCPU selection and CPU
  1680. * burning. Giving priority for a potential lock-holder increases lock
  1681. * progress.
  1682. *
  1683. * Since algorithm is based on heuristics, accessing another VCPU data without
  1684. * locking does not harm. It may result in trying to yield to same VCPU, fail
  1685. * and continue with next VCPU and so on.
  1686. */
  1687. static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
  1688. {
  1689. #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
  1690. bool eligible;
  1691. eligible = !vcpu->spin_loop.in_spin_loop ||
  1692. vcpu->spin_loop.dy_eligible;
  1693. if (vcpu->spin_loop.in_spin_loop)
  1694. kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
  1695. return eligible;
  1696. #else
  1697. return true;
  1698. #endif
  1699. }
  1700. void kvm_vcpu_on_spin(struct kvm_vcpu *me)
  1701. {
  1702. struct kvm *kvm = me->kvm;
  1703. struct kvm_vcpu *vcpu;
  1704. int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
  1705. int yielded = 0;
  1706. int try = 3;
  1707. int pass;
  1708. int i;
  1709. kvm_vcpu_set_in_spin_loop(me, true);
  1710. /*
  1711. * We boost the priority of a VCPU that is runnable but not
  1712. * currently running, because it got preempted by something
  1713. * else and called schedule in __vcpu_run. Hopefully that
  1714. * VCPU is holding the lock that we need and will release it.
  1715. * We approximate round-robin by starting at the last boosted VCPU.
  1716. */
  1717. for (pass = 0; pass < 2 && !yielded && try; pass++) {
  1718. kvm_for_each_vcpu(i, vcpu, kvm) {
  1719. if (!pass && i <= last_boosted_vcpu) {
  1720. i = last_boosted_vcpu;
  1721. continue;
  1722. } else if (pass && i > last_boosted_vcpu)
  1723. break;
  1724. if (!ACCESS_ONCE(vcpu->preempted))
  1725. continue;
  1726. if (vcpu == me)
  1727. continue;
  1728. if (waitqueue_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu))
  1729. continue;
  1730. if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
  1731. continue;
  1732. yielded = kvm_vcpu_yield_to(vcpu);
  1733. if (yielded > 0) {
  1734. kvm->last_boosted_vcpu = i;
  1735. break;
  1736. } else if (yielded < 0) {
  1737. try--;
  1738. if (!try)
  1739. break;
  1740. }
  1741. }
  1742. }
  1743. kvm_vcpu_set_in_spin_loop(me, false);
  1744. /* Ensure vcpu is not eligible during next spinloop */
  1745. kvm_vcpu_set_dy_eligible(me, false);
  1746. }
  1747. EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
  1748. static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  1749. {
  1750. struct kvm_vcpu *vcpu = vma->vm_file->private_data;
  1751. struct page *page;
  1752. if (vmf->pgoff == 0)
  1753. page = virt_to_page(vcpu->run);
  1754. #ifdef CONFIG_X86
  1755. else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
  1756. page = virt_to_page(vcpu->arch.pio_data);
  1757. #endif
  1758. #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  1759. else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
  1760. page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
  1761. #endif
  1762. else
  1763. return kvm_arch_vcpu_fault(vcpu, vmf);
  1764. get_page(page);
  1765. vmf->page = page;
  1766. return 0;
  1767. }
  1768. static const struct vm_operations_struct kvm_vcpu_vm_ops = {
  1769. .fault = kvm_vcpu_fault,
  1770. };
  1771. static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
  1772. {
  1773. vma->vm_ops = &kvm_vcpu_vm_ops;
  1774. return 0;
  1775. }
  1776. static int kvm_vcpu_release(struct inode *inode, struct file *filp)
  1777. {
  1778. struct kvm_vcpu *vcpu = filp->private_data;
  1779. kvm_put_kvm(vcpu->kvm);
  1780. return 0;
  1781. }
  1782. static struct file_operations kvm_vcpu_fops = {
  1783. .release = kvm_vcpu_release,
  1784. .unlocked_ioctl = kvm_vcpu_ioctl,
  1785. #ifdef CONFIG_KVM_COMPAT
  1786. .compat_ioctl = kvm_vcpu_compat_ioctl,
  1787. #endif
  1788. .mmap = kvm_vcpu_mmap,
  1789. .llseek = noop_llseek,
  1790. };
  1791. /*
  1792. * Allocates an inode for the vcpu.
  1793. */
  1794. static int create_vcpu_fd(struct kvm_vcpu *vcpu)
  1795. {
  1796. return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
  1797. }
  1798. /*
  1799. * Creates some virtual cpus. Good luck creating more than one.
  1800. */
  1801. static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
  1802. {
  1803. int r;
  1804. struct kvm_vcpu *vcpu, *v;
  1805. if (id >= KVM_MAX_VCPUS)
  1806. return -EINVAL;
  1807. vcpu = kvm_arch_vcpu_create(kvm, id);
  1808. if (IS_ERR(vcpu))
  1809. return PTR_ERR(vcpu);
  1810. preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
  1811. r = kvm_arch_vcpu_setup(vcpu);
  1812. if (r)
  1813. goto vcpu_destroy;
  1814. mutex_lock(&kvm->lock);
  1815. if (!kvm_vcpu_compatible(vcpu)) {
  1816. r = -EINVAL;
  1817. goto unlock_vcpu_destroy;
  1818. }
  1819. if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
  1820. r = -EINVAL;
  1821. goto unlock_vcpu_destroy;
  1822. }
  1823. kvm_for_each_vcpu(r, v, kvm)
  1824. if (v->vcpu_id == id) {
  1825. r = -EEXIST;
  1826. goto unlock_vcpu_destroy;
  1827. }
  1828. BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
  1829. /* Now it's all set up, let userspace reach it */
  1830. kvm_get_kvm(kvm);
  1831. r = create_vcpu_fd(vcpu);
  1832. if (r < 0) {
  1833. kvm_put_kvm(kvm);
  1834. goto unlock_vcpu_destroy;
  1835. }
  1836. kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
  1837. smp_wmb();
  1838. atomic_inc(&kvm->online_vcpus);
  1839. mutex_unlock(&kvm->lock);
  1840. kvm_arch_vcpu_postcreate(vcpu);
  1841. return r;
  1842. unlock_vcpu_destroy:
  1843. mutex_unlock(&kvm->lock);
  1844. vcpu_destroy:
  1845. kvm_arch_vcpu_destroy(vcpu);
  1846. return r;
  1847. }
  1848. static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
  1849. {
  1850. if (sigset) {
  1851. sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
  1852. vcpu->sigset_active = 1;
  1853. vcpu->sigset = *sigset;
  1854. } else
  1855. vcpu->sigset_active = 0;
  1856. return 0;
  1857. }
  1858. static long kvm_vcpu_ioctl(struct file *filp,
  1859. unsigned int ioctl, unsigned long arg)
  1860. {
  1861. struct kvm_vcpu *vcpu = filp->private_data;
  1862. void __user *argp = (void __user *)arg;
  1863. int r;
  1864. struct kvm_fpu *fpu = NULL;
  1865. struct kvm_sregs *kvm_sregs = NULL;
  1866. if (vcpu->kvm->mm != current->mm)
  1867. return -EIO;
  1868. if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
  1869. return -EINVAL;
  1870. #if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS)
  1871. /*
  1872. * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
  1873. * so vcpu_load() would break it.
  1874. */
  1875. if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_S390_IRQ || ioctl == KVM_INTERRUPT)
  1876. return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
  1877. #endif
  1878. r = vcpu_load(vcpu);
  1879. if (r)
  1880. return r;
  1881. switch (ioctl) {
  1882. case KVM_RUN:
  1883. r = -EINVAL;
  1884. if (arg)
  1885. goto out;
  1886. if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
  1887. /* The thread running this VCPU changed. */
  1888. struct pid *oldpid = vcpu->pid;
  1889. struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
  1890. rcu_assign_pointer(vcpu->pid, newpid);
  1891. if (oldpid)
  1892. synchronize_rcu();
  1893. put_pid(oldpid);
  1894. }
  1895. r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
  1896. trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
  1897. break;
  1898. case KVM_GET_REGS: {
  1899. struct kvm_regs *kvm_regs;
  1900. r = -ENOMEM;
  1901. kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
  1902. if (!kvm_regs)
  1903. goto out;
  1904. r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
  1905. if (r)
  1906. goto out_free1;
  1907. r = -EFAULT;
  1908. if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
  1909. goto out_free1;
  1910. r = 0;
  1911. out_free1:
  1912. kfree(kvm_regs);
  1913. break;
  1914. }
  1915. case KVM_SET_REGS: {
  1916. struct kvm_regs *kvm_regs;
  1917. r = -ENOMEM;
  1918. kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
  1919. if (IS_ERR(kvm_regs)) {
  1920. r = PTR_ERR(kvm_regs);
  1921. goto out;
  1922. }
  1923. r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
  1924. kfree(kvm_regs);
  1925. break;
  1926. }
  1927. case KVM_GET_SREGS: {
  1928. kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
  1929. r = -ENOMEM;
  1930. if (!kvm_sregs)
  1931. goto out;
  1932. r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
  1933. if (r)
  1934. goto out;
  1935. r = -EFAULT;
  1936. if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
  1937. goto out;
  1938. r = 0;
  1939. break;
  1940. }
  1941. case KVM_SET_SREGS: {
  1942. kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
  1943. if (IS_ERR(kvm_sregs)) {
  1944. r = PTR_ERR(kvm_sregs);
  1945. kvm_sregs = NULL;
  1946. goto out;
  1947. }
  1948. r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
  1949. break;
  1950. }
  1951. case KVM_GET_MP_STATE: {
  1952. struct kvm_mp_state mp_state;
  1953. r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
  1954. if (r)
  1955. goto out;
  1956. r = -EFAULT;
  1957. if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
  1958. goto out;
  1959. r = 0;
  1960. break;
  1961. }
  1962. case KVM_SET_MP_STATE: {
  1963. struct kvm_mp_state mp_state;
  1964. r = -EFAULT;
  1965. if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
  1966. goto out;
  1967. r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
  1968. break;
  1969. }
  1970. case KVM_TRANSLATE: {
  1971. struct kvm_translation tr;
  1972. r = -EFAULT;
  1973. if (copy_from_user(&tr, argp, sizeof(tr)))
  1974. goto out;
  1975. r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
  1976. if (r)
  1977. goto out;
  1978. r = -EFAULT;
  1979. if (copy_to_user(argp, &tr, sizeof(tr)))
  1980. goto out;
  1981. r = 0;
  1982. break;
  1983. }
  1984. case KVM_SET_GUEST_DEBUG: {
  1985. struct kvm_guest_debug dbg;
  1986. r = -EFAULT;
  1987. if (copy_from_user(&dbg, argp, sizeof(dbg)))
  1988. goto out;
  1989. r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
  1990. break;
  1991. }
  1992. case KVM_SET_SIGNAL_MASK: {
  1993. struct kvm_signal_mask __user *sigmask_arg = argp;
  1994. struct kvm_signal_mask kvm_sigmask;
  1995. sigset_t sigset, *p;
  1996. p = NULL;
  1997. if (argp) {
  1998. r = -EFAULT;
  1999. if (copy_from_user(&kvm_sigmask, argp,
  2000. sizeof(kvm_sigmask)))
  2001. goto out;
  2002. r = -EINVAL;
  2003. if (kvm_sigmask.len != sizeof(sigset))
  2004. goto out;
  2005. r = -EFAULT;
  2006. if (copy_from_user(&sigset, sigmask_arg->sigset,
  2007. sizeof(sigset)))
  2008. goto out;
  2009. p = &sigset;
  2010. }
  2011. r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
  2012. break;
  2013. }
  2014. case KVM_GET_FPU: {
  2015. fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
  2016. r = -ENOMEM;
  2017. if (!fpu)
  2018. goto out;
  2019. r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
  2020. if (r)
  2021. goto out;
  2022. r = -EFAULT;
  2023. if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
  2024. goto out;
  2025. r = 0;
  2026. break;
  2027. }
  2028. case KVM_SET_FPU: {
  2029. fpu = memdup_user(argp, sizeof(*fpu));
  2030. if (IS_ERR(fpu)) {
  2031. r = PTR_ERR(fpu);
  2032. fpu = NULL;
  2033. goto out;
  2034. }
  2035. r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
  2036. break;
  2037. }
  2038. default:
  2039. r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
  2040. }
  2041. out:
  2042. vcpu_put(vcpu);
  2043. kfree(fpu);
  2044. kfree(kvm_sregs);
  2045. return r;
  2046. }
  2047. #ifdef CONFIG_KVM_COMPAT
  2048. static long kvm_vcpu_compat_ioctl(struct file *filp,
  2049. unsigned int ioctl, unsigned long arg)
  2050. {
  2051. struct kvm_vcpu *vcpu = filp->private_data;
  2052. void __user *argp = compat_ptr(arg);
  2053. int r;
  2054. if (vcpu->kvm->mm != current->mm)
  2055. return -EIO;
  2056. switch (ioctl) {
  2057. case KVM_SET_SIGNAL_MASK: {
  2058. struct kvm_signal_mask __user *sigmask_arg = argp;
  2059. struct kvm_signal_mask kvm_sigmask;
  2060. compat_sigset_t csigset;
  2061. sigset_t sigset;
  2062. if (argp) {
  2063. r = -EFAULT;
  2064. if (copy_from_user(&kvm_sigmask, argp,
  2065. sizeof(kvm_sigmask)))
  2066. goto out;
  2067. r = -EINVAL;
  2068. if (kvm_sigmask.len != sizeof(csigset))
  2069. goto out;
  2070. r = -EFAULT;
  2071. if (copy_from_user(&csigset, sigmask_arg->sigset,
  2072. sizeof(csigset)))
  2073. goto out;
  2074. sigset_from_compat(&sigset, &csigset);
  2075. r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
  2076. } else
  2077. r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
  2078. break;
  2079. }
  2080. default:
  2081. r = kvm_vcpu_ioctl(filp, ioctl, arg);
  2082. }
  2083. out:
  2084. return r;
  2085. }
  2086. #endif
  2087. static int kvm_device_ioctl_attr(struct kvm_device *dev,
  2088. int (*accessor)(struct kvm_device *dev,
  2089. struct kvm_device_attr *attr),
  2090. unsigned long arg)
  2091. {
  2092. struct kvm_device_attr attr;
  2093. if (!accessor)
  2094. return -EPERM;
  2095. if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
  2096. return -EFAULT;
  2097. return accessor(dev, &attr);
  2098. }
  2099. static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
  2100. unsigned long arg)
  2101. {
  2102. struct kvm_device *dev = filp->private_data;
  2103. switch (ioctl) {
  2104. case KVM_SET_DEVICE_ATTR:
  2105. return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
  2106. case KVM_GET_DEVICE_ATTR:
  2107. return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
  2108. case KVM_HAS_DEVICE_ATTR:
  2109. return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
  2110. default:
  2111. if (dev->ops->ioctl)
  2112. return dev->ops->ioctl(dev, ioctl, arg);
  2113. return -ENOTTY;
  2114. }
  2115. }
  2116. static int kvm_device_release(struct inode *inode, struct file *filp)
  2117. {
  2118. struct kvm_device *dev = filp->private_data;
  2119. struct kvm *kvm = dev->kvm;
  2120. kvm_put_kvm(kvm);
  2121. return 0;
  2122. }
  2123. static const struct file_operations kvm_device_fops = {
  2124. .unlocked_ioctl = kvm_device_ioctl,
  2125. #ifdef CONFIG_KVM_COMPAT
  2126. .compat_ioctl = kvm_device_ioctl,
  2127. #endif
  2128. .release = kvm_device_release,
  2129. };
  2130. struct kvm_device *kvm_device_from_filp(struct file *filp)
  2131. {
  2132. if (filp->f_op != &kvm_device_fops)
  2133. return NULL;
  2134. return filp->private_data;
  2135. }
  2136. static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
  2137. #ifdef CONFIG_KVM_MPIC
  2138. [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops,
  2139. [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops,
  2140. #endif
  2141. #ifdef CONFIG_KVM_XICS
  2142. [KVM_DEV_TYPE_XICS] = &kvm_xics_ops,
  2143. #endif
  2144. };
  2145. int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type)
  2146. {
  2147. if (type >= ARRAY_SIZE(kvm_device_ops_table))
  2148. return -ENOSPC;
  2149. if (kvm_device_ops_table[type] != NULL)
  2150. return -EEXIST;
  2151. kvm_device_ops_table[type] = ops;
  2152. return 0;
  2153. }
  2154. void kvm_unregister_device_ops(u32 type)
  2155. {
  2156. if (kvm_device_ops_table[type] != NULL)
  2157. kvm_device_ops_table[type] = NULL;
  2158. }
  2159. static int kvm_ioctl_create_device(struct kvm *kvm,
  2160. struct kvm_create_device *cd)
  2161. {
  2162. struct kvm_device_ops *ops = NULL;
  2163. struct kvm_device *dev;
  2164. bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
  2165. int ret;
  2166. if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
  2167. return -ENODEV;
  2168. ops = kvm_device_ops_table[cd->type];
  2169. if (ops == NULL)
  2170. return -ENODEV;
  2171. if (test)
  2172. return 0;
  2173. dev = kzalloc(sizeof(*dev), GFP_KERNEL);
  2174. if (!dev)
  2175. return -ENOMEM;
  2176. dev->ops = ops;
  2177. dev->kvm = kvm;
  2178. ret = ops->create(dev, cd->type);
  2179. if (ret < 0) {
  2180. kfree(dev);
  2181. return ret;
  2182. }
  2183. ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
  2184. if (ret < 0) {
  2185. ops->destroy(dev);
  2186. return ret;
  2187. }
  2188. list_add(&dev->vm_node, &kvm->devices);
  2189. kvm_get_kvm(kvm);
  2190. cd->fd = ret;
  2191. return 0;
  2192. }
  2193. static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
  2194. {
  2195. switch (arg) {
  2196. case KVM_CAP_USER_MEMORY:
  2197. case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
  2198. case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
  2199. #ifdef CONFIG_KVM_APIC_ARCHITECTURE
  2200. case KVM_CAP_SET_BOOT_CPU_ID:
  2201. #endif
  2202. case KVM_CAP_INTERNAL_ERROR_DATA:
  2203. #ifdef CONFIG_HAVE_KVM_MSI
  2204. case KVM_CAP_SIGNAL_MSI:
  2205. #endif
  2206. #ifdef CONFIG_HAVE_KVM_IRQFD
  2207. case KVM_CAP_IRQFD:
  2208. case KVM_CAP_IRQFD_RESAMPLE:
  2209. #endif
  2210. case KVM_CAP_CHECK_EXTENSION_VM:
  2211. return 1;
  2212. #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
  2213. case KVM_CAP_IRQ_ROUTING:
  2214. return KVM_MAX_IRQ_ROUTES;
  2215. #endif
  2216. #if KVM_ADDRESS_SPACE_NUM > 1
  2217. case KVM_CAP_MULTI_ADDRESS_SPACE:
  2218. return KVM_ADDRESS_SPACE_NUM;
  2219. #endif
  2220. default:
  2221. break;
  2222. }
  2223. return kvm_vm_ioctl_check_extension(kvm, arg);
  2224. }
  2225. static long kvm_vm_ioctl(struct file *filp,
  2226. unsigned int ioctl, unsigned long arg)
  2227. {
  2228. struct kvm *kvm = filp->private_data;
  2229. void __user *argp = (void __user *)arg;
  2230. int r;
  2231. if (kvm->mm != current->mm)
  2232. return -EIO;
  2233. switch (ioctl) {
  2234. case KVM_CREATE_VCPU:
  2235. r = kvm_vm_ioctl_create_vcpu(kvm, arg);
  2236. break;
  2237. case KVM_SET_USER_MEMORY_REGION: {
  2238. struct kvm_userspace_memory_region kvm_userspace_mem;
  2239. r = -EFAULT;
  2240. if (copy_from_user(&kvm_userspace_mem, argp,
  2241. sizeof(kvm_userspace_mem)))
  2242. goto out;
  2243. r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
  2244. break;
  2245. }
  2246. case KVM_GET_DIRTY_LOG: {
  2247. struct kvm_dirty_log log;
  2248. r = -EFAULT;
  2249. if (copy_from_user(&log, argp, sizeof(log)))
  2250. goto out;
  2251. r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
  2252. break;
  2253. }
  2254. #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  2255. case KVM_REGISTER_COALESCED_MMIO: {
  2256. struct kvm_coalesced_mmio_zone zone;
  2257. r = -EFAULT;
  2258. if (copy_from_user(&zone, argp, sizeof(zone)))
  2259. goto out;
  2260. r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
  2261. break;
  2262. }
  2263. case KVM_UNREGISTER_COALESCED_MMIO: {
  2264. struct kvm_coalesced_mmio_zone zone;
  2265. r = -EFAULT;
  2266. if (copy_from_user(&zone, argp, sizeof(zone)))
  2267. goto out;
  2268. r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
  2269. break;
  2270. }
  2271. #endif
  2272. case KVM_IRQFD: {
  2273. struct kvm_irqfd data;
  2274. r = -EFAULT;
  2275. if (copy_from_user(&data, argp, sizeof(data)))
  2276. goto out;
  2277. r = kvm_irqfd(kvm, &data);
  2278. break;
  2279. }
  2280. case KVM_IOEVENTFD: {
  2281. struct kvm_ioeventfd data;
  2282. r = -EFAULT;
  2283. if (copy_from_user(&data, argp, sizeof(data)))
  2284. goto out;
  2285. r = kvm_ioeventfd(kvm, &data);
  2286. break;
  2287. }
  2288. #ifdef CONFIG_KVM_APIC_ARCHITECTURE
  2289. case KVM_SET_BOOT_CPU_ID:
  2290. r = 0;
  2291. mutex_lock(&kvm->lock);
  2292. if (atomic_read(&kvm->online_vcpus) != 0)
  2293. r = -EBUSY;
  2294. else
  2295. kvm->bsp_vcpu_id = arg;
  2296. mutex_unlock(&kvm->lock);
  2297. break;
  2298. #endif
  2299. #ifdef CONFIG_HAVE_KVM_MSI
  2300. case KVM_SIGNAL_MSI: {
  2301. struct kvm_msi msi;
  2302. r = -EFAULT;
  2303. if (copy_from_user(&msi, argp, sizeof(msi)))
  2304. goto out;
  2305. r = kvm_send_userspace_msi(kvm, &msi);
  2306. break;
  2307. }
  2308. #endif
  2309. #ifdef __KVM_HAVE_IRQ_LINE
  2310. case KVM_IRQ_LINE_STATUS:
  2311. case KVM_IRQ_LINE: {
  2312. struct kvm_irq_level irq_event;
  2313. r = -EFAULT;
  2314. if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
  2315. goto out;
  2316. r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
  2317. ioctl == KVM_IRQ_LINE_STATUS);
  2318. if (r)
  2319. goto out;
  2320. r = -EFAULT;
  2321. if (ioctl == KVM_IRQ_LINE_STATUS) {
  2322. if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
  2323. goto out;
  2324. }
  2325. r = 0;
  2326. break;
  2327. }
  2328. #endif
  2329. #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
  2330. case KVM_SET_GSI_ROUTING: {
  2331. struct kvm_irq_routing routing;
  2332. struct kvm_irq_routing __user *urouting;
  2333. struct kvm_irq_routing_entry *entries;
  2334. r = -EFAULT;
  2335. if (copy_from_user(&routing, argp, sizeof(routing)))
  2336. goto out;
  2337. r = -EINVAL;
  2338. if (routing.nr >= KVM_MAX_IRQ_ROUTES)
  2339. goto out;
  2340. if (routing.flags)
  2341. goto out;
  2342. r = -ENOMEM;
  2343. entries = vmalloc(routing.nr * sizeof(*entries));
  2344. if (!entries)
  2345. goto out;
  2346. r = -EFAULT;
  2347. urouting = argp;
  2348. if (copy_from_user(entries, urouting->entries,
  2349. routing.nr * sizeof(*entries)))
  2350. goto out_free_irq_routing;
  2351. r = kvm_set_irq_routing(kvm, entries, routing.nr,
  2352. routing.flags);
  2353. out_free_irq_routing:
  2354. vfree(entries);
  2355. break;
  2356. }
  2357. #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
  2358. case KVM_CREATE_DEVICE: {
  2359. struct kvm_create_device cd;
  2360. r = -EFAULT;
  2361. if (copy_from_user(&cd, argp, sizeof(cd)))
  2362. goto out;
  2363. r = kvm_ioctl_create_device(kvm, &cd);
  2364. if (r)
  2365. goto out;
  2366. r = -EFAULT;
  2367. if (copy_to_user(argp, &cd, sizeof(cd)))
  2368. goto out;
  2369. r = 0;
  2370. break;
  2371. }
  2372. case KVM_CHECK_EXTENSION:
  2373. r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
  2374. break;
  2375. default:
  2376. r = kvm_arch_vm_ioctl(filp, ioctl, arg);
  2377. }
  2378. out:
  2379. return r;
  2380. }
  2381. #ifdef CONFIG_KVM_COMPAT
  2382. struct compat_kvm_dirty_log {
  2383. __u32 slot;
  2384. __u32 padding1;
  2385. union {
  2386. compat_uptr_t dirty_bitmap; /* one bit per page */
  2387. __u64 padding2;
  2388. };
  2389. };
  2390. static long kvm_vm_compat_ioctl(struct file *filp,
  2391. unsigned int ioctl, unsigned long arg)
  2392. {
  2393. struct kvm *kvm = filp->private_data;
  2394. int r;
  2395. if (kvm->mm != current->mm)
  2396. return -EIO;
  2397. switch (ioctl) {
  2398. case KVM_GET_DIRTY_LOG: {
  2399. struct compat_kvm_dirty_log compat_log;
  2400. struct kvm_dirty_log log;
  2401. r = -EFAULT;
  2402. if (copy_from_user(&compat_log, (void __user *)arg,
  2403. sizeof(compat_log)))
  2404. goto out;
  2405. log.slot = compat_log.slot;
  2406. log.padding1 = compat_log.padding1;
  2407. log.padding2 = compat_log.padding2;
  2408. log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
  2409. r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
  2410. break;
  2411. }
  2412. default:
  2413. r = kvm_vm_ioctl(filp, ioctl, arg);
  2414. }
  2415. out:
  2416. return r;
  2417. }
  2418. #endif
  2419. static struct file_operations kvm_vm_fops = {
  2420. .release = kvm_vm_release,
  2421. .unlocked_ioctl = kvm_vm_ioctl,
  2422. #ifdef CONFIG_KVM_COMPAT
  2423. .compat_ioctl = kvm_vm_compat_ioctl,
  2424. #endif
  2425. .llseek = noop_llseek,
  2426. };
  2427. static int kvm_dev_ioctl_create_vm(unsigned long type)
  2428. {
  2429. int r;
  2430. struct kvm *kvm;
  2431. kvm = kvm_create_vm(type);
  2432. if (IS_ERR(kvm))
  2433. return PTR_ERR(kvm);
  2434. #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  2435. r = kvm_coalesced_mmio_init(kvm);
  2436. if (r < 0) {
  2437. kvm_put_kvm(kvm);
  2438. return r;
  2439. }
  2440. #endif
  2441. r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR | O_CLOEXEC);
  2442. if (r < 0)
  2443. kvm_put_kvm(kvm);
  2444. return r;
  2445. }
  2446. static long kvm_dev_ioctl(struct file *filp,
  2447. unsigned int ioctl, unsigned long arg)
  2448. {
  2449. long r = -EINVAL;
  2450. switch (ioctl) {
  2451. case KVM_GET_API_VERSION:
  2452. if (arg)
  2453. goto out;
  2454. r = KVM_API_VERSION;
  2455. break;
  2456. case KVM_CREATE_VM:
  2457. r = kvm_dev_ioctl_create_vm(arg);
  2458. break;
  2459. case KVM_CHECK_EXTENSION:
  2460. r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
  2461. break;
  2462. case KVM_GET_VCPU_MMAP_SIZE:
  2463. if (arg)
  2464. goto out;
  2465. r = PAGE_SIZE; /* struct kvm_run */
  2466. #ifdef CONFIG_X86
  2467. r += PAGE_SIZE; /* pio data page */
  2468. #endif
  2469. #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  2470. r += PAGE_SIZE; /* coalesced mmio ring page */
  2471. #endif
  2472. break;
  2473. case KVM_TRACE_ENABLE:
  2474. case KVM_TRACE_PAUSE:
  2475. case KVM_TRACE_DISABLE:
  2476. r = -EOPNOTSUPP;
  2477. break;
  2478. default:
  2479. return kvm_arch_dev_ioctl(filp, ioctl, arg);
  2480. }
  2481. out:
  2482. return r;
  2483. }
  2484. static struct file_operations kvm_chardev_ops = {
  2485. .unlocked_ioctl = kvm_dev_ioctl,
  2486. .compat_ioctl = kvm_dev_ioctl,
  2487. .llseek = noop_llseek,
  2488. };
  2489. static struct miscdevice kvm_dev = {
  2490. KVM_MINOR,
  2491. "kvm",
  2492. &kvm_chardev_ops,
  2493. };
  2494. static void hardware_enable_nolock(void *junk)
  2495. {
  2496. int cpu = raw_smp_processor_id();
  2497. int r;
  2498. if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
  2499. return;
  2500. cpumask_set_cpu(cpu, cpus_hardware_enabled);
  2501. r = kvm_arch_hardware_enable();
  2502. if (r) {
  2503. cpumask_clear_cpu(cpu, cpus_hardware_enabled);
  2504. atomic_inc(&hardware_enable_failed);
  2505. pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu);
  2506. }
  2507. }
  2508. static void hardware_enable(void)
  2509. {
  2510. raw_spin_lock(&kvm_count_lock);
  2511. if (kvm_usage_count)
  2512. hardware_enable_nolock(NULL);
  2513. raw_spin_unlock(&kvm_count_lock);
  2514. }
  2515. static void hardware_disable_nolock(void *junk)
  2516. {
  2517. int cpu = raw_smp_processor_id();
  2518. if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
  2519. return;
  2520. cpumask_clear_cpu(cpu, cpus_hardware_enabled);
  2521. kvm_arch_hardware_disable();
  2522. }
  2523. static void hardware_disable(void)
  2524. {
  2525. raw_spin_lock(&kvm_count_lock);
  2526. if (kvm_usage_count)
  2527. hardware_disable_nolock(NULL);
  2528. raw_spin_unlock(&kvm_count_lock);
  2529. }
  2530. static void hardware_disable_all_nolock(void)
  2531. {
  2532. BUG_ON(!kvm_usage_count);
  2533. kvm_usage_count--;
  2534. if (!kvm_usage_count)
  2535. on_each_cpu(hardware_disable_nolock, NULL, 1);
  2536. }
  2537. static void hardware_disable_all(void)
  2538. {
  2539. raw_spin_lock(&kvm_count_lock);
  2540. hardware_disable_all_nolock();
  2541. raw_spin_unlock(&kvm_count_lock);
  2542. }
  2543. static int hardware_enable_all(void)
  2544. {
  2545. int r = 0;
  2546. raw_spin_lock(&kvm_count_lock);
  2547. kvm_usage_count++;
  2548. if (kvm_usage_count == 1) {
  2549. atomic_set(&hardware_enable_failed, 0);
  2550. on_each_cpu(hardware_enable_nolock, NULL, 1);
  2551. if (atomic_read(&hardware_enable_failed)) {
  2552. hardware_disable_all_nolock();
  2553. r = -EBUSY;
  2554. }
  2555. }
  2556. raw_spin_unlock(&kvm_count_lock);
  2557. return r;
  2558. }
  2559. static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
  2560. void *v)
  2561. {
  2562. val &= ~CPU_TASKS_FROZEN;
  2563. switch (val) {
  2564. case CPU_DYING:
  2565. hardware_disable();
  2566. break;
  2567. case CPU_STARTING:
  2568. hardware_enable();
  2569. break;
  2570. }
  2571. return NOTIFY_OK;
  2572. }
  2573. static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
  2574. void *v)
  2575. {
  2576. /*
  2577. * Some (well, at least mine) BIOSes hang on reboot if
  2578. * in vmx root mode.
  2579. *
  2580. * And Intel TXT required VMX off for all cpu when system shutdown.
  2581. */
  2582. pr_info("kvm: exiting hardware virtualization\n");
  2583. kvm_rebooting = true;
  2584. on_each_cpu(hardware_disable_nolock, NULL, 1);
  2585. return NOTIFY_OK;
  2586. }
  2587. static struct notifier_block kvm_reboot_notifier = {
  2588. .notifier_call = kvm_reboot,
  2589. .priority = 0,
  2590. };
  2591. static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
  2592. {
  2593. int i;
  2594. for (i = 0; i < bus->dev_count; i++) {
  2595. struct kvm_io_device *pos = bus->range[i].dev;
  2596. kvm_iodevice_destructor(pos);
  2597. }
  2598. kfree(bus);
  2599. }
  2600. static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
  2601. const struct kvm_io_range *r2)
  2602. {
  2603. if (r1->addr < r2->addr)
  2604. return -1;
  2605. if (r1->addr + r1->len > r2->addr + r2->len)
  2606. return 1;
  2607. return 0;
  2608. }
  2609. static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
  2610. {
  2611. return kvm_io_bus_cmp(p1, p2);
  2612. }
  2613. static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev,
  2614. gpa_t addr, int len)
  2615. {
  2616. bus->range[bus->dev_count++] = (struct kvm_io_range) {
  2617. .addr = addr,
  2618. .len = len,
  2619. .dev = dev,
  2620. };
  2621. sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range),
  2622. kvm_io_bus_sort_cmp, NULL);
  2623. return 0;
  2624. }
  2625. static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
  2626. gpa_t addr, int len)
  2627. {
  2628. struct kvm_io_range *range, key;
  2629. int off;
  2630. key = (struct kvm_io_range) {
  2631. .addr = addr,
  2632. .len = len,
  2633. };
  2634. range = bsearch(&key, bus->range, bus->dev_count,
  2635. sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
  2636. if (range == NULL)
  2637. return -ENOENT;
  2638. off = range - bus->range;
  2639. while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
  2640. off--;
  2641. return off;
  2642. }
  2643. static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
  2644. struct kvm_io_range *range, const void *val)
  2645. {
  2646. int idx;
  2647. idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
  2648. if (idx < 0)
  2649. return -EOPNOTSUPP;
  2650. while (idx < bus->dev_count &&
  2651. kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
  2652. if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
  2653. range->len, val))
  2654. return idx;
  2655. idx++;
  2656. }
  2657. return -EOPNOTSUPP;
  2658. }
  2659. /* kvm_io_bus_write - called under kvm->slots_lock */
  2660. int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
  2661. int len, const void *val)
  2662. {
  2663. struct kvm_io_bus *bus;
  2664. struct kvm_io_range range;
  2665. int r;
  2666. range = (struct kvm_io_range) {
  2667. .addr = addr,
  2668. .len = len,
  2669. };
  2670. bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
  2671. r = __kvm_io_bus_write(vcpu, bus, &range, val);
  2672. return r < 0 ? r : 0;
  2673. }
  2674. /* kvm_io_bus_write_cookie - called under kvm->slots_lock */
  2675. int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
  2676. gpa_t addr, int len, const void *val, long cookie)
  2677. {
  2678. struct kvm_io_bus *bus;
  2679. struct kvm_io_range range;
  2680. range = (struct kvm_io_range) {
  2681. .addr = addr,
  2682. .len = len,
  2683. };
  2684. bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
  2685. /* First try the device referenced by cookie. */
  2686. if ((cookie >= 0) && (cookie < bus->dev_count) &&
  2687. (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
  2688. if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
  2689. val))
  2690. return cookie;
  2691. /*
  2692. * cookie contained garbage; fall back to search and return the
  2693. * correct cookie value.
  2694. */
  2695. return __kvm_io_bus_write(vcpu, bus, &range, val);
  2696. }
  2697. static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
  2698. struct kvm_io_range *range, void *val)
  2699. {
  2700. int idx;
  2701. idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
  2702. if (idx < 0)
  2703. return -EOPNOTSUPP;
  2704. while (idx < bus->dev_count &&
  2705. kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
  2706. if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
  2707. range->len, val))
  2708. return idx;
  2709. idx++;
  2710. }
  2711. return -EOPNOTSUPP;
  2712. }
  2713. EXPORT_SYMBOL_GPL(kvm_io_bus_write);
  2714. /* kvm_io_bus_read - called under kvm->slots_lock */
  2715. int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
  2716. int len, void *val)
  2717. {
  2718. struct kvm_io_bus *bus;
  2719. struct kvm_io_range range;
  2720. int r;
  2721. range = (struct kvm_io_range) {
  2722. .addr = addr,
  2723. .len = len,
  2724. };
  2725. bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
  2726. r = __kvm_io_bus_read(vcpu, bus, &range, val);
  2727. return r < 0 ? r : 0;
  2728. }
  2729. /* Caller must hold slots_lock. */
  2730. int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
  2731. int len, struct kvm_io_device *dev)
  2732. {
  2733. struct kvm_io_bus *new_bus, *bus;
  2734. bus = kvm->buses[bus_idx];
  2735. /* exclude ioeventfd which is limited by maximum fd */
  2736. if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
  2737. return -ENOSPC;
  2738. new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count + 1) *
  2739. sizeof(struct kvm_io_range)), GFP_KERNEL);
  2740. if (!new_bus)
  2741. return -ENOMEM;
  2742. memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count *
  2743. sizeof(struct kvm_io_range)));
  2744. kvm_io_bus_insert_dev(new_bus, dev, addr, len);
  2745. rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
  2746. synchronize_srcu_expedited(&kvm->srcu);
  2747. kfree(bus);
  2748. return 0;
  2749. }
  2750. /* Caller must hold slots_lock. */
  2751. int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
  2752. struct kvm_io_device *dev)
  2753. {
  2754. int i, r;
  2755. struct kvm_io_bus *new_bus, *bus;
  2756. bus = kvm->buses[bus_idx];
  2757. r = -ENOENT;
  2758. for (i = 0; i < bus->dev_count; i++)
  2759. if (bus->range[i].dev == dev) {
  2760. r = 0;
  2761. break;
  2762. }
  2763. if (r)
  2764. return r;
  2765. new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count - 1) *
  2766. sizeof(struct kvm_io_range)), GFP_KERNEL);
  2767. if (!new_bus)
  2768. return -ENOMEM;
  2769. memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
  2770. new_bus->dev_count--;
  2771. memcpy(new_bus->range + i, bus->range + i + 1,
  2772. (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
  2773. rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
  2774. synchronize_srcu_expedited(&kvm->srcu);
  2775. kfree(bus);
  2776. return r;
  2777. }
  2778. static struct notifier_block kvm_cpu_notifier = {
  2779. .notifier_call = kvm_cpu_hotplug,
  2780. };
  2781. static int vm_stat_get(void *_offset, u64 *val)
  2782. {
  2783. unsigned offset = (long)_offset;
  2784. struct kvm *kvm;
  2785. *val = 0;
  2786. spin_lock(&kvm_lock);
  2787. list_for_each_entry(kvm, &vm_list, vm_list)
  2788. *val += *(u32 *)((void *)kvm + offset);
  2789. spin_unlock(&kvm_lock);
  2790. return 0;
  2791. }
  2792. DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
  2793. static int vcpu_stat_get(void *_offset, u64 *val)
  2794. {
  2795. unsigned offset = (long)_offset;
  2796. struct kvm *kvm;
  2797. struct kvm_vcpu *vcpu;
  2798. int i;
  2799. *val = 0;
  2800. spin_lock(&kvm_lock);
  2801. list_for_each_entry(kvm, &vm_list, vm_list)
  2802. kvm_for_each_vcpu(i, vcpu, kvm)
  2803. *val += *(u32 *)((void *)vcpu + offset);
  2804. spin_unlock(&kvm_lock);
  2805. return 0;
  2806. }
  2807. DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
  2808. static const struct file_operations *stat_fops[] = {
  2809. [KVM_STAT_VCPU] = &vcpu_stat_fops,
  2810. [KVM_STAT_VM] = &vm_stat_fops,
  2811. };
  2812. static int kvm_init_debug(void)
  2813. {
  2814. int r = -EEXIST;
  2815. struct kvm_stats_debugfs_item *p;
  2816. kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
  2817. if (kvm_debugfs_dir == NULL)
  2818. goto out;
  2819. for (p = debugfs_entries; p->name; ++p) {
  2820. p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
  2821. (void *)(long)p->offset,
  2822. stat_fops[p->kind]);
  2823. if (p->dentry == NULL)
  2824. goto out_dir;
  2825. }
  2826. return 0;
  2827. out_dir:
  2828. debugfs_remove_recursive(kvm_debugfs_dir);
  2829. out:
  2830. return r;
  2831. }
  2832. static void kvm_exit_debug(void)
  2833. {
  2834. struct kvm_stats_debugfs_item *p;
  2835. for (p = debugfs_entries; p->name; ++p)
  2836. debugfs_remove(p->dentry);
  2837. debugfs_remove(kvm_debugfs_dir);
  2838. }
  2839. static int kvm_suspend(void)
  2840. {
  2841. if (kvm_usage_count)
  2842. hardware_disable_nolock(NULL);
  2843. return 0;
  2844. }
  2845. static void kvm_resume(void)
  2846. {
  2847. if (kvm_usage_count) {
  2848. WARN_ON(raw_spin_is_locked(&kvm_count_lock));
  2849. hardware_enable_nolock(NULL);
  2850. }
  2851. }
  2852. static struct syscore_ops kvm_syscore_ops = {
  2853. .suspend = kvm_suspend,
  2854. .resume = kvm_resume,
  2855. };
  2856. static inline
  2857. struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
  2858. {
  2859. return container_of(pn, struct kvm_vcpu, preempt_notifier);
  2860. }
  2861. static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
  2862. {
  2863. struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
  2864. if (vcpu->preempted)
  2865. vcpu->preempted = false;
  2866. kvm_arch_sched_in(vcpu, cpu);
  2867. kvm_arch_vcpu_load(vcpu, cpu);
  2868. }
  2869. static void kvm_sched_out(struct preempt_notifier *pn,
  2870. struct task_struct *next)
  2871. {
  2872. struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
  2873. if (current->state == TASK_RUNNING)
  2874. vcpu->preempted = true;
  2875. kvm_arch_vcpu_put(vcpu);
  2876. }
  2877. int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
  2878. struct module *module)
  2879. {
  2880. int r;
  2881. int cpu;
  2882. r = kvm_arch_init(opaque);
  2883. if (r)
  2884. goto out_fail;
  2885. /*
  2886. * kvm_arch_init makes sure there's at most one caller
  2887. * for architectures that support multiple implementations,
  2888. * like intel and amd on x86.
  2889. * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
  2890. * conflicts in case kvm is already setup for another implementation.
  2891. */
  2892. r = kvm_irqfd_init();
  2893. if (r)
  2894. goto out_irqfd;
  2895. if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
  2896. r = -ENOMEM;
  2897. goto out_free_0;
  2898. }
  2899. r = kvm_arch_hardware_setup();
  2900. if (r < 0)
  2901. goto out_free_0a;
  2902. for_each_online_cpu(cpu) {
  2903. smp_call_function_single(cpu,
  2904. kvm_arch_check_processor_compat,
  2905. &r, 1);
  2906. if (r < 0)
  2907. goto out_free_1;
  2908. }
  2909. r = register_cpu_notifier(&kvm_cpu_notifier);
  2910. if (r)
  2911. goto out_free_2;
  2912. register_reboot_notifier(&kvm_reboot_notifier);
  2913. /* A kmem cache lets us meet the alignment requirements of fx_save. */
  2914. if (!vcpu_align)
  2915. vcpu_align = __alignof__(struct kvm_vcpu);
  2916. kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
  2917. 0, NULL);
  2918. if (!kvm_vcpu_cache) {
  2919. r = -ENOMEM;
  2920. goto out_free_3;
  2921. }
  2922. r = kvm_async_pf_init();
  2923. if (r)
  2924. goto out_free;
  2925. kvm_chardev_ops.owner = module;
  2926. kvm_vm_fops.owner = module;
  2927. kvm_vcpu_fops.owner = module;
  2928. r = misc_register(&kvm_dev);
  2929. if (r) {
  2930. pr_err("kvm: misc device register failed\n");
  2931. goto out_unreg;
  2932. }
  2933. register_syscore_ops(&kvm_syscore_ops);
  2934. kvm_preempt_ops.sched_in = kvm_sched_in;
  2935. kvm_preempt_ops.sched_out = kvm_sched_out;
  2936. r = kvm_init_debug();
  2937. if (r) {
  2938. pr_err("kvm: create debugfs files failed\n");
  2939. goto out_undebugfs;
  2940. }
  2941. r = kvm_vfio_ops_init();
  2942. WARN_ON(r);
  2943. return 0;
  2944. out_undebugfs:
  2945. unregister_syscore_ops(&kvm_syscore_ops);
  2946. misc_deregister(&kvm_dev);
  2947. out_unreg:
  2948. kvm_async_pf_deinit();
  2949. out_free:
  2950. kmem_cache_destroy(kvm_vcpu_cache);
  2951. out_free_3:
  2952. unregister_reboot_notifier(&kvm_reboot_notifier);
  2953. unregister_cpu_notifier(&kvm_cpu_notifier);
  2954. out_free_2:
  2955. out_free_1:
  2956. kvm_arch_hardware_unsetup();
  2957. out_free_0a:
  2958. free_cpumask_var(cpus_hardware_enabled);
  2959. out_free_0:
  2960. kvm_irqfd_exit();
  2961. out_irqfd:
  2962. kvm_arch_exit();
  2963. out_fail:
  2964. return r;
  2965. }
  2966. EXPORT_SYMBOL_GPL(kvm_init);
  2967. void kvm_exit(void)
  2968. {
  2969. kvm_exit_debug();
  2970. misc_deregister(&kvm_dev);
  2971. kmem_cache_destroy(kvm_vcpu_cache);
  2972. kvm_async_pf_deinit();
  2973. unregister_syscore_ops(&kvm_syscore_ops);
  2974. unregister_reboot_notifier(&kvm_reboot_notifier);
  2975. unregister_cpu_notifier(&kvm_cpu_notifier);
  2976. on_each_cpu(hardware_disable_nolock, NULL, 1);
  2977. kvm_arch_hardware_unsetup();
  2978. kvm_arch_exit();
  2979. kvm_irqfd_exit();
  2980. free_cpumask_var(cpus_hardware_enabled);
  2981. kvm_vfio_ops_exit();
  2982. }
  2983. EXPORT_SYMBOL_GPL(kvm_exit);