12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369 |
- #include <linux/errno.h>
- #include <linux/mm.h>
- #include <linux/fs.h>
- #include <linux/mman.h>
- #include <linux/sched.h>
- #include <linux/rwsem.h>
- #include <linux/pagemap.h>
- #include <linux/rmap.h>
- #include <linux/spinlock.h>
- #include <linux/jhash.h>
- #include <linux/delay.h>
- #include <linux/kthread.h>
- #include <linux/wait.h>
- #include <linux/slab.h>
- #include <linux/rbtree.h>
- #include <linux/memory.h>
- #include <linux/mmu_notifier.h>
- #include <linux/swap.h>
- #include <linux/ksm.h>
- #include <linux/hashtable.h>
- #include <linux/freezer.h>
- #include <linux/oom.h>
- #include <linux/numa.h>
- #include <asm/tlbflush.h>
- #include "internal.h"
- #ifdef CONFIG_NUMA
- #define NUMA(x) (x)
- #define DO_NUMA(x) do { (x); } while (0)
- #else
- #define NUMA(x) (0)
- #define DO_NUMA(x) do { } while (0)
- #endif
- struct mm_slot {
- struct hlist_node link;
- struct list_head mm_list;
- struct rmap_item *rmap_list;
- struct mm_struct *mm;
- };
- struct ksm_scan {
- struct mm_slot *mm_slot;
- unsigned long address;
- struct rmap_item **rmap_list;
- unsigned long seqnr;
- };
- struct stable_node {
- union {
- struct rb_node node;
- struct {
- struct list_head *head;
- struct list_head list;
- };
- };
- struct hlist_head hlist;
- unsigned long kpfn;
- #ifdef CONFIG_NUMA
- int nid;
- #endif
- };
- struct rmap_item {
- struct rmap_item *rmap_list;
- union {
- struct anon_vma *anon_vma;
- #ifdef CONFIG_NUMA
- int nid;
- #endif
- };
- struct mm_struct *mm;
- unsigned long address;
- unsigned int oldchecksum;
- union {
- struct rb_node node;
- struct {
- struct stable_node *head;
- struct hlist_node hlist;
- };
- };
- };
- #define SEQNR_MASK 0x0ff
- #define UNSTABLE_FLAG 0x100
- #define STABLE_FLAG 0x200
- static struct rb_root one_stable_tree[1] = { RB_ROOT };
- static struct rb_root one_unstable_tree[1] = { RB_ROOT };
- static struct rb_root *root_stable_tree = one_stable_tree;
- static struct rb_root *root_unstable_tree = one_unstable_tree;
- static LIST_HEAD(migrate_nodes);
- #define MM_SLOTS_HASH_BITS 10
- static DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
- static struct mm_slot ksm_mm_head = {
- .mm_list = LIST_HEAD_INIT(ksm_mm_head.mm_list),
- };
- static struct ksm_scan ksm_scan = {
- .mm_slot = &ksm_mm_head,
- };
- static struct kmem_cache *rmap_item_cache;
- static struct kmem_cache *stable_node_cache;
- static struct kmem_cache *mm_slot_cache;
- static unsigned long ksm_pages_shared;
- static unsigned long ksm_pages_sharing;
- static unsigned long ksm_pages_unshared;
- static unsigned long ksm_rmap_items;
- static unsigned int ksm_thread_pages_to_scan = 100;
- static unsigned int ksm_thread_sleep_millisecs = 20;
- #ifdef CONFIG_NUMA
- static unsigned int ksm_merge_across_nodes = 1;
- static int ksm_nr_node_ids = 1;
- #else
- #define ksm_merge_across_nodes 1U
- #define ksm_nr_node_ids 1
- #endif
- #define KSM_RUN_STOP 0
- #define KSM_RUN_MERGE 1
- #define KSM_RUN_UNMERGE 2
- #define KSM_RUN_OFFLINE 4
- static unsigned long ksm_run = KSM_RUN_STOP;
- static void wait_while_offlining(void);
- static DECLARE_WAIT_QUEUE_HEAD(ksm_thread_wait);
- static DEFINE_MUTEX(ksm_thread_mutex);
- static DEFINE_SPINLOCK(ksm_mmlist_lock);
- #define KSM_KMEM_CACHE(__struct, __flags) kmem_cache_create("ksm_"#__struct,\
- sizeof(struct __struct), __alignof__(struct __struct),\
- (__flags), NULL)
- static int __init ksm_slab_init(void)
- {
- rmap_item_cache = KSM_KMEM_CACHE(rmap_item, 0);
- if (!rmap_item_cache)
- goto out;
- stable_node_cache = KSM_KMEM_CACHE(stable_node, 0);
- if (!stable_node_cache)
- goto out_free1;
- mm_slot_cache = KSM_KMEM_CACHE(mm_slot, 0);
- if (!mm_slot_cache)
- goto out_free2;
- return 0;
- out_free2:
- kmem_cache_destroy(stable_node_cache);
- out_free1:
- kmem_cache_destroy(rmap_item_cache);
- out:
- return -ENOMEM;
- }
- static void __init ksm_slab_free(void)
- {
- kmem_cache_destroy(mm_slot_cache);
- kmem_cache_destroy(stable_node_cache);
- kmem_cache_destroy(rmap_item_cache);
- mm_slot_cache = NULL;
- }
- static inline struct rmap_item *alloc_rmap_item(void)
- {
- struct rmap_item *rmap_item;
- rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL |
- __GFP_NORETRY | __GFP_NOWARN);
- if (rmap_item)
- ksm_rmap_items++;
- return rmap_item;
- }
- static inline void free_rmap_item(struct rmap_item *rmap_item)
- {
- ksm_rmap_items--;
- rmap_item->mm = NULL;
- kmem_cache_free(rmap_item_cache, rmap_item);
- }
- static inline struct stable_node *alloc_stable_node(void)
- {
-
- return kmem_cache_alloc(stable_node_cache, GFP_KERNEL | __GFP_HIGH);
- }
- static inline void free_stable_node(struct stable_node *stable_node)
- {
- kmem_cache_free(stable_node_cache, stable_node);
- }
- static inline struct mm_slot *alloc_mm_slot(void)
- {
- if (!mm_slot_cache)
- return NULL;
- return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL);
- }
- static inline void free_mm_slot(struct mm_slot *mm_slot)
- {
- kmem_cache_free(mm_slot_cache, mm_slot);
- }
- static struct mm_slot *get_mm_slot(struct mm_struct *mm)
- {
- struct mm_slot *slot;
- hash_for_each_possible(mm_slots_hash, slot, link, (unsigned long)mm)
- if (slot->mm == mm)
- return slot;
- return NULL;
- }
- static void insert_to_mm_slots_hash(struct mm_struct *mm,
- struct mm_slot *mm_slot)
- {
- mm_slot->mm = mm;
- hash_add(mm_slots_hash, &mm_slot->link, (unsigned long)mm);
- }
- static inline bool ksm_test_exit(struct mm_struct *mm)
- {
- return atomic_read(&mm->mm_users) == 0;
- }
- static int break_ksm(struct vm_area_struct *vma, unsigned long addr)
- {
- struct page *page;
- int ret = 0;
- do {
- cond_resched();
- page = follow_page(vma, addr,
- FOLL_GET | FOLL_MIGRATION | FOLL_REMOTE);
- if (IS_ERR_OR_NULL(page))
- break;
- if (PageKsm(page))
- ret = handle_mm_fault(vma, addr,
- FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE);
- else
- ret = VM_FAULT_WRITE;
- put_page(page);
- } while (!(ret & (VM_FAULT_WRITE | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | VM_FAULT_OOM)));
-
- return (ret & VM_FAULT_OOM) ? -ENOMEM : 0;
- }
- static struct vm_area_struct *find_mergeable_vma(struct mm_struct *mm,
- unsigned long addr)
- {
- struct vm_area_struct *vma;
- if (ksm_test_exit(mm))
- return NULL;
- vma = find_vma(mm, addr);
- if (!vma || vma->vm_start > addr)
- return NULL;
- if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
- return NULL;
- return vma;
- }
- static void break_cow(struct rmap_item *rmap_item)
- {
- struct mm_struct *mm = rmap_item->mm;
- unsigned long addr = rmap_item->address;
- struct vm_area_struct *vma;
-
- put_anon_vma(rmap_item->anon_vma);
- down_read(&mm->mmap_sem);
- vma = find_mergeable_vma(mm, addr);
- if (vma)
- break_ksm(vma, addr);
- up_read(&mm->mmap_sem);
- }
- static struct page *get_mergeable_page(struct rmap_item *rmap_item)
- {
- struct mm_struct *mm = rmap_item->mm;
- unsigned long addr = rmap_item->address;
- struct vm_area_struct *vma;
- struct page *page;
- down_read(&mm->mmap_sem);
- vma = find_mergeable_vma(mm, addr);
- if (!vma)
- goto out;
- page = follow_page(vma, addr, FOLL_GET);
- if (IS_ERR_OR_NULL(page))
- goto out;
- if (PageAnon(page)) {
- flush_anon_page(vma, page, addr);
- flush_dcache_page(page);
- } else {
- put_page(page);
- out:
- page = NULL;
- }
- up_read(&mm->mmap_sem);
- return page;
- }
- static inline int get_kpfn_nid(unsigned long kpfn)
- {
- return ksm_merge_across_nodes ? 0 : NUMA(pfn_to_nid(kpfn));
- }
- static void remove_node_from_stable_tree(struct stable_node *stable_node)
- {
- struct rmap_item *rmap_item;
- hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
- if (rmap_item->hlist.next)
- ksm_pages_sharing--;
- else
- ksm_pages_shared--;
- put_anon_vma(rmap_item->anon_vma);
- rmap_item->address &= PAGE_MASK;
- cond_resched();
- }
- if (stable_node->head == &migrate_nodes)
- list_del(&stable_node->list);
- else
- rb_erase(&stable_node->node,
- root_stable_tree + NUMA(stable_node->nid));
- free_stable_node(stable_node);
- }
- static struct page *get_ksm_page(struct stable_node *stable_node, bool lock_it)
- {
- struct page *page;
- void *expected_mapping;
- unsigned long kpfn;
- expected_mapping = (void *)((unsigned long)stable_node |
- PAGE_MAPPING_KSM);
- again:
- kpfn = READ_ONCE(stable_node->kpfn);
- page = pfn_to_page(kpfn);
-
- smp_read_barrier_depends();
- if (READ_ONCE(page->mapping) != expected_mapping)
- goto stale;
-
- while (!get_page_unless_zero(page)) {
-
- if (!PageSwapCache(page))
- goto stale;
- cpu_relax();
- }
- if (READ_ONCE(page->mapping) != expected_mapping) {
- put_page(page);
- goto stale;
- }
- if (lock_it) {
- lock_page(page);
- if (READ_ONCE(page->mapping) != expected_mapping) {
- unlock_page(page);
- put_page(page);
- goto stale;
- }
- }
- return page;
- stale:
-
- smp_rmb();
- if (READ_ONCE(stable_node->kpfn) != kpfn)
- goto again;
- remove_node_from_stable_tree(stable_node);
- return NULL;
- }
- static void remove_rmap_item_from_tree(struct rmap_item *rmap_item)
- {
- if (rmap_item->address & STABLE_FLAG) {
- struct stable_node *stable_node;
- struct page *page;
- stable_node = rmap_item->head;
- page = get_ksm_page(stable_node, true);
- if (!page)
- goto out;
- hlist_del(&rmap_item->hlist);
- unlock_page(page);
- put_page(page);
- if (!hlist_empty(&stable_node->hlist))
- ksm_pages_sharing--;
- else
- ksm_pages_shared--;
- put_anon_vma(rmap_item->anon_vma);
- rmap_item->address &= PAGE_MASK;
- } else if (rmap_item->address & UNSTABLE_FLAG) {
- unsigned char age;
-
- age = (unsigned char)(ksm_scan.seqnr - rmap_item->address);
- BUG_ON(age > 1);
- if (!age)
- rb_erase(&rmap_item->node,
- root_unstable_tree + NUMA(rmap_item->nid));
- ksm_pages_unshared--;
- rmap_item->address &= PAGE_MASK;
- }
- out:
- cond_resched();
- }
- static void remove_trailing_rmap_items(struct mm_slot *mm_slot,
- struct rmap_item **rmap_list)
- {
- while (*rmap_list) {
- struct rmap_item *rmap_item = *rmap_list;
- *rmap_list = rmap_item->rmap_list;
- remove_rmap_item_from_tree(rmap_item);
- free_rmap_item(rmap_item);
- }
- }
- static int unmerge_ksm_pages(struct vm_area_struct *vma,
- unsigned long start, unsigned long end)
- {
- unsigned long addr;
- int err = 0;
- for (addr = start; addr < end && !err; addr += PAGE_SIZE) {
- if (ksm_test_exit(vma->vm_mm))
- break;
- if (signal_pending(current))
- err = -ERESTARTSYS;
- else
- err = break_ksm(vma, addr);
- }
- return err;
- }
- #ifdef CONFIG_SYSFS
- static int remove_stable_node(struct stable_node *stable_node)
- {
- struct page *page;
- int err;
- page = get_ksm_page(stable_node, true);
- if (!page) {
-
- return 0;
- }
- if (WARN_ON_ONCE(page_mapped(page))) {
-
- err = -EBUSY;
- } else {
-
- set_page_stable_node(page, NULL);
- remove_node_from_stable_tree(stable_node);
- err = 0;
- }
- unlock_page(page);
- put_page(page);
- return err;
- }
- static int remove_all_stable_nodes(void)
- {
- struct stable_node *stable_node, *next;
- int nid;
- int err = 0;
- for (nid = 0; nid < ksm_nr_node_ids; nid++) {
- while (root_stable_tree[nid].rb_node) {
- stable_node = rb_entry(root_stable_tree[nid].rb_node,
- struct stable_node, node);
- if (remove_stable_node(stable_node)) {
- err = -EBUSY;
- break;
- }
- cond_resched();
- }
- }
- list_for_each_entry_safe(stable_node, next, &migrate_nodes, list) {
- if (remove_stable_node(stable_node))
- err = -EBUSY;
- cond_resched();
- }
- return err;
- }
- static int unmerge_and_remove_all_rmap_items(void)
- {
- struct mm_slot *mm_slot;
- struct mm_struct *mm;
- struct vm_area_struct *vma;
- int err = 0;
- spin_lock(&ksm_mmlist_lock);
- ksm_scan.mm_slot = list_entry(ksm_mm_head.mm_list.next,
- struct mm_slot, mm_list);
- spin_unlock(&ksm_mmlist_lock);
- for (mm_slot = ksm_scan.mm_slot;
- mm_slot != &ksm_mm_head; mm_slot = ksm_scan.mm_slot) {
- mm = mm_slot->mm;
- down_read(&mm->mmap_sem);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
- if (ksm_test_exit(mm))
- break;
- if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
- continue;
- err = unmerge_ksm_pages(vma,
- vma->vm_start, vma->vm_end);
- if (err)
- goto error;
- }
- remove_trailing_rmap_items(mm_slot, &mm_slot->rmap_list);
- up_read(&mm->mmap_sem);
- spin_lock(&ksm_mmlist_lock);
- ksm_scan.mm_slot = list_entry(mm_slot->mm_list.next,
- struct mm_slot, mm_list);
- if (ksm_test_exit(mm)) {
- hash_del(&mm_slot->link);
- list_del(&mm_slot->mm_list);
- spin_unlock(&ksm_mmlist_lock);
- free_mm_slot(mm_slot);
- clear_bit(MMF_VM_MERGEABLE, &mm->flags);
- mmdrop(mm);
- } else
- spin_unlock(&ksm_mmlist_lock);
- }
-
- remove_all_stable_nodes();
- ksm_scan.seqnr = 0;
- return 0;
- error:
- up_read(&mm->mmap_sem);
- spin_lock(&ksm_mmlist_lock);
- ksm_scan.mm_slot = &ksm_mm_head;
- spin_unlock(&ksm_mmlist_lock);
- return err;
- }
- #endif
- static u32 calc_checksum(struct page *page)
- {
- u32 checksum;
- void *addr = kmap_atomic(page);
- checksum = jhash2(addr, PAGE_SIZE / 4, 17);
- kunmap_atomic(addr);
- return checksum;
- }
- static int memcmp_pages(struct page *page1, struct page *page2)
- {
- char *addr1, *addr2;
- int ret;
- addr1 = kmap_atomic(page1);
- addr2 = kmap_atomic(page2);
- ret = memcmp(addr1, addr2, PAGE_SIZE);
- kunmap_atomic(addr2);
- kunmap_atomic(addr1);
- return ret;
- }
- static inline int pages_identical(struct page *page1, struct page *page2)
- {
- return !memcmp_pages(page1, page2);
- }
- static int write_protect_page(struct vm_area_struct *vma, struct page *page,
- pte_t *orig_pte)
- {
- struct mm_struct *mm = vma->vm_mm;
- unsigned long addr;
- pte_t *ptep;
- spinlock_t *ptl;
- int swapped;
- int err = -EFAULT;
- unsigned long mmun_start;
- unsigned long mmun_end;
- addr = page_address_in_vma(page, vma);
- if (addr == -EFAULT)
- goto out;
- BUG_ON(PageTransCompound(page));
- mmun_start = addr;
- mmun_end = addr + PAGE_SIZE;
- mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
- ptep = page_check_address(page, mm, addr, &ptl, 0);
- if (!ptep)
- goto out_mn;
- if (pte_write(*ptep) || pte_dirty(*ptep)) {
- pte_t entry;
- swapped = PageSwapCache(page);
- flush_cache_page(vma, addr, page_to_pfn(page));
-
- entry = ptep_clear_flush_notify(vma, addr, ptep);
-
- if (page_mapcount(page) + 1 + swapped != page_count(page)) {
- set_pte_at(mm, addr, ptep, entry);
- goto out_unlock;
- }
- if (pte_dirty(entry))
- set_page_dirty(page);
- entry = pte_mkclean(pte_wrprotect(entry));
- set_pte_at_notify(mm, addr, ptep, entry);
- }
- *orig_pte = *ptep;
- err = 0;
- out_unlock:
- pte_unmap_unlock(ptep, ptl);
- out_mn:
- mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
- out:
- return err;
- }
- static int replace_page(struct vm_area_struct *vma, struct page *page,
- struct page *kpage, pte_t orig_pte)
- {
- struct mm_struct *mm = vma->vm_mm;
- pmd_t *pmd;
- pte_t *ptep;
- spinlock_t *ptl;
- unsigned long addr;
- int err = -EFAULT;
- unsigned long mmun_start;
- unsigned long mmun_end;
- addr = page_address_in_vma(page, vma);
- if (addr == -EFAULT)
- goto out;
- pmd = mm_find_pmd(mm, addr);
- if (!pmd)
- goto out;
- mmun_start = addr;
- mmun_end = addr + PAGE_SIZE;
- mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
- ptep = pte_offset_map_lock(mm, pmd, addr, &ptl);
- if (!pte_same(*ptep, orig_pte)) {
- pte_unmap_unlock(ptep, ptl);
- goto out_mn;
- }
- get_page(kpage);
- page_add_anon_rmap(kpage, vma, addr, false);
- flush_cache_page(vma, addr, pte_pfn(*ptep));
- ptep_clear_flush_notify(vma, addr, ptep);
- set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));
- page_remove_rmap(page, false);
- if (!page_mapped(page))
- try_to_free_swap(page);
- put_page(page);
- pte_unmap_unlock(ptep, ptl);
- err = 0;
- out_mn:
- mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
- out:
- return err;
- }
- static int try_to_merge_one_page(struct vm_area_struct *vma,
- struct page *page, struct page *kpage)
- {
- pte_t orig_pte = __pte(0);
- int err = -EFAULT;
- if (page == kpage)
- return 0;
- if (!PageAnon(page))
- goto out;
-
- if (!trylock_page(page))
- goto out;
- if (PageTransCompound(page)) {
- if (split_huge_page(page))
- goto out_unlock;
- }
-
- if (write_protect_page(vma, page, &orig_pte) == 0) {
- if (!kpage) {
-
- set_page_stable_node(page, NULL);
- mark_page_accessed(page);
-
- if (!PageDirty(page))
- SetPageDirty(page);
- err = 0;
- } else if (pages_identical(page, kpage))
- err = replace_page(vma, page, kpage, orig_pte);
- }
- if ((vma->vm_flags & VM_LOCKED) && kpage && !err) {
- munlock_vma_page(page);
- if (!PageMlocked(kpage)) {
- unlock_page(page);
- lock_page(kpage);
- mlock_vma_page(kpage);
- page = kpage;
- }
- }
- out_unlock:
- unlock_page(page);
- out:
- return err;
- }
- static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item,
- struct page *page, struct page *kpage)
- {
- struct mm_struct *mm = rmap_item->mm;
- struct vm_area_struct *vma;
- int err = -EFAULT;
- down_read(&mm->mmap_sem);
- vma = find_mergeable_vma(mm, rmap_item->address);
- if (!vma)
- goto out;
- err = try_to_merge_one_page(vma, page, kpage);
- if (err)
- goto out;
-
- remove_rmap_item_from_tree(rmap_item);
-
- rmap_item->anon_vma = vma->anon_vma;
- get_anon_vma(vma->anon_vma);
- out:
- up_read(&mm->mmap_sem);
- return err;
- }
- static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item,
- struct page *page,
- struct rmap_item *tree_rmap_item,
- struct page *tree_page)
- {
- int err;
- err = try_to_merge_with_ksm_page(rmap_item, page, NULL);
- if (!err) {
- err = try_to_merge_with_ksm_page(tree_rmap_item,
- tree_page, page);
-
- if (err)
- break_cow(rmap_item);
- }
- return err ? NULL : page;
- }
- static struct page *stable_tree_search(struct page *page)
- {
- int nid;
- struct rb_root *root;
- struct rb_node **new;
- struct rb_node *parent;
- struct stable_node *stable_node;
- struct stable_node *page_node;
- page_node = page_stable_node(page);
- if (page_node && page_node->head != &migrate_nodes) {
-
- get_page(page);
- return page;
- }
- nid = get_kpfn_nid(page_to_pfn(page));
- root = root_stable_tree + nid;
- again:
- new = &root->rb_node;
- parent = NULL;
- while (*new) {
- struct page *tree_page;
- int ret;
- cond_resched();
- stable_node = rb_entry(*new, struct stable_node, node);
- tree_page = get_ksm_page(stable_node, false);
- if (!tree_page) {
-
- goto again;
- }
- ret = memcmp_pages(page, tree_page);
- put_page(tree_page);
- parent = *new;
- if (ret < 0)
- new = &parent->rb_left;
- else if (ret > 0)
- new = &parent->rb_right;
- else {
-
- tree_page = get_ksm_page(stable_node, true);
- if (tree_page) {
- unlock_page(tree_page);
- if (get_kpfn_nid(stable_node->kpfn) !=
- NUMA(stable_node->nid)) {
- put_page(tree_page);
- goto replace;
- }
- return tree_page;
- }
-
- if (page_node)
- goto again;
- return NULL;
- }
- }
- if (!page_node)
- return NULL;
- list_del(&page_node->list);
- DO_NUMA(page_node->nid = nid);
- rb_link_node(&page_node->node, parent, new);
- rb_insert_color(&page_node->node, root);
- get_page(page);
- return page;
- replace:
- if (page_node) {
- list_del(&page_node->list);
- DO_NUMA(page_node->nid = nid);
- rb_replace_node(&stable_node->node, &page_node->node, root);
- get_page(page);
- } else {
- rb_erase(&stable_node->node, root);
- page = NULL;
- }
- stable_node->head = &migrate_nodes;
- list_add(&stable_node->list, stable_node->head);
- return page;
- }
- static struct stable_node *stable_tree_insert(struct page *kpage)
- {
- int nid;
- unsigned long kpfn;
- struct rb_root *root;
- struct rb_node **new;
- struct rb_node *parent;
- struct stable_node *stable_node;
- kpfn = page_to_pfn(kpage);
- nid = get_kpfn_nid(kpfn);
- root = root_stable_tree + nid;
- again:
- parent = NULL;
- new = &root->rb_node;
- while (*new) {
- struct page *tree_page;
- int ret;
- cond_resched();
- stable_node = rb_entry(*new, struct stable_node, node);
- tree_page = get_ksm_page(stable_node, false);
- if (!tree_page) {
-
- goto again;
- }
- ret = memcmp_pages(kpage, tree_page);
- put_page(tree_page);
- parent = *new;
- if (ret < 0)
- new = &parent->rb_left;
- else if (ret > 0)
- new = &parent->rb_right;
- else {
-
- return NULL;
- }
- }
- stable_node = alloc_stable_node();
- if (!stable_node)
- return NULL;
- INIT_HLIST_HEAD(&stable_node->hlist);
- stable_node->kpfn = kpfn;
- set_page_stable_node(kpage, stable_node);
- DO_NUMA(stable_node->nid = nid);
- rb_link_node(&stable_node->node, parent, new);
- rb_insert_color(&stable_node->node, root);
- return stable_node;
- }
- static
- struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item,
- struct page *page,
- struct page **tree_pagep)
- {
- struct rb_node **new;
- struct rb_root *root;
- struct rb_node *parent = NULL;
- int nid;
- nid = get_kpfn_nid(page_to_pfn(page));
- root = root_unstable_tree + nid;
- new = &root->rb_node;
- while (*new) {
- struct rmap_item *tree_rmap_item;
- struct page *tree_page;
- int ret;
- cond_resched();
- tree_rmap_item = rb_entry(*new, struct rmap_item, node);
- tree_page = get_mergeable_page(tree_rmap_item);
- if (!tree_page)
- return NULL;
-
- if (page == tree_page) {
- put_page(tree_page);
- return NULL;
- }
- ret = memcmp_pages(page, tree_page);
- parent = *new;
- if (ret < 0) {
- put_page(tree_page);
- new = &parent->rb_left;
- } else if (ret > 0) {
- put_page(tree_page);
- new = &parent->rb_right;
- } else if (!ksm_merge_across_nodes &&
- page_to_nid(tree_page) != nid) {
-
- put_page(tree_page);
- return NULL;
- } else {
- *tree_pagep = tree_page;
- return tree_rmap_item;
- }
- }
- rmap_item->address |= UNSTABLE_FLAG;
- rmap_item->address |= (ksm_scan.seqnr & SEQNR_MASK);
- DO_NUMA(rmap_item->nid = nid);
- rb_link_node(&rmap_item->node, parent, new);
- rb_insert_color(&rmap_item->node, root);
- ksm_pages_unshared++;
- return NULL;
- }
- static void stable_tree_append(struct rmap_item *rmap_item,
- struct stable_node *stable_node)
- {
- rmap_item->head = stable_node;
- rmap_item->address |= STABLE_FLAG;
- hlist_add_head(&rmap_item->hlist, &stable_node->hlist);
- if (rmap_item->hlist.next)
- ksm_pages_sharing++;
- else
- ksm_pages_shared++;
- }
- static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item)
- {
- struct rmap_item *tree_rmap_item;
- struct page *tree_page = NULL;
- struct stable_node *stable_node;
- struct page *kpage;
- unsigned int checksum;
- int err;
- stable_node = page_stable_node(page);
- if (stable_node) {
- if (stable_node->head != &migrate_nodes &&
- get_kpfn_nid(stable_node->kpfn) != NUMA(stable_node->nid)) {
- rb_erase(&stable_node->node,
- root_stable_tree + NUMA(stable_node->nid));
- stable_node->head = &migrate_nodes;
- list_add(&stable_node->list, stable_node->head);
- }
- if (stable_node->head != &migrate_nodes &&
- rmap_item->head == stable_node)
- return;
- }
-
- kpage = stable_tree_search(page);
- if (kpage == page && rmap_item->head == stable_node) {
- put_page(kpage);
- return;
- }
- remove_rmap_item_from_tree(rmap_item);
- if (kpage) {
- err = try_to_merge_with_ksm_page(rmap_item, page, kpage);
- if (!err) {
-
- lock_page(kpage);
- stable_tree_append(rmap_item, page_stable_node(kpage));
- unlock_page(kpage);
- }
- put_page(kpage);
- return;
- }
-
- checksum = calc_checksum(page);
- if (rmap_item->oldchecksum != checksum) {
- rmap_item->oldchecksum = checksum;
- return;
- }
- tree_rmap_item =
- unstable_tree_search_insert(rmap_item, page, &tree_page);
- if (tree_rmap_item) {
- bool split;
- kpage = try_to_merge_two_pages(rmap_item, page,
- tree_rmap_item, tree_page);
-
- split = PageTransCompound(page)
- && compound_head(page) == compound_head(tree_page);
- put_page(tree_page);
- if (kpage) {
-
- lock_page(kpage);
- stable_node = stable_tree_insert(kpage);
- if (stable_node) {
- stable_tree_append(tree_rmap_item, stable_node);
- stable_tree_append(rmap_item, stable_node);
- }
- unlock_page(kpage);
-
- if (!stable_node) {
- break_cow(tree_rmap_item);
- break_cow(rmap_item);
- }
- } else if (split) {
-
- if (!trylock_page(page))
- return;
- split_huge_page(page);
- unlock_page(page);
- }
- }
- }
- static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot,
- struct rmap_item **rmap_list,
- unsigned long addr)
- {
- struct rmap_item *rmap_item;
- while (*rmap_list) {
- rmap_item = *rmap_list;
- if ((rmap_item->address & PAGE_MASK) == addr)
- return rmap_item;
- if (rmap_item->address > addr)
- break;
- *rmap_list = rmap_item->rmap_list;
- remove_rmap_item_from_tree(rmap_item);
- free_rmap_item(rmap_item);
- }
- rmap_item = alloc_rmap_item();
- if (rmap_item) {
-
- rmap_item->mm = mm_slot->mm;
- rmap_item->address = addr;
- rmap_item->rmap_list = *rmap_list;
- *rmap_list = rmap_item;
- }
- return rmap_item;
- }
- static struct rmap_item *scan_get_next_rmap_item(struct page **page)
- {
- struct mm_struct *mm;
- struct mm_slot *slot;
- struct vm_area_struct *vma;
- struct rmap_item *rmap_item;
- int nid;
- if (list_empty(&ksm_mm_head.mm_list))
- return NULL;
- slot = ksm_scan.mm_slot;
- if (slot == &ksm_mm_head) {
-
- lru_add_drain_all();
-
- if (!ksm_merge_across_nodes) {
- struct stable_node *stable_node, *next;
- struct page *page;
- list_for_each_entry_safe(stable_node, next,
- &migrate_nodes, list) {
- page = get_ksm_page(stable_node, false);
- if (page)
- put_page(page);
- cond_resched();
- }
- }
- for (nid = 0; nid < ksm_nr_node_ids; nid++)
- root_unstable_tree[nid] = RB_ROOT;
- spin_lock(&ksm_mmlist_lock);
- slot = list_entry(slot->mm_list.next, struct mm_slot, mm_list);
- ksm_scan.mm_slot = slot;
- spin_unlock(&ksm_mmlist_lock);
-
- if (slot == &ksm_mm_head)
- return NULL;
- next_mm:
- ksm_scan.address = 0;
- ksm_scan.rmap_list = &slot->rmap_list;
- }
- mm = slot->mm;
- down_read(&mm->mmap_sem);
- if (ksm_test_exit(mm))
- vma = NULL;
- else
- vma = find_vma(mm, ksm_scan.address);
- for (; vma; vma = vma->vm_next) {
- if (!(vma->vm_flags & VM_MERGEABLE))
- continue;
- if (ksm_scan.address < vma->vm_start)
- ksm_scan.address = vma->vm_start;
- if (!vma->anon_vma)
- ksm_scan.address = vma->vm_end;
- while (ksm_scan.address < vma->vm_end) {
- if (ksm_test_exit(mm))
- break;
- *page = follow_page(vma, ksm_scan.address, FOLL_GET);
- if (IS_ERR_OR_NULL(*page)) {
- ksm_scan.address += PAGE_SIZE;
- cond_resched();
- continue;
- }
- if (PageAnon(*page)) {
- flush_anon_page(vma, *page, ksm_scan.address);
- flush_dcache_page(*page);
- rmap_item = get_next_rmap_item(slot,
- ksm_scan.rmap_list, ksm_scan.address);
- if (rmap_item) {
- ksm_scan.rmap_list =
- &rmap_item->rmap_list;
- ksm_scan.address += PAGE_SIZE;
- } else
- put_page(*page);
- up_read(&mm->mmap_sem);
- return rmap_item;
- }
- put_page(*page);
- ksm_scan.address += PAGE_SIZE;
- cond_resched();
- }
- }
- if (ksm_test_exit(mm)) {
- ksm_scan.address = 0;
- ksm_scan.rmap_list = &slot->rmap_list;
- }
-
- remove_trailing_rmap_items(slot, ksm_scan.rmap_list);
- spin_lock(&ksm_mmlist_lock);
- ksm_scan.mm_slot = list_entry(slot->mm_list.next,
- struct mm_slot, mm_list);
- if (ksm_scan.address == 0) {
-
- hash_del(&slot->link);
- list_del(&slot->mm_list);
- spin_unlock(&ksm_mmlist_lock);
- free_mm_slot(slot);
- clear_bit(MMF_VM_MERGEABLE, &mm->flags);
- up_read(&mm->mmap_sem);
- mmdrop(mm);
- } else {
- up_read(&mm->mmap_sem);
-
- spin_unlock(&ksm_mmlist_lock);
- }
-
- slot = ksm_scan.mm_slot;
- if (slot != &ksm_mm_head)
- goto next_mm;
- ksm_scan.seqnr++;
- return NULL;
- }
- static void ksm_do_scan(unsigned int scan_npages)
- {
- struct rmap_item *rmap_item;
- struct page *uninitialized_var(page);
- while (scan_npages-- && likely(!freezing(current))) {
- cond_resched();
- rmap_item = scan_get_next_rmap_item(&page);
- if (!rmap_item)
- return;
- cmp_and_merge_page(page, rmap_item);
- put_page(page);
- }
- }
- static int ksmd_should_run(void)
- {
- return (ksm_run & KSM_RUN_MERGE) && !list_empty(&ksm_mm_head.mm_list);
- }
- static int ksm_scan_thread(void *nothing)
- {
- set_freezable();
- set_user_nice(current, 5);
- while (!kthread_should_stop()) {
- mutex_lock(&ksm_thread_mutex);
- wait_while_offlining();
- if (ksmd_should_run())
- ksm_do_scan(ksm_thread_pages_to_scan);
- mutex_unlock(&ksm_thread_mutex);
- try_to_freeze();
- if (ksmd_should_run()) {
- schedule_timeout_interruptible(
- msecs_to_jiffies(ksm_thread_sleep_millisecs));
- } else {
- wait_event_freezable(ksm_thread_wait,
- ksmd_should_run() || kthread_should_stop());
- }
- }
- return 0;
- }
- int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
- unsigned long end, int advice, unsigned long *vm_flags)
- {
- struct mm_struct *mm = vma->vm_mm;
- int err;
- switch (advice) {
- case MADV_MERGEABLE:
-
- if (*vm_flags & (VM_MERGEABLE | VM_SHARED | VM_MAYSHARE |
- VM_PFNMAP | VM_IO | VM_DONTEXPAND |
- VM_HUGETLB | VM_MIXEDMAP))
- return 0;
- #ifdef VM_SAO
- if (*vm_flags & VM_SAO)
- return 0;
- #endif
- if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) {
- err = __ksm_enter(mm);
- if (err)
- return err;
- }
- *vm_flags |= VM_MERGEABLE;
- break;
- case MADV_UNMERGEABLE:
- if (!(*vm_flags & VM_MERGEABLE))
- return 0;
- if (vma->anon_vma) {
- err = unmerge_ksm_pages(vma, start, end);
- if (err)
- return err;
- }
- *vm_flags &= ~VM_MERGEABLE;
- break;
- }
- return 0;
- }
- int __ksm_enter(struct mm_struct *mm)
- {
- struct mm_slot *mm_slot;
- int needs_wakeup;
- mm_slot = alloc_mm_slot();
- if (!mm_slot)
- return -ENOMEM;
-
- needs_wakeup = list_empty(&ksm_mm_head.mm_list);
- spin_lock(&ksm_mmlist_lock);
- insert_to_mm_slots_hash(mm, mm_slot);
-
- if (ksm_run & KSM_RUN_UNMERGE)
- list_add_tail(&mm_slot->mm_list, &ksm_mm_head.mm_list);
- else
- list_add_tail(&mm_slot->mm_list, &ksm_scan.mm_slot->mm_list);
- spin_unlock(&ksm_mmlist_lock);
- set_bit(MMF_VM_MERGEABLE, &mm->flags);
- atomic_inc(&mm->mm_count);
- if (needs_wakeup)
- wake_up_interruptible(&ksm_thread_wait);
- return 0;
- }
- void __ksm_exit(struct mm_struct *mm)
- {
- struct mm_slot *mm_slot;
- int easy_to_free = 0;
-
- spin_lock(&ksm_mmlist_lock);
- mm_slot = get_mm_slot(mm);
- if (mm_slot && ksm_scan.mm_slot != mm_slot) {
- if (!mm_slot->rmap_list) {
- hash_del(&mm_slot->link);
- list_del(&mm_slot->mm_list);
- easy_to_free = 1;
- } else {
- list_move(&mm_slot->mm_list,
- &ksm_scan.mm_slot->mm_list);
- }
- }
- spin_unlock(&ksm_mmlist_lock);
- if (easy_to_free) {
- free_mm_slot(mm_slot);
- clear_bit(MMF_VM_MERGEABLE, &mm->flags);
- mmdrop(mm);
- } else if (mm_slot) {
- down_write(&mm->mmap_sem);
- up_write(&mm->mmap_sem);
- }
- }
- struct page *ksm_might_need_to_copy(struct page *page,
- struct vm_area_struct *vma, unsigned long address)
- {
- struct anon_vma *anon_vma = page_anon_vma(page);
- struct page *new_page;
- if (PageKsm(page)) {
- if (page_stable_node(page) &&
- !(ksm_run & KSM_RUN_UNMERGE))
- return page;
- } else if (!anon_vma) {
- return page;
- } else if (anon_vma->root == vma->anon_vma->root &&
- page->index == linear_page_index(vma, address)) {
- return page;
- }
- if (!PageUptodate(page))
- return page;
- new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
- if (new_page) {
- copy_user_highpage(new_page, page, address, vma);
- SetPageDirty(new_page);
- __SetPageUptodate(new_page);
- __SetPageLocked(new_page);
- }
- return new_page;
- }
- int rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc)
- {
- struct stable_node *stable_node;
- struct rmap_item *rmap_item;
- int ret = SWAP_AGAIN;
- int search_new_forks = 0;
- VM_BUG_ON_PAGE(!PageKsm(page), page);
-
- VM_BUG_ON_PAGE(!PageLocked(page), page);
- stable_node = page_stable_node(page);
- if (!stable_node)
- return ret;
- again:
- hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
- struct anon_vma *anon_vma = rmap_item->anon_vma;
- struct anon_vma_chain *vmac;
- struct vm_area_struct *vma;
- cond_resched();
- anon_vma_lock_read(anon_vma);
- anon_vma_interval_tree_foreach(vmac, &anon_vma->rb_root,
- 0, ULONG_MAX) {
- cond_resched();
- vma = vmac->vma;
- if (rmap_item->address < vma->vm_start ||
- rmap_item->address >= vma->vm_end)
- continue;
-
- if ((rmap_item->mm == vma->vm_mm) == search_new_forks)
- continue;
- if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
- continue;
- ret = rwc->rmap_one(page, vma,
- rmap_item->address, rwc->arg);
- if (ret != SWAP_AGAIN) {
- anon_vma_unlock_read(anon_vma);
- goto out;
- }
- if (rwc->done && rwc->done(page)) {
- anon_vma_unlock_read(anon_vma);
- goto out;
- }
- }
- anon_vma_unlock_read(anon_vma);
- }
- if (!search_new_forks++)
- goto again;
- out:
- return ret;
- }
- #ifdef CONFIG_MIGRATION
- void ksm_migrate_page(struct page *newpage, struct page *oldpage)
- {
- struct stable_node *stable_node;
- VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
- VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
- VM_BUG_ON_PAGE(newpage->mapping != oldpage->mapping, newpage);
- stable_node = page_stable_node(newpage);
- if (stable_node) {
- VM_BUG_ON_PAGE(stable_node->kpfn != page_to_pfn(oldpage), oldpage);
- stable_node->kpfn = page_to_pfn(newpage);
-
- smp_wmb();
- set_page_stable_node(oldpage, NULL);
- }
- }
- #endif
- #ifdef CONFIG_MEMORY_HOTREMOVE
- static void wait_while_offlining(void)
- {
- while (ksm_run & KSM_RUN_OFFLINE) {
- mutex_unlock(&ksm_thread_mutex);
- wait_on_bit(&ksm_run, ilog2(KSM_RUN_OFFLINE),
- TASK_UNINTERRUPTIBLE);
- mutex_lock(&ksm_thread_mutex);
- }
- }
- static void ksm_check_stable_tree(unsigned long start_pfn,
- unsigned long end_pfn)
- {
- struct stable_node *stable_node, *next;
- struct rb_node *node;
- int nid;
- for (nid = 0; nid < ksm_nr_node_ids; nid++) {
- node = rb_first(root_stable_tree + nid);
- while (node) {
- stable_node = rb_entry(node, struct stable_node, node);
- if (stable_node->kpfn >= start_pfn &&
- stable_node->kpfn < end_pfn) {
-
- remove_node_from_stable_tree(stable_node);
- node = rb_first(root_stable_tree + nid);
- } else
- node = rb_next(node);
- cond_resched();
- }
- }
- list_for_each_entry_safe(stable_node, next, &migrate_nodes, list) {
- if (stable_node->kpfn >= start_pfn &&
- stable_node->kpfn < end_pfn)
- remove_node_from_stable_tree(stable_node);
- cond_resched();
- }
- }
- static int ksm_memory_callback(struct notifier_block *self,
- unsigned long action, void *arg)
- {
- struct memory_notify *mn = arg;
- switch (action) {
- case MEM_GOING_OFFLINE:
-
- mutex_lock(&ksm_thread_mutex);
- ksm_run |= KSM_RUN_OFFLINE;
- mutex_unlock(&ksm_thread_mutex);
- break;
- case MEM_OFFLINE:
-
- ksm_check_stable_tree(mn->start_pfn,
- mn->start_pfn + mn->nr_pages);
-
- case MEM_CANCEL_OFFLINE:
- mutex_lock(&ksm_thread_mutex);
- ksm_run &= ~KSM_RUN_OFFLINE;
- mutex_unlock(&ksm_thread_mutex);
- smp_mb();
- wake_up_bit(&ksm_run, ilog2(KSM_RUN_OFFLINE));
- break;
- }
- return NOTIFY_OK;
- }
- #else
- static void wait_while_offlining(void)
- {
- }
- #endif
- #ifdef CONFIG_SYSFS
- #define KSM_ATTR_RO(_name) \
- static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
- #define KSM_ATTR(_name) \
- static struct kobj_attribute _name##_attr = \
- __ATTR(_name, 0644, _name##_show, _name##_store)
- static ssize_t sleep_millisecs_show(struct kobject *kobj,
- struct kobj_attribute *attr, char *buf)
- {
- return sprintf(buf, "%u\n", ksm_thread_sleep_millisecs);
- }
- static ssize_t sleep_millisecs_store(struct kobject *kobj,
- struct kobj_attribute *attr,
- const char *buf, size_t count)
- {
- unsigned long msecs;
- int err;
- err = kstrtoul(buf, 10, &msecs);
- if (err || msecs > UINT_MAX)
- return -EINVAL;
- ksm_thread_sleep_millisecs = msecs;
- return count;
- }
- KSM_ATTR(sleep_millisecs);
- static ssize_t pages_to_scan_show(struct kobject *kobj,
- struct kobj_attribute *attr, char *buf)
- {
- return sprintf(buf, "%u\n", ksm_thread_pages_to_scan);
- }
- static ssize_t pages_to_scan_store(struct kobject *kobj,
- struct kobj_attribute *attr,
- const char *buf, size_t count)
- {
- int err;
- unsigned long nr_pages;
- err = kstrtoul(buf, 10, &nr_pages);
- if (err || nr_pages > UINT_MAX)
- return -EINVAL;
- ksm_thread_pages_to_scan = nr_pages;
- return count;
- }
- KSM_ATTR(pages_to_scan);
- static ssize_t run_show(struct kobject *kobj, struct kobj_attribute *attr,
- char *buf)
- {
- return sprintf(buf, "%lu\n", ksm_run);
- }
- static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr,
- const char *buf, size_t count)
- {
- int err;
- unsigned long flags;
- err = kstrtoul(buf, 10, &flags);
- if (err || flags > UINT_MAX)
- return -EINVAL;
- if (flags > KSM_RUN_UNMERGE)
- return -EINVAL;
-
- mutex_lock(&ksm_thread_mutex);
- wait_while_offlining();
- if (ksm_run != flags) {
- ksm_run = flags;
- if (flags & KSM_RUN_UNMERGE) {
- set_current_oom_origin();
- err = unmerge_and_remove_all_rmap_items();
- clear_current_oom_origin();
- if (err) {
- ksm_run = KSM_RUN_STOP;
- count = err;
- }
- }
- }
- mutex_unlock(&ksm_thread_mutex);
- if (flags & KSM_RUN_MERGE)
- wake_up_interruptible(&ksm_thread_wait);
- return count;
- }
- KSM_ATTR(run);
- #ifdef CONFIG_NUMA
- static ssize_t merge_across_nodes_show(struct kobject *kobj,
- struct kobj_attribute *attr, char *buf)
- {
- return sprintf(buf, "%u\n", ksm_merge_across_nodes);
- }
- static ssize_t merge_across_nodes_store(struct kobject *kobj,
- struct kobj_attribute *attr,
- const char *buf, size_t count)
- {
- int err;
- unsigned long knob;
- err = kstrtoul(buf, 10, &knob);
- if (err)
- return err;
- if (knob > 1)
- return -EINVAL;
- mutex_lock(&ksm_thread_mutex);
- wait_while_offlining();
- if (ksm_merge_across_nodes != knob) {
- if (ksm_pages_shared || remove_all_stable_nodes())
- err = -EBUSY;
- else if (root_stable_tree == one_stable_tree) {
- struct rb_root *buf;
-
- buf = kcalloc(nr_node_ids + nr_node_ids, sizeof(*buf),
- GFP_KERNEL);
-
- if (!buf)
- err = -ENOMEM;
- else {
- root_stable_tree = buf;
- root_unstable_tree = buf + nr_node_ids;
-
- root_unstable_tree[0] = one_unstable_tree[0];
- }
- }
- if (!err) {
- ksm_merge_across_nodes = knob;
- ksm_nr_node_ids = knob ? 1 : nr_node_ids;
- }
- }
- mutex_unlock(&ksm_thread_mutex);
- return err ? err : count;
- }
- KSM_ATTR(merge_across_nodes);
- #endif
- static ssize_t pages_shared_show(struct kobject *kobj,
- struct kobj_attribute *attr, char *buf)
- {
- return sprintf(buf, "%lu\n", ksm_pages_shared);
- }
- KSM_ATTR_RO(pages_shared);
- static ssize_t pages_sharing_show(struct kobject *kobj,
- struct kobj_attribute *attr, char *buf)
- {
- return sprintf(buf, "%lu\n", ksm_pages_sharing);
- }
- KSM_ATTR_RO(pages_sharing);
- static ssize_t pages_unshared_show(struct kobject *kobj,
- struct kobj_attribute *attr, char *buf)
- {
- return sprintf(buf, "%lu\n", ksm_pages_unshared);
- }
- KSM_ATTR_RO(pages_unshared);
- static ssize_t pages_volatile_show(struct kobject *kobj,
- struct kobj_attribute *attr, char *buf)
- {
- long ksm_pages_volatile;
- ksm_pages_volatile = ksm_rmap_items - ksm_pages_shared
- - ksm_pages_sharing - ksm_pages_unshared;
-
- if (ksm_pages_volatile < 0)
- ksm_pages_volatile = 0;
- return sprintf(buf, "%ld\n", ksm_pages_volatile);
- }
- KSM_ATTR_RO(pages_volatile);
- static ssize_t full_scans_show(struct kobject *kobj,
- struct kobj_attribute *attr, char *buf)
- {
- return sprintf(buf, "%lu\n", ksm_scan.seqnr);
- }
- KSM_ATTR_RO(full_scans);
- static struct attribute *ksm_attrs[] = {
- &sleep_millisecs_attr.attr,
- &pages_to_scan_attr.attr,
- &run_attr.attr,
- &pages_shared_attr.attr,
- &pages_sharing_attr.attr,
- &pages_unshared_attr.attr,
- &pages_volatile_attr.attr,
- &full_scans_attr.attr,
- #ifdef CONFIG_NUMA
- &merge_across_nodes_attr.attr,
- #endif
- NULL,
- };
- static struct attribute_group ksm_attr_group = {
- .attrs = ksm_attrs,
- .name = "ksm",
- };
- #endif
- static int __init ksm_init(void)
- {
- struct task_struct *ksm_thread;
- int err;
- err = ksm_slab_init();
- if (err)
- goto out;
- ksm_thread = kthread_run(ksm_scan_thread, NULL, "ksmd");
- if (IS_ERR(ksm_thread)) {
- pr_err("ksm: creating kthread failed\n");
- err = PTR_ERR(ksm_thread);
- goto out_free;
- }
- #ifdef CONFIG_SYSFS
- err = sysfs_create_group(mm_kobj, &ksm_attr_group);
- if (err) {
- pr_err("ksm: register sysfs failed\n");
- kthread_stop(ksm_thread);
- goto out_free;
- }
- #else
- ksm_run = KSM_RUN_MERGE;
- #endif
- #ifdef CONFIG_MEMORY_HOTREMOVE
-
- hotplug_memory_notifier(ksm_memory_callback, 100);
- #endif
- return 0;
- out_free:
- ksm_slab_free();
- out:
- return err;
- }
- subsys_initcall(ksm_init);
|