12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488 |
- #include <linux/kernel.h>
- #include <linux/syscalls.h>
- #include <linux/fs.h>
- #include <linux/iomap.h>
- #include <linux/mm.h>
- #include <linux/percpu.h>
- #include <linux/slab.h>
- #include <linux/capability.h>
- #include <linux/blkdev.h>
- #include <linux/file.h>
- #include <linux/quotaops.h>
- #include <linux/highmem.h>
- #include <linux/export.h>
- #include <linux/backing-dev.h>
- #include <linux/writeback.h>
- #include <linux/hash.h>
- #include <linux/suspend.h>
- #include <linux/buffer_head.h>
- #include <linux/task_io_accounting_ops.h>
- #include <linux/bio.h>
- #include <linux/notifier.h>
- #include <linux/cpu.h>
- #include <linux/bitops.h>
- #include <linux/mpage.h>
- #include <linux/bit_spinlock.h>
- #include <trace/events/block.h>
- static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
- static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
- unsigned long bio_flags,
- struct writeback_control *wbc);
- #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
- void init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
- {
- bh->b_end_io = handler;
- bh->b_private = private;
- }
- EXPORT_SYMBOL(init_buffer);
- inline void touch_buffer(struct buffer_head *bh)
- {
- trace_block_touch_buffer(bh);
- mark_page_accessed(bh->b_page);
- }
- EXPORT_SYMBOL(touch_buffer);
- void __lock_buffer(struct buffer_head *bh)
- {
- wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
- }
- EXPORT_SYMBOL(__lock_buffer);
- void unlock_buffer(struct buffer_head *bh)
- {
- clear_bit_unlock(BH_Lock, &bh->b_state);
- smp_mb__after_atomic();
- wake_up_bit(&bh->b_state, BH_Lock);
- }
- EXPORT_SYMBOL(unlock_buffer);
- void buffer_check_dirty_writeback(struct page *page,
- bool *dirty, bool *writeback)
- {
- struct buffer_head *head, *bh;
- *dirty = false;
- *writeback = false;
- BUG_ON(!PageLocked(page));
- if (!page_has_buffers(page))
- return;
- if (PageWriteback(page))
- *writeback = true;
- head = page_buffers(page);
- bh = head;
- do {
- if (buffer_locked(bh))
- *writeback = true;
- if (buffer_dirty(bh))
- *dirty = true;
- bh = bh->b_this_page;
- } while (bh != head);
- }
- EXPORT_SYMBOL(buffer_check_dirty_writeback);
- void __wait_on_buffer(struct buffer_head * bh)
- {
- wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
- }
- EXPORT_SYMBOL(__wait_on_buffer);
- static void
- __clear_page_buffers(struct page *page)
- {
- ClearPagePrivate(page);
- set_page_private(page, 0);
- put_page(page);
- }
- static void buffer_io_error(struct buffer_head *bh, char *msg)
- {
- if (!test_bit(BH_Quiet, &bh->b_state))
- printk_ratelimited(KERN_ERR
- "Buffer I/O error on dev %pg, logical block %llu%s\n",
- bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
- }
- static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
- {
- if (uptodate) {
- set_buffer_uptodate(bh);
- } else {
-
- clear_buffer_uptodate(bh);
- }
- unlock_buffer(bh);
- }
- void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
- {
- __end_buffer_read_notouch(bh, uptodate);
- put_bh(bh);
- }
- EXPORT_SYMBOL(end_buffer_read_sync);
- void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
- {
- if (uptodate) {
- set_buffer_uptodate(bh);
- } else {
- buffer_io_error(bh, ", lost sync page write");
- set_buffer_write_io_error(bh);
- clear_buffer_uptodate(bh);
- }
- unlock_buffer(bh);
- put_bh(bh);
- }
- EXPORT_SYMBOL(end_buffer_write_sync);
- static struct buffer_head *
- __find_get_block_slow(struct block_device *bdev, sector_t block)
- {
- struct inode *bd_inode = bdev->bd_inode;
- struct address_space *bd_mapping = bd_inode->i_mapping;
- struct buffer_head *ret = NULL;
- pgoff_t index;
- struct buffer_head *bh;
- struct buffer_head *head;
- struct page *page;
- int all_mapped = 1;
- index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
- page = find_get_page_flags(bd_mapping, index, FGP_ACCESSED);
- if (!page)
- goto out;
- spin_lock(&bd_mapping->private_lock);
- if (!page_has_buffers(page))
- goto out_unlock;
- head = page_buffers(page);
- bh = head;
- do {
- if (!buffer_mapped(bh))
- all_mapped = 0;
- else if (bh->b_blocknr == block) {
- ret = bh;
- get_bh(bh);
- goto out_unlock;
- }
- bh = bh->b_this_page;
- } while (bh != head);
-
- if (all_mapped) {
- printk("__find_get_block_slow() failed. "
- "block=%llu, b_blocknr=%llu\n",
- (unsigned long long)block,
- (unsigned long long)bh->b_blocknr);
- printk("b_state=0x%08lx, b_size=%zu\n",
- bh->b_state, bh->b_size);
- printk("device %pg blocksize: %d\n", bdev,
- 1 << bd_inode->i_blkbits);
- }
- out_unlock:
- spin_unlock(&bd_mapping->private_lock);
- put_page(page);
- out:
- return ret;
- }
- static void free_more_memory(void)
- {
- struct zoneref *z;
- int nid;
- wakeup_flusher_threads(1024, WB_REASON_FREE_MORE_MEM);
- yield();
- for_each_online_node(nid) {
- z = first_zones_zonelist(node_zonelist(nid, GFP_NOFS),
- gfp_zone(GFP_NOFS), NULL);
- if (z->zone)
- try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0,
- GFP_NOFS, NULL);
- }
- }
- static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
- {
- unsigned long flags;
- struct buffer_head *first;
- struct buffer_head *tmp;
- struct page *page;
- int page_uptodate = 1;
- BUG_ON(!buffer_async_read(bh));
- page = bh->b_page;
- if (uptodate) {
- set_buffer_uptodate(bh);
- } else {
- clear_buffer_uptodate(bh);
- buffer_io_error(bh, ", async page read");
- SetPageError(page);
- }
-
- first = page_buffers(page);
- local_irq_save(flags);
- bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
- clear_buffer_async_read(bh);
- unlock_buffer(bh);
- tmp = bh;
- do {
- if (!buffer_uptodate(tmp))
- page_uptodate = 0;
- if (buffer_async_read(tmp)) {
- BUG_ON(!buffer_locked(tmp));
- goto still_busy;
- }
- tmp = tmp->b_this_page;
- } while (tmp != bh);
- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
- local_irq_restore(flags);
-
- if (page_uptodate && !PageError(page))
- SetPageUptodate(page);
- unlock_page(page);
- return;
- still_busy:
- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
- local_irq_restore(flags);
- return;
- }
- void end_buffer_async_write(struct buffer_head *bh, int uptodate)
- {
- unsigned long flags;
- struct buffer_head *first;
- struct buffer_head *tmp;
- struct page *page;
- BUG_ON(!buffer_async_write(bh));
- page = bh->b_page;
- if (uptodate) {
- set_buffer_uptodate(bh);
- } else {
- buffer_io_error(bh, ", lost async page write");
- mapping_set_error(page->mapping, -EIO);
- set_buffer_write_io_error(bh);
- clear_buffer_uptodate(bh);
- SetPageError(page);
- }
- first = page_buffers(page);
- local_irq_save(flags);
- bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
- clear_buffer_async_write(bh);
- unlock_buffer(bh);
- tmp = bh->b_this_page;
- while (tmp != bh) {
- if (buffer_async_write(tmp)) {
- BUG_ON(!buffer_locked(tmp));
- goto still_busy;
- }
- tmp = tmp->b_this_page;
- }
- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
- local_irq_restore(flags);
- end_page_writeback(page);
- return;
- still_busy:
- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
- local_irq_restore(flags);
- return;
- }
- EXPORT_SYMBOL(end_buffer_async_write);
- static void mark_buffer_async_read(struct buffer_head *bh)
- {
- bh->b_end_io = end_buffer_async_read;
- set_buffer_async_read(bh);
- }
- static void mark_buffer_async_write_endio(struct buffer_head *bh,
- bh_end_io_t *handler)
- {
- bh->b_end_io = handler;
- set_buffer_async_write(bh);
- }
- void mark_buffer_async_write(struct buffer_head *bh)
- {
- mark_buffer_async_write_endio(bh, end_buffer_async_write);
- }
- EXPORT_SYMBOL(mark_buffer_async_write);
- static void __remove_assoc_queue(struct buffer_head *bh)
- {
- list_del_init(&bh->b_assoc_buffers);
- WARN_ON(!bh->b_assoc_map);
- if (buffer_write_io_error(bh))
- set_bit(AS_EIO, &bh->b_assoc_map->flags);
- bh->b_assoc_map = NULL;
- }
- int inode_has_buffers(struct inode *inode)
- {
- return !list_empty(&inode->i_data.private_list);
- }
- static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
- {
- struct buffer_head *bh;
- struct list_head *p;
- int err = 0;
- spin_lock(lock);
- repeat:
- list_for_each_prev(p, list) {
- bh = BH_ENTRY(p);
- if (buffer_locked(bh)) {
- get_bh(bh);
- spin_unlock(lock);
- wait_on_buffer(bh);
- if (!buffer_uptodate(bh))
- err = -EIO;
- brelse(bh);
- spin_lock(lock);
- goto repeat;
- }
- }
- spin_unlock(lock);
- return err;
- }
- static void do_thaw_one(struct super_block *sb, void *unused)
- {
- while (sb->s_bdev && !thaw_bdev(sb->s_bdev, sb))
- printk(KERN_WARNING "Emergency Thaw on %pg\n", sb->s_bdev);
- }
- static void do_thaw_all(struct work_struct *work)
- {
- iterate_supers(do_thaw_one, NULL);
- kfree(work);
- printk(KERN_WARNING "Emergency Thaw complete\n");
- }
- void emergency_thaw_all(void)
- {
- struct work_struct *work;
- work = kmalloc(sizeof(*work), GFP_ATOMIC);
- if (work) {
- INIT_WORK(work, do_thaw_all);
- schedule_work(work);
- }
- }
- int sync_mapping_buffers(struct address_space *mapping)
- {
- struct address_space *buffer_mapping = mapping->private_data;
- if (buffer_mapping == NULL || list_empty(&mapping->private_list))
- return 0;
- return fsync_buffers_list(&buffer_mapping->private_lock,
- &mapping->private_list);
- }
- EXPORT_SYMBOL(sync_mapping_buffers);
- void write_boundary_block(struct block_device *bdev,
- sector_t bblock, unsigned blocksize)
- {
- struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
- if (bh) {
- if (buffer_dirty(bh))
- ll_rw_block(REQ_OP_WRITE, 0, 1, &bh);
- put_bh(bh);
- }
- }
- void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
- {
- struct address_space *mapping = inode->i_mapping;
- struct address_space *buffer_mapping = bh->b_page->mapping;
- mark_buffer_dirty(bh);
- if (!mapping->private_data) {
- mapping->private_data = buffer_mapping;
- } else {
- BUG_ON(mapping->private_data != buffer_mapping);
- }
- if (!bh->b_assoc_map) {
- spin_lock(&buffer_mapping->private_lock);
- list_move_tail(&bh->b_assoc_buffers,
- &mapping->private_list);
- bh->b_assoc_map = mapping;
- spin_unlock(&buffer_mapping->private_lock);
- }
- }
- EXPORT_SYMBOL(mark_buffer_dirty_inode);
- static void __set_page_dirty(struct page *page, struct address_space *mapping,
- int warn)
- {
- unsigned long flags;
- spin_lock_irqsave(&mapping->tree_lock, flags);
- if (page->mapping) {
- WARN_ON_ONCE(warn && !PageUptodate(page));
- account_page_dirtied(page, mapping);
- radix_tree_tag_set(&mapping->page_tree,
- page_index(page), PAGECACHE_TAG_DIRTY);
- }
- spin_unlock_irqrestore(&mapping->tree_lock, flags);
- }
- int __set_page_dirty_buffers(struct page *page)
- {
- int newly_dirty;
- struct address_space *mapping = page_mapping(page);
- if (unlikely(!mapping))
- return !TestSetPageDirty(page);
- spin_lock(&mapping->private_lock);
- if (page_has_buffers(page)) {
- struct buffer_head *head = page_buffers(page);
- struct buffer_head *bh = head;
- do {
- set_buffer_dirty(bh);
- bh = bh->b_this_page;
- } while (bh != head);
- }
-
- lock_page_memcg(page);
- newly_dirty = !TestSetPageDirty(page);
- spin_unlock(&mapping->private_lock);
- if (newly_dirty)
- __set_page_dirty(page, mapping, 1);
- unlock_page_memcg(page);
- if (newly_dirty)
- __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
- return newly_dirty;
- }
- EXPORT_SYMBOL(__set_page_dirty_buffers);
- static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
- {
- struct buffer_head *bh;
- struct list_head tmp;
- struct address_space *mapping;
- int err = 0, err2;
- struct blk_plug plug;
- INIT_LIST_HEAD(&tmp);
- blk_start_plug(&plug);
- spin_lock(lock);
- while (!list_empty(list)) {
- bh = BH_ENTRY(list->next);
- mapping = bh->b_assoc_map;
- __remove_assoc_queue(bh);
-
- smp_mb();
- if (buffer_dirty(bh) || buffer_locked(bh)) {
- list_add(&bh->b_assoc_buffers, &tmp);
- bh->b_assoc_map = mapping;
- if (buffer_dirty(bh)) {
- get_bh(bh);
- spin_unlock(lock);
-
- write_dirty_buffer(bh, WRITE_SYNC);
-
- brelse(bh);
- spin_lock(lock);
- }
- }
- }
- spin_unlock(lock);
- blk_finish_plug(&plug);
- spin_lock(lock);
- while (!list_empty(&tmp)) {
- bh = BH_ENTRY(tmp.prev);
- get_bh(bh);
- mapping = bh->b_assoc_map;
- __remove_assoc_queue(bh);
-
- smp_mb();
- if (buffer_dirty(bh)) {
- list_add(&bh->b_assoc_buffers,
- &mapping->private_list);
- bh->b_assoc_map = mapping;
- }
- spin_unlock(lock);
- wait_on_buffer(bh);
- if (!buffer_uptodate(bh))
- err = -EIO;
- brelse(bh);
- spin_lock(lock);
- }
-
- spin_unlock(lock);
- err2 = osync_buffers_list(lock, list);
- if (err)
- return err;
- else
- return err2;
- }
- void invalidate_inode_buffers(struct inode *inode)
- {
- if (inode_has_buffers(inode)) {
- struct address_space *mapping = &inode->i_data;
- struct list_head *list = &mapping->private_list;
- struct address_space *buffer_mapping = mapping->private_data;
- spin_lock(&buffer_mapping->private_lock);
- while (!list_empty(list))
- __remove_assoc_queue(BH_ENTRY(list->next));
- spin_unlock(&buffer_mapping->private_lock);
- }
- }
- EXPORT_SYMBOL(invalidate_inode_buffers);
- int remove_inode_buffers(struct inode *inode)
- {
- int ret = 1;
- if (inode_has_buffers(inode)) {
- struct address_space *mapping = &inode->i_data;
- struct list_head *list = &mapping->private_list;
- struct address_space *buffer_mapping = mapping->private_data;
- spin_lock(&buffer_mapping->private_lock);
- while (!list_empty(list)) {
- struct buffer_head *bh = BH_ENTRY(list->next);
- if (buffer_dirty(bh)) {
- ret = 0;
- break;
- }
- __remove_assoc_queue(bh);
- }
- spin_unlock(&buffer_mapping->private_lock);
- }
- return ret;
- }
- struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
- int retry)
- {
- struct buffer_head *bh, *head;
- long offset;
- try_again:
- head = NULL;
- offset = PAGE_SIZE;
- while ((offset -= size) >= 0) {
- bh = alloc_buffer_head(GFP_NOFS);
- if (!bh)
- goto no_grow;
- bh->b_this_page = head;
- bh->b_blocknr = -1;
- head = bh;
- bh->b_size = size;
-
- set_bh_page(bh, page, offset);
- }
- return head;
- no_grow:
- if (head) {
- do {
- bh = head;
- head = head->b_this_page;
- free_buffer_head(bh);
- } while (head);
- }
-
- if (!retry)
- return NULL;
-
- free_more_memory();
- goto try_again;
- }
- EXPORT_SYMBOL_GPL(alloc_page_buffers);
- static inline void
- link_dev_buffers(struct page *page, struct buffer_head *head)
- {
- struct buffer_head *bh, *tail;
- bh = head;
- do {
- tail = bh;
- bh = bh->b_this_page;
- } while (bh);
- tail->b_this_page = head;
- attach_page_buffers(page, head);
- }
- static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
- {
- sector_t retval = ~((sector_t)0);
- loff_t sz = i_size_read(bdev->bd_inode);
- if (sz) {
- unsigned int sizebits = blksize_bits(size);
- retval = (sz >> sizebits);
- }
- return retval;
- }
-
- static sector_t
- init_page_buffers(struct page *page, struct block_device *bdev,
- sector_t block, int size)
- {
- struct buffer_head *head = page_buffers(page);
- struct buffer_head *bh = head;
- int uptodate = PageUptodate(page);
- sector_t end_block = blkdev_max_block(I_BDEV(bdev->bd_inode), size);
- do {
- if (!buffer_mapped(bh)) {
- init_buffer(bh, NULL, NULL);
- bh->b_bdev = bdev;
- bh->b_blocknr = block;
- if (uptodate)
- set_buffer_uptodate(bh);
- if (block < end_block)
- set_buffer_mapped(bh);
- }
- block++;
- bh = bh->b_this_page;
- } while (bh != head);
-
- return end_block;
- }
- static int
- grow_dev_page(struct block_device *bdev, sector_t block,
- pgoff_t index, int size, int sizebits, gfp_t gfp)
- {
- struct inode *inode = bdev->bd_inode;
- struct page *page;
- struct buffer_head *bh;
- sector_t end_block;
- int ret = 0;
- gfp_t gfp_mask;
- gfp_mask = mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS) | gfp;
-
- gfp_mask |= __GFP_NOFAIL;
- page = find_or_create_page(inode->i_mapping, index, gfp_mask);
- if (!page)
- return ret;
- BUG_ON(!PageLocked(page));
- if (page_has_buffers(page)) {
- bh = page_buffers(page);
- if (bh->b_size == size) {
- end_block = init_page_buffers(page, bdev,
- (sector_t)index << sizebits,
- size);
- goto done;
- }
- if (!try_to_free_buffers(page))
- goto failed;
- }
-
- bh = alloc_page_buffers(page, size, 0);
- if (!bh)
- goto failed;
-
- spin_lock(&inode->i_mapping->private_lock);
- link_dev_buffers(page, bh);
- end_block = init_page_buffers(page, bdev, (sector_t)index << sizebits,
- size);
- spin_unlock(&inode->i_mapping->private_lock);
- done:
- ret = (block < end_block) ? 1 : -ENXIO;
- failed:
- unlock_page(page);
- put_page(page);
- return ret;
- }
- static int
- grow_buffers(struct block_device *bdev, sector_t block, int size, gfp_t gfp)
- {
- pgoff_t index;
- int sizebits;
- sizebits = -1;
- do {
- sizebits++;
- } while ((size << sizebits) < PAGE_SIZE);
- index = block >> sizebits;
-
- if (unlikely(index != block >> sizebits)) {
- printk(KERN_ERR "%s: requested out-of-range block %llu for "
- "device %pg\n",
- __func__, (unsigned long long)block,
- bdev);
- return -EIO;
- }
-
- return grow_dev_page(bdev, block, index, size, sizebits, gfp);
- }
- static struct buffer_head *
- __getblk_slow(struct block_device *bdev, sector_t block,
- unsigned size, gfp_t gfp)
- {
-
- if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
- (size < 512 || size > PAGE_SIZE))) {
- printk(KERN_ERR "getblk(): invalid block size %d requested\n",
- size);
- printk(KERN_ERR "logical block size: %d\n",
- bdev_logical_block_size(bdev));
- dump_stack();
- return NULL;
- }
- for (;;) {
- struct buffer_head *bh;
- int ret;
- bh = __find_get_block(bdev, block, size);
- if (bh)
- return bh;
- ret = grow_buffers(bdev, block, size, gfp);
- if (ret < 0)
- return NULL;
- if (ret == 0)
- free_more_memory();
- }
- }
- void mark_buffer_dirty(struct buffer_head *bh)
- {
- WARN_ON_ONCE(!buffer_uptodate(bh));
- trace_block_dirty_buffer(bh);
-
- if (buffer_dirty(bh)) {
- smp_mb();
- if (buffer_dirty(bh))
- return;
- }
- if (!test_set_buffer_dirty(bh)) {
- struct page *page = bh->b_page;
- struct address_space *mapping = NULL;
- lock_page_memcg(page);
- if (!TestSetPageDirty(page)) {
- mapping = page_mapping(page);
- if (mapping)
- __set_page_dirty(page, mapping, 0);
- }
- unlock_page_memcg(page);
- if (mapping)
- __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
- }
- }
- EXPORT_SYMBOL(mark_buffer_dirty);
- void __brelse(struct buffer_head * buf)
- {
- if (atomic_read(&buf->b_count)) {
- put_bh(buf);
- return;
- }
- WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
- }
- EXPORT_SYMBOL(__brelse);
- void __bforget(struct buffer_head *bh)
- {
- clear_buffer_dirty(bh);
- if (bh->b_assoc_map) {
- struct address_space *buffer_mapping = bh->b_page->mapping;
- spin_lock(&buffer_mapping->private_lock);
- list_del_init(&bh->b_assoc_buffers);
- bh->b_assoc_map = NULL;
- spin_unlock(&buffer_mapping->private_lock);
- }
- __brelse(bh);
- }
- EXPORT_SYMBOL(__bforget);
- static struct buffer_head *__bread_slow(struct buffer_head *bh)
- {
- lock_buffer(bh);
- if (buffer_uptodate(bh)) {
- unlock_buffer(bh);
- return bh;
- } else {
- get_bh(bh);
- bh->b_end_io = end_buffer_read_sync;
- submit_bh(REQ_OP_READ, 0, bh);
- wait_on_buffer(bh);
- if (buffer_uptodate(bh))
- return bh;
- }
- brelse(bh);
- return NULL;
- }
- #define BH_LRU_SIZE 16
- struct bh_lru {
- struct buffer_head *bhs[BH_LRU_SIZE];
- };
- static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
- #ifdef CONFIG_SMP
- #define bh_lru_lock() local_irq_disable()
- #define bh_lru_unlock() local_irq_enable()
- #else
- #define bh_lru_lock() preempt_disable()
- #define bh_lru_unlock() preempt_enable()
- #endif
- static inline void check_irqs_on(void)
- {
- #ifdef irqs_disabled
- BUG_ON(irqs_disabled());
- #endif
- }
- static void bh_lru_install(struct buffer_head *bh)
- {
- struct buffer_head *evictee = NULL;
- check_irqs_on();
- bh_lru_lock();
- if (__this_cpu_read(bh_lrus.bhs[0]) != bh) {
- struct buffer_head *bhs[BH_LRU_SIZE];
- int in;
- int out = 0;
- get_bh(bh);
- bhs[out++] = bh;
- for (in = 0; in < BH_LRU_SIZE; in++) {
- struct buffer_head *bh2 =
- __this_cpu_read(bh_lrus.bhs[in]);
- if (bh2 == bh) {
- __brelse(bh2);
- } else {
- if (out >= BH_LRU_SIZE) {
- BUG_ON(evictee != NULL);
- evictee = bh2;
- } else {
- bhs[out++] = bh2;
- }
- }
- }
- while (out < BH_LRU_SIZE)
- bhs[out++] = NULL;
- memcpy(this_cpu_ptr(&bh_lrus.bhs), bhs, sizeof(bhs));
- }
- bh_lru_unlock();
- if (evictee)
- __brelse(evictee);
- }
- static struct buffer_head *
- lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
- {
- struct buffer_head *ret = NULL;
- unsigned int i;
- check_irqs_on();
- bh_lru_lock();
- for (i = 0; i < BH_LRU_SIZE; i++) {
- struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
- if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
- bh->b_size == size) {
- if (i) {
- while (i) {
- __this_cpu_write(bh_lrus.bhs[i],
- __this_cpu_read(bh_lrus.bhs[i - 1]));
- i--;
- }
- __this_cpu_write(bh_lrus.bhs[0], bh);
- }
- get_bh(bh);
- ret = bh;
- break;
- }
- }
- bh_lru_unlock();
- return ret;
- }
- struct buffer_head *
- __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
- {
- struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
- if (bh == NULL) {
-
- bh = __find_get_block_slow(bdev, block);
- if (bh)
- bh_lru_install(bh);
- } else
- touch_buffer(bh);
- return bh;
- }
- EXPORT_SYMBOL(__find_get_block);
- struct buffer_head *
- __getblk_gfp(struct block_device *bdev, sector_t block,
- unsigned size, gfp_t gfp)
- {
- struct buffer_head *bh = __find_get_block(bdev, block, size);
- might_sleep();
- if (bh == NULL)
- bh = __getblk_slow(bdev, block, size, gfp);
- return bh;
- }
- EXPORT_SYMBOL(__getblk_gfp);
- void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
- {
- struct buffer_head *bh = __getblk(bdev, block, size);
- if (likely(bh)) {
- ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, &bh);
- brelse(bh);
- }
- }
- EXPORT_SYMBOL(__breadahead);
- struct buffer_head *
- __bread_gfp(struct block_device *bdev, sector_t block,
- unsigned size, gfp_t gfp)
- {
- struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
- if (likely(bh) && !buffer_uptodate(bh))
- bh = __bread_slow(bh);
- return bh;
- }
- EXPORT_SYMBOL(__bread_gfp);
- static void invalidate_bh_lru(void *arg)
- {
- struct bh_lru *b = &get_cpu_var(bh_lrus);
- int i;
- for (i = 0; i < BH_LRU_SIZE; i++) {
- brelse(b->bhs[i]);
- b->bhs[i] = NULL;
- }
- put_cpu_var(bh_lrus);
- }
- static bool has_bh_in_lru(int cpu, void *dummy)
- {
- struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
- int i;
-
- for (i = 0; i < BH_LRU_SIZE; i++) {
- if (b->bhs[i])
- return 1;
- }
- return 0;
- }
- void invalidate_bh_lrus(void)
- {
- on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1, GFP_KERNEL);
- }
- EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
- void set_bh_page(struct buffer_head *bh,
- struct page *page, unsigned long offset)
- {
- bh->b_page = page;
- BUG_ON(offset >= PAGE_SIZE);
- if (PageHighMem(page))
-
- bh->b_data = (char *)(0 + offset);
- else
- bh->b_data = page_address(page) + offset;
- }
- EXPORT_SYMBOL(set_bh_page);
- #define BUFFER_FLAGS_DISCARD \
- (1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
- 1 << BH_Delay | 1 << BH_Unwritten)
- static void discard_buffer(struct buffer_head * bh)
- {
- unsigned long b_state, b_state_old;
- lock_buffer(bh);
- clear_buffer_dirty(bh);
- bh->b_bdev = NULL;
- b_state = bh->b_state;
- for (;;) {
- b_state_old = cmpxchg(&bh->b_state, b_state,
- (b_state & ~BUFFER_FLAGS_DISCARD));
- if (b_state_old == b_state)
- break;
- b_state = b_state_old;
- }
- unlock_buffer(bh);
- }
- void block_invalidatepage(struct page *page, unsigned int offset,
- unsigned int length)
- {
- struct buffer_head *head, *bh, *next;
- unsigned int curr_off = 0;
- unsigned int stop = length + offset;
- BUG_ON(!PageLocked(page));
- if (!page_has_buffers(page))
- goto out;
-
- BUG_ON(stop > PAGE_SIZE || stop < length);
- head = page_buffers(page);
- bh = head;
- do {
- unsigned int next_off = curr_off + bh->b_size;
- next = bh->b_this_page;
-
- if (next_off > stop)
- goto out;
-
- if (offset <= curr_off)
- discard_buffer(bh);
- curr_off = next_off;
- bh = next;
- } while (bh != head);
-
- if (offset == 0)
- try_to_release_page(page, 0);
- out:
- return;
- }
- EXPORT_SYMBOL(block_invalidatepage);
- void create_empty_buffers(struct page *page,
- unsigned long blocksize, unsigned long b_state)
- {
- struct buffer_head *bh, *head, *tail;
- head = alloc_page_buffers(page, blocksize, 1);
- bh = head;
- do {
- bh->b_state |= b_state;
- tail = bh;
- bh = bh->b_this_page;
- } while (bh);
- tail->b_this_page = head;
- spin_lock(&page->mapping->private_lock);
- if (PageUptodate(page) || PageDirty(page)) {
- bh = head;
- do {
- if (PageDirty(page))
- set_buffer_dirty(bh);
- if (PageUptodate(page))
- set_buffer_uptodate(bh);
- bh = bh->b_this_page;
- } while (bh != head);
- }
- attach_page_buffers(page, head);
- spin_unlock(&page->mapping->private_lock);
- }
- EXPORT_SYMBOL(create_empty_buffers);
- void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
- {
- struct buffer_head *old_bh;
- might_sleep();
- old_bh = __find_get_block_slow(bdev, block);
- if (old_bh) {
- clear_buffer_dirty(old_bh);
- wait_on_buffer(old_bh);
- clear_buffer_req(old_bh);
- __brelse(old_bh);
- }
- }
- EXPORT_SYMBOL(unmap_underlying_metadata);
- static inline int block_size_bits(unsigned int blocksize)
- {
- return ilog2(blocksize);
- }
- static struct buffer_head *create_page_buffers(struct page *page, struct inode *inode, unsigned int b_state)
- {
- BUG_ON(!PageLocked(page));
- if (!page_has_buffers(page))
- create_empty_buffers(page, 1 << ACCESS_ONCE(inode->i_blkbits), b_state);
- return page_buffers(page);
- }
- int __block_write_full_page(struct inode *inode, struct page *page,
- get_block_t *get_block, struct writeback_control *wbc,
- bh_end_io_t *handler)
- {
- int err;
- sector_t block;
- sector_t last_block;
- struct buffer_head *bh, *head;
- unsigned int blocksize, bbits;
- int nr_underway = 0;
- int write_flags = (wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : 0);
- head = create_page_buffers(page, inode,
- (1 << BH_Dirty)|(1 << BH_Uptodate));
-
- bh = head;
- blocksize = bh->b_size;
- bbits = block_size_bits(blocksize);
- block = (sector_t)page->index << (PAGE_SHIFT - bbits);
- last_block = (i_size_read(inode) - 1) >> bbits;
-
- do {
- if (block > last_block) {
-
-
- clear_buffer_dirty(bh);
- set_buffer_uptodate(bh);
- } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
- buffer_dirty(bh)) {
- WARN_ON(bh->b_size != blocksize);
- err = get_block(inode, block, bh, 1);
- if (err)
- goto recover;
- clear_buffer_delay(bh);
- if (buffer_new(bh)) {
-
- clear_buffer_new(bh);
- unmap_underlying_metadata(bh->b_bdev,
- bh->b_blocknr);
- }
- }
- bh = bh->b_this_page;
- block++;
- } while (bh != head);
- do {
- if (!buffer_mapped(bh))
- continue;
-
- if (wbc->sync_mode != WB_SYNC_NONE) {
- lock_buffer(bh);
- } else if (!trylock_buffer(bh)) {
- redirty_page_for_writepage(wbc, page);
- continue;
- }
- if (test_clear_buffer_dirty(bh)) {
- mark_buffer_async_write_endio(bh, handler);
- } else {
- unlock_buffer(bh);
- }
- } while ((bh = bh->b_this_page) != head);
-
- BUG_ON(PageWriteback(page));
- set_page_writeback(page);
- do {
- struct buffer_head *next = bh->b_this_page;
- if (buffer_async_write(bh)) {
- submit_bh_wbc(REQ_OP_WRITE, write_flags, bh, 0, wbc);
- nr_underway++;
- }
- bh = next;
- } while (bh != head);
- unlock_page(page);
- err = 0;
- done:
- if (nr_underway == 0) {
-
- end_page_writeback(page);
-
- }
- return err;
- recover:
-
- bh = head;
-
- do {
- if (buffer_mapped(bh) && buffer_dirty(bh) &&
- !buffer_delay(bh)) {
- lock_buffer(bh);
- mark_buffer_async_write_endio(bh, handler);
- } else {
-
- clear_buffer_dirty(bh);
- }
- } while ((bh = bh->b_this_page) != head);
- SetPageError(page);
- BUG_ON(PageWriteback(page));
- mapping_set_error(page->mapping, err);
- set_page_writeback(page);
- do {
- struct buffer_head *next = bh->b_this_page;
- if (buffer_async_write(bh)) {
- clear_buffer_dirty(bh);
- submit_bh_wbc(REQ_OP_WRITE, write_flags, bh, 0, wbc);
- nr_underway++;
- }
- bh = next;
- } while (bh != head);
- unlock_page(page);
- goto done;
- }
- EXPORT_SYMBOL(__block_write_full_page);
- void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
- {
- unsigned int block_start, block_end;
- struct buffer_head *head, *bh;
- BUG_ON(!PageLocked(page));
- if (!page_has_buffers(page))
- return;
- bh = head = page_buffers(page);
- block_start = 0;
- do {
- block_end = block_start + bh->b_size;
- if (buffer_new(bh)) {
- if (block_end > from && block_start < to) {
- if (!PageUptodate(page)) {
- unsigned start, size;
- start = max(from, block_start);
- size = min(to, block_end) - start;
- zero_user(page, start, size);
- set_buffer_uptodate(bh);
- }
- clear_buffer_new(bh);
- mark_buffer_dirty(bh);
- }
- }
- block_start = block_end;
- bh = bh->b_this_page;
- } while (bh != head);
- }
- EXPORT_SYMBOL(page_zero_new_buffers);
- static void
- iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh,
- struct iomap *iomap)
- {
- loff_t offset = block << inode->i_blkbits;
- bh->b_bdev = iomap->bdev;
-
- BUG_ON(offset >= iomap->offset + iomap->length);
- switch (iomap->type) {
- case IOMAP_HOLE:
-
- if (!buffer_uptodate(bh) ||
- (offset >= i_size_read(inode)))
- set_buffer_new(bh);
- break;
- case IOMAP_DELALLOC:
- if (!buffer_uptodate(bh) ||
- (offset >= i_size_read(inode)))
- set_buffer_new(bh);
- set_buffer_uptodate(bh);
- set_buffer_mapped(bh);
- set_buffer_delay(bh);
- break;
- case IOMAP_UNWRITTEN:
-
- set_buffer_new(bh);
- set_buffer_unwritten(bh);
-
- case IOMAP_MAPPED:
- if (offset >= i_size_read(inode))
- set_buffer_new(bh);
- bh->b_blocknr = (iomap->blkno >> (inode->i_blkbits - 9)) +
- ((offset - iomap->offset) >> inode->i_blkbits);
- set_buffer_mapped(bh);
- break;
- }
- }
- int __block_write_begin_int(struct page *page, loff_t pos, unsigned len,
- get_block_t *get_block, struct iomap *iomap)
- {
- unsigned from = pos & (PAGE_SIZE - 1);
- unsigned to = from + len;
- struct inode *inode = page->mapping->host;
- unsigned block_start, block_end;
- sector_t block;
- int err = 0;
- unsigned blocksize, bbits;
- struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
- BUG_ON(!PageLocked(page));
- BUG_ON(from > PAGE_SIZE);
- BUG_ON(to > PAGE_SIZE);
- BUG_ON(from > to);
- head = create_page_buffers(page, inode, 0);
- blocksize = head->b_size;
- bbits = block_size_bits(blocksize);
- block = (sector_t)page->index << (PAGE_SHIFT - bbits);
- for(bh = head, block_start = 0; bh != head || !block_start;
- block++, block_start=block_end, bh = bh->b_this_page) {
- block_end = block_start + blocksize;
- if (block_end <= from || block_start >= to) {
- if (PageUptodate(page)) {
- if (!buffer_uptodate(bh))
- set_buffer_uptodate(bh);
- }
- continue;
- }
- if (buffer_new(bh))
- clear_buffer_new(bh);
- if (!buffer_mapped(bh)) {
- WARN_ON(bh->b_size != blocksize);
- if (get_block) {
- err = get_block(inode, block, bh, 1);
- if (err)
- break;
- } else {
- iomap_to_bh(inode, block, bh, iomap);
- }
- if (buffer_new(bh)) {
- unmap_underlying_metadata(bh->b_bdev,
- bh->b_blocknr);
- if (PageUptodate(page)) {
- clear_buffer_new(bh);
- set_buffer_uptodate(bh);
- mark_buffer_dirty(bh);
- continue;
- }
- if (block_end > to || block_start < from)
- zero_user_segments(page,
- to, block_end,
- block_start, from);
- continue;
- }
- }
- if (PageUptodate(page)) {
- if (!buffer_uptodate(bh))
- set_buffer_uptodate(bh);
- continue;
- }
- if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
- !buffer_unwritten(bh) &&
- (block_start < from || block_end > to)) {
- ll_rw_block(REQ_OP_READ, 0, 1, &bh);
- *wait_bh++=bh;
- }
- }
-
- while(wait_bh > wait) {
- wait_on_buffer(*--wait_bh);
- if (!buffer_uptodate(*wait_bh))
- err = -EIO;
- }
- if (unlikely(err))
- page_zero_new_buffers(page, from, to);
- return err;
- }
- int __block_write_begin(struct page *page, loff_t pos, unsigned len,
- get_block_t *get_block)
- {
- return __block_write_begin_int(page, pos, len, get_block, NULL);
- }
- EXPORT_SYMBOL(__block_write_begin);
- static int __block_commit_write(struct inode *inode, struct page *page,
- unsigned from, unsigned to)
- {
- unsigned block_start, block_end;
- int partial = 0;
- unsigned blocksize;
- struct buffer_head *bh, *head;
- bh = head = page_buffers(page);
- blocksize = bh->b_size;
- block_start = 0;
- do {
- block_end = block_start + blocksize;
- if (block_end <= from || block_start >= to) {
- if (!buffer_uptodate(bh))
- partial = 1;
- } else {
- set_buffer_uptodate(bh);
- mark_buffer_dirty(bh);
- }
- clear_buffer_new(bh);
- block_start = block_end;
- bh = bh->b_this_page;
- } while (bh != head);
-
- if (!partial)
- SetPageUptodate(page);
- return 0;
- }
- int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
- unsigned flags, struct page **pagep, get_block_t *get_block)
- {
- pgoff_t index = pos >> PAGE_SHIFT;
- struct page *page;
- int status;
- page = grab_cache_page_write_begin(mapping, index, flags);
- if (!page)
- return -ENOMEM;
- status = __block_write_begin(page, pos, len, get_block);
- if (unlikely(status)) {
- unlock_page(page);
- put_page(page);
- page = NULL;
- }
- *pagep = page;
- return status;
- }
- EXPORT_SYMBOL(block_write_begin);
- int block_write_end(struct file *file, struct address_space *mapping,
- loff_t pos, unsigned len, unsigned copied,
- struct page *page, void *fsdata)
- {
- struct inode *inode = mapping->host;
- unsigned start;
- start = pos & (PAGE_SIZE - 1);
- if (unlikely(copied < len)) {
-
- if (!PageUptodate(page))
- copied = 0;
- page_zero_new_buffers(page, start+copied, start+len);
- }
- flush_dcache_page(page);
-
- __block_commit_write(inode, page, start, start+copied);
- return copied;
- }
- EXPORT_SYMBOL(block_write_end);
- int generic_write_end(struct file *file, struct address_space *mapping,
- loff_t pos, unsigned len, unsigned copied,
- struct page *page, void *fsdata)
- {
- struct inode *inode = mapping->host;
- loff_t old_size = inode->i_size;
- int i_size_changed = 0;
- copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
-
- if (pos+copied > inode->i_size) {
- i_size_write(inode, pos+copied);
- i_size_changed = 1;
- }
- unlock_page(page);
- put_page(page);
- if (old_size < pos)
- pagecache_isize_extended(inode, old_size, pos);
-
- if (i_size_changed)
- mark_inode_dirty(inode);
- return copied;
- }
- EXPORT_SYMBOL(generic_write_end);
- int block_is_partially_uptodate(struct page *page, unsigned long from,
- unsigned long count)
- {
- unsigned block_start, block_end, blocksize;
- unsigned to;
- struct buffer_head *bh, *head;
- int ret = 1;
- if (!page_has_buffers(page))
- return 0;
- head = page_buffers(page);
- blocksize = head->b_size;
- to = min_t(unsigned, PAGE_SIZE - from, count);
- to = from + to;
- if (from < blocksize && to > PAGE_SIZE - blocksize)
- return 0;
- bh = head;
- block_start = 0;
- do {
- block_end = block_start + blocksize;
- if (block_end > from && block_start < to) {
- if (!buffer_uptodate(bh)) {
- ret = 0;
- break;
- }
- if (block_end >= to)
- break;
- }
- block_start = block_end;
- bh = bh->b_this_page;
- } while (bh != head);
- return ret;
- }
- EXPORT_SYMBOL(block_is_partially_uptodate);
- int block_read_full_page(struct page *page, get_block_t *get_block)
- {
- struct inode *inode = page->mapping->host;
- sector_t iblock, lblock;
- struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
- unsigned int blocksize, bbits;
- int nr, i;
- int fully_mapped = 1;
- head = create_page_buffers(page, inode, 0);
- blocksize = head->b_size;
- bbits = block_size_bits(blocksize);
- iblock = (sector_t)page->index << (PAGE_SHIFT - bbits);
- lblock = (i_size_read(inode)+blocksize-1) >> bbits;
- bh = head;
- nr = 0;
- i = 0;
- do {
- if (buffer_uptodate(bh))
- continue;
- if (!buffer_mapped(bh)) {
- int err = 0;
- fully_mapped = 0;
- if (iblock < lblock) {
- WARN_ON(bh->b_size != blocksize);
- err = get_block(inode, iblock, bh, 0);
- if (err)
- SetPageError(page);
- }
- if (!buffer_mapped(bh)) {
- zero_user(page, i * blocksize, blocksize);
- if (!err)
- set_buffer_uptodate(bh);
- continue;
- }
-
- if (buffer_uptodate(bh))
- continue;
- }
- arr[nr++] = bh;
- } while (i++, iblock++, (bh = bh->b_this_page) != head);
- if (fully_mapped)
- SetPageMappedToDisk(page);
- if (!nr) {
-
- if (!PageError(page))
- SetPageUptodate(page);
- unlock_page(page);
- return 0;
- }
-
- for (i = 0; i < nr; i++) {
- bh = arr[i];
- lock_buffer(bh);
- mark_buffer_async_read(bh);
- }
-
- for (i = 0; i < nr; i++) {
- bh = arr[i];
- if (buffer_uptodate(bh))
- end_buffer_async_read(bh, 1);
- else
- submit_bh(REQ_OP_READ, 0, bh);
- }
- return 0;
- }
- EXPORT_SYMBOL(block_read_full_page);
- int generic_cont_expand_simple(struct inode *inode, loff_t size)
- {
- struct address_space *mapping = inode->i_mapping;
- struct page *page;
- void *fsdata;
- int err;
- err = inode_newsize_ok(inode, size);
- if (err)
- goto out;
- err = pagecache_write_begin(NULL, mapping, size, 0,
- AOP_FLAG_UNINTERRUPTIBLE|AOP_FLAG_CONT_EXPAND,
- &page, &fsdata);
- if (err)
- goto out;
- err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
- BUG_ON(err > 0);
- out:
- return err;
- }
- EXPORT_SYMBOL(generic_cont_expand_simple);
- static int cont_expand_zero(struct file *file, struct address_space *mapping,
- loff_t pos, loff_t *bytes)
- {
- struct inode *inode = mapping->host;
- unsigned int blocksize = i_blocksize(inode);
- struct page *page;
- void *fsdata;
- pgoff_t index, curidx;
- loff_t curpos;
- unsigned zerofrom, offset, len;
- int err = 0;
- index = pos >> PAGE_SHIFT;
- offset = pos & ~PAGE_MASK;
- while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) {
- zerofrom = curpos & ~PAGE_MASK;
- if (zerofrom & (blocksize-1)) {
- *bytes |= (blocksize-1);
- (*bytes)++;
- }
- len = PAGE_SIZE - zerofrom;
- err = pagecache_write_begin(file, mapping, curpos, len,
- AOP_FLAG_UNINTERRUPTIBLE,
- &page, &fsdata);
- if (err)
- goto out;
- zero_user(page, zerofrom, len);
- err = pagecache_write_end(file, mapping, curpos, len, len,
- page, fsdata);
- if (err < 0)
- goto out;
- BUG_ON(err != len);
- err = 0;
- balance_dirty_pages_ratelimited(mapping);
- if (unlikely(fatal_signal_pending(current))) {
- err = -EINTR;
- goto out;
- }
- }
-
- if (index == curidx) {
- zerofrom = curpos & ~PAGE_MASK;
-
- if (offset <= zerofrom) {
- goto out;
- }
- if (zerofrom & (blocksize-1)) {
- *bytes |= (blocksize-1);
- (*bytes)++;
- }
- len = offset - zerofrom;
- err = pagecache_write_begin(file, mapping, curpos, len,
- AOP_FLAG_UNINTERRUPTIBLE,
- &page, &fsdata);
- if (err)
- goto out;
- zero_user(page, zerofrom, len);
- err = pagecache_write_end(file, mapping, curpos, len, len,
- page, fsdata);
- if (err < 0)
- goto out;
- BUG_ON(err != len);
- err = 0;
- }
- out:
- return err;
- }
- int cont_write_begin(struct file *file, struct address_space *mapping,
- loff_t pos, unsigned len, unsigned flags,
- struct page **pagep, void **fsdata,
- get_block_t *get_block, loff_t *bytes)
- {
- struct inode *inode = mapping->host;
- unsigned int blocksize = i_blocksize(inode);
- unsigned int zerofrom;
- int err;
- err = cont_expand_zero(file, mapping, pos, bytes);
- if (err)
- return err;
- zerofrom = *bytes & ~PAGE_MASK;
- if (pos+len > *bytes && zerofrom & (blocksize-1)) {
- *bytes |= (blocksize-1);
- (*bytes)++;
- }
- return block_write_begin(mapping, pos, len, flags, pagep, get_block);
- }
- EXPORT_SYMBOL(cont_write_begin);
- int block_commit_write(struct page *page, unsigned from, unsigned to)
- {
- struct inode *inode = page->mapping->host;
- __block_commit_write(inode,page,from,to);
- return 0;
- }
- EXPORT_SYMBOL(block_commit_write);
- int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
- get_block_t get_block)
- {
- struct page *page = vmf->page;
- struct inode *inode = file_inode(vma->vm_file);
- unsigned long end;
- loff_t size;
- int ret;
- lock_page(page);
- size = i_size_read(inode);
- if ((page->mapping != inode->i_mapping) ||
- (page_offset(page) > size)) {
-
- ret = -EFAULT;
- goto out_unlock;
- }
-
- if (((page->index + 1) << PAGE_SHIFT) > size)
- end = size & ~PAGE_MASK;
- else
- end = PAGE_SIZE;
- ret = __block_write_begin(page, 0, end, get_block);
- if (!ret)
- ret = block_commit_write(page, 0, end);
- if (unlikely(ret < 0))
- goto out_unlock;
- set_page_dirty(page);
- wait_for_stable_page(page);
- return 0;
- out_unlock:
- unlock_page(page);
- return ret;
- }
- EXPORT_SYMBOL(block_page_mkwrite);
- static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
- {
- __end_buffer_read_notouch(bh, uptodate);
- }
- static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
- {
- struct buffer_head *bh;
- BUG_ON(!PageLocked(page));
- spin_lock(&page->mapping->private_lock);
- bh = head;
- do {
- if (PageDirty(page))
- set_buffer_dirty(bh);
- if (!bh->b_this_page)
- bh->b_this_page = head;
- bh = bh->b_this_page;
- } while (bh != head);
- attach_page_buffers(page, head);
- spin_unlock(&page->mapping->private_lock);
- }
- int nobh_write_begin(struct address_space *mapping,
- loff_t pos, unsigned len, unsigned flags,
- struct page **pagep, void **fsdata,
- get_block_t *get_block)
- {
- struct inode *inode = mapping->host;
- const unsigned blkbits = inode->i_blkbits;
- const unsigned blocksize = 1 << blkbits;
- struct buffer_head *head, *bh;
- struct page *page;
- pgoff_t index;
- unsigned from, to;
- unsigned block_in_page;
- unsigned block_start, block_end;
- sector_t block_in_file;
- int nr_reads = 0;
- int ret = 0;
- int is_mapped_to_disk = 1;
- index = pos >> PAGE_SHIFT;
- from = pos & (PAGE_SIZE - 1);
- to = from + len;
- page = grab_cache_page_write_begin(mapping, index, flags);
- if (!page)
- return -ENOMEM;
- *pagep = page;
- *fsdata = NULL;
- if (page_has_buffers(page)) {
- ret = __block_write_begin(page, pos, len, get_block);
- if (unlikely(ret))
- goto out_release;
- return ret;
- }
- if (PageMappedToDisk(page))
- return 0;
-
- head = alloc_page_buffers(page, blocksize, 0);
- if (!head) {
- ret = -ENOMEM;
- goto out_release;
- }
- block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
-
- for (block_start = 0, block_in_page = 0, bh = head;
- block_start < PAGE_SIZE;
- block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
- int create;
- block_end = block_start + blocksize;
- bh->b_state = 0;
- create = 1;
- if (block_start >= to)
- create = 0;
- ret = get_block(inode, block_in_file + block_in_page,
- bh, create);
- if (ret)
- goto failed;
- if (!buffer_mapped(bh))
- is_mapped_to_disk = 0;
- if (buffer_new(bh))
- unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
- if (PageUptodate(page)) {
- set_buffer_uptodate(bh);
- continue;
- }
- if (buffer_new(bh) || !buffer_mapped(bh)) {
- zero_user_segments(page, block_start, from,
- to, block_end);
- continue;
- }
- if (buffer_uptodate(bh))
- continue;
- if (block_start < from || block_end > to) {
- lock_buffer(bh);
- bh->b_end_io = end_buffer_read_nobh;
- submit_bh(REQ_OP_READ, 0, bh);
- nr_reads++;
- }
- }
- if (nr_reads) {
-
- for (bh = head; bh; bh = bh->b_this_page) {
- wait_on_buffer(bh);
- if (!buffer_uptodate(bh))
- ret = -EIO;
- }
- if (ret)
- goto failed;
- }
- if (is_mapped_to_disk)
- SetPageMappedToDisk(page);
- *fsdata = head;
- return 0;
- failed:
- BUG_ON(!ret);
-
- attach_nobh_buffers(page, head);
- page_zero_new_buffers(page, from, to);
- out_release:
- unlock_page(page);
- put_page(page);
- *pagep = NULL;
- return ret;
- }
- EXPORT_SYMBOL(nobh_write_begin);
- int nobh_write_end(struct file *file, struct address_space *mapping,
- loff_t pos, unsigned len, unsigned copied,
- struct page *page, void *fsdata)
- {
- struct inode *inode = page->mapping->host;
- struct buffer_head *head = fsdata;
- struct buffer_head *bh;
- BUG_ON(fsdata != NULL && page_has_buffers(page));
- if (unlikely(copied < len) && head)
- attach_nobh_buffers(page, head);
- if (page_has_buffers(page))
- return generic_write_end(file, mapping, pos, len,
- copied, page, fsdata);
- SetPageUptodate(page);
- set_page_dirty(page);
- if (pos+copied > inode->i_size) {
- i_size_write(inode, pos+copied);
- mark_inode_dirty(inode);
- }
- unlock_page(page);
- put_page(page);
- while (head) {
- bh = head;
- head = head->b_this_page;
- free_buffer_head(bh);
- }
- return copied;
- }
- EXPORT_SYMBOL(nobh_write_end);
- int nobh_writepage(struct page *page, get_block_t *get_block,
- struct writeback_control *wbc)
- {
- struct inode * const inode = page->mapping->host;
- loff_t i_size = i_size_read(inode);
- const pgoff_t end_index = i_size >> PAGE_SHIFT;
- unsigned offset;
- int ret;
-
- if (page->index < end_index)
- goto out;
-
- offset = i_size & (PAGE_SIZE-1);
- if (page->index >= end_index+1 || !offset) {
-
- #if 0
-
- if (page->mapping->a_ops->invalidatepage)
- page->mapping->a_ops->invalidatepage(page, offset);
- #endif
- unlock_page(page);
- return 0;
- }
-
- zero_user_segment(page, offset, PAGE_SIZE);
- out:
- ret = mpage_writepage(page, get_block, wbc);
- if (ret == -EAGAIN)
- ret = __block_write_full_page(inode, page, get_block, wbc,
- end_buffer_async_write);
- return ret;
- }
- EXPORT_SYMBOL(nobh_writepage);
- int nobh_truncate_page(struct address_space *mapping,
- loff_t from, get_block_t *get_block)
- {
- pgoff_t index = from >> PAGE_SHIFT;
- unsigned offset = from & (PAGE_SIZE-1);
- unsigned blocksize;
- sector_t iblock;
- unsigned length, pos;
- struct inode *inode = mapping->host;
- struct page *page;
- struct buffer_head map_bh;
- int err;
- blocksize = i_blocksize(inode);
- length = offset & (blocksize - 1);
-
- if (!length)
- return 0;
- length = blocksize - length;
- iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
- page = grab_cache_page(mapping, index);
- err = -ENOMEM;
- if (!page)
- goto out;
- if (page_has_buffers(page)) {
- has_buffers:
- unlock_page(page);
- put_page(page);
- return block_truncate_page(mapping, from, get_block);
- }
-
- pos = blocksize;
- while (offset >= pos) {
- iblock++;
- pos += blocksize;
- }
- map_bh.b_size = blocksize;
- map_bh.b_state = 0;
- err = get_block(inode, iblock, &map_bh, 0);
- if (err)
- goto unlock;
-
- if (!buffer_mapped(&map_bh))
- goto unlock;
-
- if (!PageUptodate(page)) {
- err = mapping->a_ops->readpage(NULL, page);
- if (err) {
- put_page(page);
- goto out;
- }
- lock_page(page);
- if (!PageUptodate(page)) {
- err = -EIO;
- goto unlock;
- }
- if (page_has_buffers(page))
- goto has_buffers;
- }
- zero_user(page, offset, length);
- set_page_dirty(page);
- err = 0;
- unlock:
- unlock_page(page);
- put_page(page);
- out:
- return err;
- }
- EXPORT_SYMBOL(nobh_truncate_page);
- int block_truncate_page(struct address_space *mapping,
- loff_t from, get_block_t *get_block)
- {
- pgoff_t index = from >> PAGE_SHIFT;
- unsigned offset = from & (PAGE_SIZE-1);
- unsigned blocksize;
- sector_t iblock;
- unsigned length, pos;
- struct inode *inode = mapping->host;
- struct page *page;
- struct buffer_head *bh;
- int err;
- blocksize = i_blocksize(inode);
- length = offset & (blocksize - 1);
-
- if (!length)
- return 0;
- length = blocksize - length;
- iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
-
- page = grab_cache_page(mapping, index);
- err = -ENOMEM;
- if (!page)
- goto out;
- if (!page_has_buffers(page))
- create_empty_buffers(page, blocksize, 0);
-
- bh = page_buffers(page);
- pos = blocksize;
- while (offset >= pos) {
- bh = bh->b_this_page;
- iblock++;
- pos += blocksize;
- }
- err = 0;
- if (!buffer_mapped(bh)) {
- WARN_ON(bh->b_size != blocksize);
- err = get_block(inode, iblock, bh, 0);
- if (err)
- goto unlock;
-
- if (!buffer_mapped(bh))
- goto unlock;
- }
-
- if (PageUptodate(page))
- set_buffer_uptodate(bh);
- if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
- err = -EIO;
- ll_rw_block(REQ_OP_READ, 0, 1, &bh);
- wait_on_buffer(bh);
-
- if (!buffer_uptodate(bh))
- goto unlock;
- }
- zero_user(page, offset, length);
- mark_buffer_dirty(bh);
- err = 0;
- unlock:
- unlock_page(page);
- put_page(page);
- out:
- return err;
- }
- EXPORT_SYMBOL(block_truncate_page);
- int block_write_full_page(struct page *page, get_block_t *get_block,
- struct writeback_control *wbc)
- {
- struct inode * const inode = page->mapping->host;
- loff_t i_size = i_size_read(inode);
- const pgoff_t end_index = i_size >> PAGE_SHIFT;
- unsigned offset;
-
- if (page->index < end_index)
- return __block_write_full_page(inode, page, get_block, wbc,
- end_buffer_async_write);
-
- offset = i_size & (PAGE_SIZE-1);
- if (page->index >= end_index+1 || !offset) {
-
- do_invalidatepage(page, 0, PAGE_SIZE);
- unlock_page(page);
- return 0;
- }
-
- zero_user_segment(page, offset, PAGE_SIZE);
- return __block_write_full_page(inode, page, get_block, wbc,
- end_buffer_async_write);
- }
- EXPORT_SYMBOL(block_write_full_page);
- sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
- get_block_t *get_block)
- {
- struct buffer_head tmp;
- struct inode *inode = mapping->host;
- tmp.b_state = 0;
- tmp.b_blocknr = 0;
- tmp.b_size = i_blocksize(inode);
- get_block(inode, block, &tmp, 0);
- return tmp.b_blocknr;
- }
- EXPORT_SYMBOL(generic_block_bmap);
- static void end_bio_bh_io_sync(struct bio *bio)
- {
- struct buffer_head *bh = bio->bi_private;
- if (unlikely(bio_flagged(bio, BIO_QUIET)))
- set_bit(BH_Quiet, &bh->b_state);
- bh->b_end_io(bh, !bio->bi_error);
- bio_put(bio);
- }
- void guard_bio_eod(int op, struct bio *bio)
- {
- sector_t maxsector;
- struct bio_vec *bvec = &bio->bi_io_vec[bio->bi_vcnt - 1];
- unsigned truncated_bytes;
- maxsector = i_size_read(bio->bi_bdev->bd_inode) >> 9;
- if (!maxsector)
- return;
-
- if (unlikely(bio->bi_iter.bi_sector >= maxsector))
- return;
- maxsector -= bio->bi_iter.bi_sector;
- if (likely((bio->bi_iter.bi_size >> 9) <= maxsector))
- return;
-
- truncated_bytes = bio->bi_iter.bi_size - (maxsector << 9);
-
- bio->bi_iter.bi_size -= truncated_bytes;
- bvec->bv_len -= truncated_bytes;
-
- if (op == REQ_OP_READ) {
- zero_user(bvec->bv_page, bvec->bv_offset + bvec->bv_len,
- truncated_bytes);
- }
- }
- static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
- unsigned long bio_flags, struct writeback_control *wbc)
- {
- struct bio *bio;
- BUG_ON(!buffer_locked(bh));
- BUG_ON(!buffer_mapped(bh));
- BUG_ON(!bh->b_end_io);
- BUG_ON(buffer_delay(bh));
- BUG_ON(buffer_unwritten(bh));
-
- if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
- clear_buffer_write_io_error(bh);
-
- bio = bio_alloc(GFP_NOIO, 1);
- if (wbc) {
- wbc_init_bio(wbc, bio);
- wbc_account_io(wbc, bh->b_page, bh->b_size);
- }
- bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
- bio->bi_bdev = bh->b_bdev;
- bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
- BUG_ON(bio->bi_iter.bi_size != bh->b_size);
- bio->bi_end_io = end_bio_bh_io_sync;
- bio->bi_private = bh;
- bio->bi_flags |= bio_flags;
-
- guard_bio_eod(op, bio);
- if (buffer_meta(bh))
- op_flags |= REQ_META;
- if (buffer_prio(bh))
- op_flags |= REQ_PRIO;
- bio_set_op_attrs(bio, op, op_flags);
- submit_bio(bio);
- return 0;
- }
- int _submit_bh(int op, int op_flags, struct buffer_head *bh,
- unsigned long bio_flags)
- {
- return submit_bh_wbc(op, op_flags, bh, bio_flags, NULL);
- }
- EXPORT_SYMBOL_GPL(_submit_bh);
- int submit_bh(int op, int op_flags, struct buffer_head *bh)
- {
- return submit_bh_wbc(op, op_flags, bh, 0, NULL);
- }
- EXPORT_SYMBOL(submit_bh);
- void ll_rw_block(int op, int op_flags, int nr, struct buffer_head *bhs[])
- {
- int i;
- for (i = 0; i < nr; i++) {
- struct buffer_head *bh = bhs[i];
- if (!trylock_buffer(bh))
- continue;
- if (op == WRITE) {
- if (test_clear_buffer_dirty(bh)) {
- bh->b_end_io = end_buffer_write_sync;
- get_bh(bh);
- submit_bh(op, op_flags, bh);
- continue;
- }
- } else {
- if (!buffer_uptodate(bh)) {
- bh->b_end_io = end_buffer_read_sync;
- get_bh(bh);
- submit_bh(op, op_flags, bh);
- continue;
- }
- }
- unlock_buffer(bh);
- }
- }
- EXPORT_SYMBOL(ll_rw_block);
- void write_dirty_buffer(struct buffer_head *bh, int op_flags)
- {
- lock_buffer(bh);
- if (!test_clear_buffer_dirty(bh)) {
- unlock_buffer(bh);
- return;
- }
- bh->b_end_io = end_buffer_write_sync;
- get_bh(bh);
- submit_bh(REQ_OP_WRITE, op_flags, bh);
- }
- EXPORT_SYMBOL(write_dirty_buffer);
- int __sync_dirty_buffer(struct buffer_head *bh, int op_flags)
- {
- int ret = 0;
- WARN_ON(atomic_read(&bh->b_count) < 1);
- lock_buffer(bh);
- if (test_clear_buffer_dirty(bh)) {
- get_bh(bh);
- bh->b_end_io = end_buffer_write_sync;
- ret = submit_bh(REQ_OP_WRITE, op_flags, bh);
- wait_on_buffer(bh);
- if (!ret && !buffer_uptodate(bh))
- ret = -EIO;
- } else {
- unlock_buffer(bh);
- }
- return ret;
- }
- EXPORT_SYMBOL(__sync_dirty_buffer);
- int sync_dirty_buffer(struct buffer_head *bh)
- {
- return __sync_dirty_buffer(bh, WRITE_SYNC);
- }
- EXPORT_SYMBOL(sync_dirty_buffer);
- static inline int buffer_busy(struct buffer_head *bh)
- {
- return atomic_read(&bh->b_count) |
- (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
- }
- static int
- drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
- {
- struct buffer_head *head = page_buffers(page);
- struct buffer_head *bh;
- bh = head;
- do {
- if (buffer_write_io_error(bh) && page->mapping)
- mapping_set_error(page->mapping, -EIO);
- if (buffer_busy(bh))
- goto failed;
- bh = bh->b_this_page;
- } while (bh != head);
- do {
- struct buffer_head *next = bh->b_this_page;
- if (bh->b_assoc_map)
- __remove_assoc_queue(bh);
- bh = next;
- } while (bh != head);
- *buffers_to_free = head;
- __clear_page_buffers(page);
- return 1;
- failed:
- return 0;
- }
- int try_to_free_buffers(struct page *page)
- {
- struct address_space * const mapping = page->mapping;
- struct buffer_head *buffers_to_free = NULL;
- int ret = 0;
- BUG_ON(!PageLocked(page));
- if (PageWriteback(page))
- return 0;
- if (mapping == NULL) {
- ret = drop_buffers(page, &buffers_to_free);
- goto out;
- }
- spin_lock(&mapping->private_lock);
- ret = drop_buffers(page, &buffers_to_free);
-
- if (ret)
- cancel_dirty_page(page);
- spin_unlock(&mapping->private_lock);
- out:
- if (buffers_to_free) {
- struct buffer_head *bh = buffers_to_free;
- do {
- struct buffer_head *next = bh->b_this_page;
- free_buffer_head(bh);
- bh = next;
- } while (bh != buffers_to_free);
- }
- return ret;
- }
- EXPORT_SYMBOL(try_to_free_buffers);
- SYSCALL_DEFINE2(bdflush, int, func, long, data)
- {
- static int msg_count;
- if (!capable(CAP_SYS_ADMIN))
- return -EPERM;
- if (msg_count < 5) {
- msg_count++;
- printk(KERN_INFO
- "warning: process `%s' used the obsolete bdflush"
- " system call\n", current->comm);
- printk(KERN_INFO "Fix your initscripts?\n");
- }
- if (func == 1)
- do_exit(0);
- return 0;
- }
- static struct kmem_cache *bh_cachep __read_mostly;
- static unsigned long max_buffer_heads;
- int buffer_heads_over_limit;
- struct bh_accounting {
- int nr;
- int ratelimit;
- };
- static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
- static void recalc_bh_state(void)
- {
- int i;
- int tot = 0;
- if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
- return;
- __this_cpu_write(bh_accounting.ratelimit, 0);
- for_each_online_cpu(i)
- tot += per_cpu(bh_accounting, i).nr;
- buffer_heads_over_limit = (tot > max_buffer_heads);
- }
- struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
- {
- struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
- if (ret) {
- INIT_LIST_HEAD(&ret->b_assoc_buffers);
- preempt_disable();
- __this_cpu_inc(bh_accounting.nr);
- recalc_bh_state();
- preempt_enable();
- }
- return ret;
- }
- EXPORT_SYMBOL(alloc_buffer_head);
- void free_buffer_head(struct buffer_head *bh)
- {
- BUG_ON(!list_empty(&bh->b_assoc_buffers));
- kmem_cache_free(bh_cachep, bh);
- preempt_disable();
- __this_cpu_dec(bh_accounting.nr);
- recalc_bh_state();
- preempt_enable();
- }
- EXPORT_SYMBOL(free_buffer_head);
- static void buffer_exit_cpu(int cpu)
- {
- int i;
- struct bh_lru *b = &per_cpu(bh_lrus, cpu);
- for (i = 0; i < BH_LRU_SIZE; i++) {
- brelse(b->bhs[i]);
- b->bhs[i] = NULL;
- }
- this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
- per_cpu(bh_accounting, cpu).nr = 0;
- }
- static int buffer_cpu_notify(struct notifier_block *self,
- unsigned long action, void *hcpu)
- {
- if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
- buffer_exit_cpu((unsigned long)hcpu);
- return NOTIFY_OK;
- }
- int bh_uptodate_or_lock(struct buffer_head *bh)
- {
- if (!buffer_uptodate(bh)) {
- lock_buffer(bh);
- if (!buffer_uptodate(bh))
- return 0;
- unlock_buffer(bh);
- }
- return 1;
- }
- EXPORT_SYMBOL(bh_uptodate_or_lock);
- int bh_submit_read(struct buffer_head *bh)
- {
- BUG_ON(!buffer_locked(bh));
- if (buffer_uptodate(bh)) {
- unlock_buffer(bh);
- return 0;
- }
- get_bh(bh);
- bh->b_end_io = end_buffer_read_sync;
- submit_bh(REQ_OP_READ, 0, bh);
- wait_on_buffer(bh);
- if (buffer_uptodate(bh))
- return 0;
- return -EIO;
- }
- EXPORT_SYMBOL(bh_submit_read);
- void __init buffer_init(void)
- {
- unsigned long nrpages;
- bh_cachep = kmem_cache_create("buffer_head",
- sizeof(struct buffer_head), 0,
- (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
- SLAB_MEM_SPREAD),
- NULL);
-
- nrpages = (nr_free_buffer_pages() * 10) / 100;
- max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
- hotcpu_notifier(buffer_cpu_notify, 0);
- }
|