dm-snap.c 68 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879
  1. /*
  2. * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
  3. *
  4. * This file is released under the GPL.
  5. */
  6. #include <linux/blkdev.h>
  7. #include <linux/device-mapper.h>
  8. #include <linux/delay.h>
  9. #include <linux/fs.h>
  10. #include <linux/init.h>
  11. #include <linux/kdev_t.h>
  12. #include <linux/list.h>
  13. #include <linux/list_bl.h>
  14. #include <linux/mempool.h>
  15. #include <linux/module.h>
  16. #include <linux/slab.h>
  17. #include <linux/vmalloc.h>
  18. #include <linux/log2.h>
  19. #include <linux/dm-kcopyd.h>
  20. #include "dm.h"
  21. #include "dm-exception-store.h"
  22. #define DM_MSG_PREFIX "snapshots"
  23. static const char dm_snapshot_merge_target_name[] = "snapshot-merge";
  24. #define dm_target_is_snapshot_merge(ti) \
  25. ((ti)->type->name == dm_snapshot_merge_target_name)
  26. /*
  27. * The size of the mempool used to track chunks in use.
  28. */
  29. #define MIN_IOS 256
  30. #define DM_TRACKED_CHUNK_HASH_SIZE 16
  31. #define DM_TRACKED_CHUNK_HASH(x) ((unsigned long)(x) & \
  32. (DM_TRACKED_CHUNK_HASH_SIZE - 1))
  33. struct dm_exception_table {
  34. uint32_t hash_mask;
  35. unsigned hash_shift;
  36. struct hlist_bl_head *table;
  37. };
  38. struct dm_snapshot {
  39. struct rw_semaphore lock;
  40. struct dm_dev *origin;
  41. struct dm_dev *cow;
  42. struct dm_target *ti;
  43. /* List of snapshots per Origin */
  44. struct list_head list;
  45. /*
  46. * You can't use a snapshot if this is 0 (e.g. if full).
  47. * A snapshot-merge target never clears this.
  48. */
  49. int valid;
  50. /*
  51. * The snapshot overflowed because of a write to the snapshot device.
  52. * We don't have to invalidate the snapshot in this case, but we need
  53. * to prevent further writes.
  54. */
  55. int snapshot_overflowed;
  56. /* Origin writes don't trigger exceptions until this is set */
  57. int active;
  58. atomic_t pending_exceptions_count;
  59. spinlock_t pe_allocation_lock;
  60. /* Protected by "pe_allocation_lock" */
  61. sector_t exception_start_sequence;
  62. /* Protected by kcopyd single-threaded callback */
  63. sector_t exception_complete_sequence;
  64. /*
  65. * A list of pending exceptions that completed out of order.
  66. * Protected by kcopyd single-threaded callback.
  67. */
  68. struct rb_root out_of_order_tree;
  69. mempool_t pending_pool;
  70. struct dm_exception_table pending;
  71. struct dm_exception_table complete;
  72. /*
  73. * pe_lock protects all pending_exception operations and access
  74. * as well as the snapshot_bios list.
  75. */
  76. spinlock_t pe_lock;
  77. /* Chunks with outstanding reads */
  78. spinlock_t tracked_chunk_lock;
  79. struct hlist_head tracked_chunk_hash[DM_TRACKED_CHUNK_HASH_SIZE];
  80. /* The on disk metadata handler */
  81. struct dm_exception_store *store;
  82. unsigned in_progress;
  83. struct wait_queue_head in_progress_wait;
  84. struct dm_kcopyd_client *kcopyd_client;
  85. /* Wait for events based on state_bits */
  86. unsigned long state_bits;
  87. /* Range of chunks currently being merged. */
  88. chunk_t first_merging_chunk;
  89. int num_merging_chunks;
  90. /*
  91. * The merge operation failed if this flag is set.
  92. * Failure modes are handled as follows:
  93. * - I/O error reading the header
  94. * => don't load the target; abort.
  95. * - Header does not have "valid" flag set
  96. * => use the origin; forget about the snapshot.
  97. * - I/O error when reading exceptions
  98. * => don't load the target; abort.
  99. * (We can't use the intermediate origin state.)
  100. * - I/O error while merging
  101. * => stop merging; set merge_failed; process I/O normally.
  102. */
  103. bool merge_failed:1;
  104. bool discard_zeroes_cow:1;
  105. bool discard_passdown_origin:1;
  106. /*
  107. * Incoming bios that overlap with chunks being merged must wait
  108. * for them to be committed.
  109. */
  110. struct bio_list bios_queued_during_merge;
  111. /*
  112. * Flush data after merge.
  113. */
  114. struct bio flush_bio;
  115. };
  116. /*
  117. * state_bits:
  118. * RUNNING_MERGE - Merge operation is in progress.
  119. * SHUTDOWN_MERGE - Set to signal that merge needs to be stopped;
  120. * cleared afterwards.
  121. */
  122. #define RUNNING_MERGE 0
  123. #define SHUTDOWN_MERGE 1
  124. /*
  125. * Maximum number of chunks being copied on write.
  126. *
  127. * The value was decided experimentally as a trade-off between memory
  128. * consumption, stalling the kernel's workqueues and maintaining a high enough
  129. * throughput.
  130. */
  131. #define DEFAULT_COW_THRESHOLD 2048
  132. static unsigned cow_threshold = DEFAULT_COW_THRESHOLD;
  133. module_param_named(snapshot_cow_threshold, cow_threshold, uint, 0644);
  134. MODULE_PARM_DESC(snapshot_cow_threshold, "Maximum number of chunks being copied on write");
  135. DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle,
  136. "A percentage of time allocated for copy on write");
  137. struct dm_dev *dm_snap_origin(struct dm_snapshot *s)
  138. {
  139. return s->origin;
  140. }
  141. EXPORT_SYMBOL(dm_snap_origin);
  142. struct dm_dev *dm_snap_cow(struct dm_snapshot *s)
  143. {
  144. return s->cow;
  145. }
  146. EXPORT_SYMBOL(dm_snap_cow);
  147. static sector_t chunk_to_sector(struct dm_exception_store *store,
  148. chunk_t chunk)
  149. {
  150. return chunk << store->chunk_shift;
  151. }
  152. static int bdev_equal(struct block_device *lhs, struct block_device *rhs)
  153. {
  154. /*
  155. * There is only ever one instance of a particular block
  156. * device so we can compare pointers safely.
  157. */
  158. return lhs == rhs;
  159. }
  160. struct dm_snap_pending_exception {
  161. struct dm_exception e;
  162. /*
  163. * Origin buffers waiting for this to complete are held
  164. * in a bio list
  165. */
  166. struct bio_list origin_bios;
  167. struct bio_list snapshot_bios;
  168. /* Pointer back to snapshot context */
  169. struct dm_snapshot *snap;
  170. /*
  171. * 1 indicates the exception has already been sent to
  172. * kcopyd.
  173. */
  174. int started;
  175. /* There was copying error. */
  176. int copy_error;
  177. /* A sequence number, it is used for in-order completion. */
  178. sector_t exception_sequence;
  179. struct rb_node out_of_order_node;
  180. /*
  181. * For writing a complete chunk, bypassing the copy.
  182. */
  183. struct bio *full_bio;
  184. bio_end_io_t *full_bio_end_io;
  185. };
  186. /*
  187. * Hash table mapping origin volumes to lists of snapshots and
  188. * a lock to protect it
  189. */
  190. static struct kmem_cache *exception_cache;
  191. static struct kmem_cache *pending_cache;
  192. struct dm_snap_tracked_chunk {
  193. struct hlist_node node;
  194. chunk_t chunk;
  195. };
  196. static void init_tracked_chunk(struct bio *bio)
  197. {
  198. struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
  199. INIT_HLIST_NODE(&c->node);
  200. }
  201. static bool is_bio_tracked(struct bio *bio)
  202. {
  203. struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
  204. return !hlist_unhashed(&c->node);
  205. }
  206. static void track_chunk(struct dm_snapshot *s, struct bio *bio, chunk_t chunk)
  207. {
  208. struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
  209. c->chunk = chunk;
  210. spin_lock_irq(&s->tracked_chunk_lock);
  211. hlist_add_head(&c->node,
  212. &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)]);
  213. spin_unlock_irq(&s->tracked_chunk_lock);
  214. }
  215. static void stop_tracking_chunk(struct dm_snapshot *s, struct bio *bio)
  216. {
  217. struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
  218. unsigned long flags;
  219. spin_lock_irqsave(&s->tracked_chunk_lock, flags);
  220. hlist_del(&c->node);
  221. spin_unlock_irqrestore(&s->tracked_chunk_lock, flags);
  222. }
  223. static int __chunk_is_tracked(struct dm_snapshot *s, chunk_t chunk)
  224. {
  225. struct dm_snap_tracked_chunk *c;
  226. int found = 0;
  227. spin_lock_irq(&s->tracked_chunk_lock);
  228. hlist_for_each_entry(c,
  229. &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)], node) {
  230. if (c->chunk == chunk) {
  231. found = 1;
  232. break;
  233. }
  234. }
  235. spin_unlock_irq(&s->tracked_chunk_lock);
  236. return found;
  237. }
  238. /*
  239. * This conflicting I/O is extremely improbable in the caller,
  240. * so msleep(1) is sufficient and there is no need for a wait queue.
  241. */
  242. static void __check_for_conflicting_io(struct dm_snapshot *s, chunk_t chunk)
  243. {
  244. while (__chunk_is_tracked(s, chunk))
  245. msleep(1);
  246. }
  247. /*
  248. * One of these per registered origin, held in the snapshot_origins hash
  249. */
  250. struct origin {
  251. /* The origin device */
  252. struct block_device *bdev;
  253. struct list_head hash_list;
  254. /* List of snapshots for this origin */
  255. struct list_head snapshots;
  256. };
  257. /*
  258. * This structure is allocated for each origin target
  259. */
  260. struct dm_origin {
  261. struct dm_dev *dev;
  262. struct dm_target *ti;
  263. unsigned split_boundary;
  264. struct list_head hash_list;
  265. };
  266. /*
  267. * Size of the hash table for origin volumes. If we make this
  268. * the size of the minors list then it should be nearly perfect
  269. */
  270. #define ORIGIN_HASH_SIZE 256
  271. #define ORIGIN_MASK 0xFF
  272. static struct list_head *_origins;
  273. static struct list_head *_dm_origins;
  274. static struct rw_semaphore _origins_lock;
  275. static DECLARE_WAIT_QUEUE_HEAD(_pending_exceptions_done);
  276. static DEFINE_SPINLOCK(_pending_exceptions_done_spinlock);
  277. static uint64_t _pending_exceptions_done_count;
  278. static int init_origin_hash(void)
  279. {
  280. int i;
  281. _origins = kmalloc_array(ORIGIN_HASH_SIZE, sizeof(struct list_head),
  282. GFP_KERNEL);
  283. if (!_origins) {
  284. DMERR("unable to allocate memory for _origins");
  285. return -ENOMEM;
  286. }
  287. for (i = 0; i < ORIGIN_HASH_SIZE; i++)
  288. INIT_LIST_HEAD(_origins + i);
  289. _dm_origins = kmalloc_array(ORIGIN_HASH_SIZE,
  290. sizeof(struct list_head),
  291. GFP_KERNEL);
  292. if (!_dm_origins) {
  293. DMERR("unable to allocate memory for _dm_origins");
  294. kfree(_origins);
  295. return -ENOMEM;
  296. }
  297. for (i = 0; i < ORIGIN_HASH_SIZE; i++)
  298. INIT_LIST_HEAD(_dm_origins + i);
  299. init_rwsem(&_origins_lock);
  300. return 0;
  301. }
  302. static void exit_origin_hash(void)
  303. {
  304. kfree(_origins);
  305. kfree(_dm_origins);
  306. }
  307. static unsigned origin_hash(struct block_device *bdev)
  308. {
  309. return bdev->bd_dev & ORIGIN_MASK;
  310. }
  311. static struct origin *__lookup_origin(struct block_device *origin)
  312. {
  313. struct list_head *ol;
  314. struct origin *o;
  315. ol = &_origins[origin_hash(origin)];
  316. list_for_each_entry (o, ol, hash_list)
  317. if (bdev_equal(o->bdev, origin))
  318. return o;
  319. return NULL;
  320. }
  321. static void __insert_origin(struct origin *o)
  322. {
  323. struct list_head *sl = &_origins[origin_hash(o->bdev)];
  324. list_add_tail(&o->hash_list, sl);
  325. }
  326. static struct dm_origin *__lookup_dm_origin(struct block_device *origin)
  327. {
  328. struct list_head *ol;
  329. struct dm_origin *o;
  330. ol = &_dm_origins[origin_hash(origin)];
  331. list_for_each_entry (o, ol, hash_list)
  332. if (bdev_equal(o->dev->bdev, origin))
  333. return o;
  334. return NULL;
  335. }
  336. static void __insert_dm_origin(struct dm_origin *o)
  337. {
  338. struct list_head *sl = &_dm_origins[origin_hash(o->dev->bdev)];
  339. list_add_tail(&o->hash_list, sl);
  340. }
  341. static void __remove_dm_origin(struct dm_origin *o)
  342. {
  343. list_del(&o->hash_list);
  344. }
  345. /*
  346. * _origins_lock must be held when calling this function.
  347. * Returns number of snapshots registered using the supplied cow device, plus:
  348. * snap_src - a snapshot suitable for use as a source of exception handover
  349. * snap_dest - a snapshot capable of receiving exception handover.
  350. * snap_merge - an existing snapshot-merge target linked to the same origin.
  351. * There can be at most one snapshot-merge target. The parameter is optional.
  352. *
  353. * Possible return values and states of snap_src and snap_dest.
  354. * 0: NULL, NULL - first new snapshot
  355. * 1: snap_src, NULL - normal snapshot
  356. * 2: snap_src, snap_dest - waiting for handover
  357. * 2: snap_src, NULL - handed over, waiting for old to be deleted
  358. * 1: NULL, snap_dest - source got destroyed without handover
  359. */
  360. static int __find_snapshots_sharing_cow(struct dm_snapshot *snap,
  361. struct dm_snapshot **snap_src,
  362. struct dm_snapshot **snap_dest,
  363. struct dm_snapshot **snap_merge)
  364. {
  365. struct dm_snapshot *s;
  366. struct origin *o;
  367. int count = 0;
  368. int active;
  369. o = __lookup_origin(snap->origin->bdev);
  370. if (!o)
  371. goto out;
  372. list_for_each_entry(s, &o->snapshots, list) {
  373. if (dm_target_is_snapshot_merge(s->ti) && snap_merge)
  374. *snap_merge = s;
  375. if (!bdev_equal(s->cow->bdev, snap->cow->bdev))
  376. continue;
  377. down_read(&s->lock);
  378. active = s->active;
  379. up_read(&s->lock);
  380. if (active) {
  381. if (snap_src)
  382. *snap_src = s;
  383. } else if (snap_dest)
  384. *snap_dest = s;
  385. count++;
  386. }
  387. out:
  388. return count;
  389. }
  390. /*
  391. * On success, returns 1 if this snapshot is a handover destination,
  392. * otherwise returns 0.
  393. */
  394. static int __validate_exception_handover(struct dm_snapshot *snap)
  395. {
  396. struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
  397. struct dm_snapshot *snap_merge = NULL;
  398. /* Does snapshot need exceptions handed over to it? */
  399. if ((__find_snapshots_sharing_cow(snap, &snap_src, &snap_dest,
  400. &snap_merge) == 2) ||
  401. snap_dest) {
  402. snap->ti->error = "Snapshot cow pairing for exception "
  403. "table handover failed";
  404. return -EINVAL;
  405. }
  406. /*
  407. * If no snap_src was found, snap cannot become a handover
  408. * destination.
  409. */
  410. if (!snap_src)
  411. return 0;
  412. /*
  413. * Non-snapshot-merge handover?
  414. */
  415. if (!dm_target_is_snapshot_merge(snap->ti))
  416. return 1;
  417. /*
  418. * Do not allow more than one merging snapshot.
  419. */
  420. if (snap_merge) {
  421. snap->ti->error = "A snapshot is already merging.";
  422. return -EINVAL;
  423. }
  424. if (!snap_src->store->type->prepare_merge ||
  425. !snap_src->store->type->commit_merge) {
  426. snap->ti->error = "Snapshot exception store does not "
  427. "support snapshot-merge.";
  428. return -EINVAL;
  429. }
  430. return 1;
  431. }
  432. static void __insert_snapshot(struct origin *o, struct dm_snapshot *s)
  433. {
  434. struct dm_snapshot *l;
  435. /* Sort the list according to chunk size, largest-first smallest-last */
  436. list_for_each_entry(l, &o->snapshots, list)
  437. if (l->store->chunk_size < s->store->chunk_size)
  438. break;
  439. list_add_tail(&s->list, &l->list);
  440. }
  441. /*
  442. * Make a note of the snapshot and its origin so we can look it
  443. * up when the origin has a write on it.
  444. *
  445. * Also validate snapshot exception store handovers.
  446. * On success, returns 1 if this registration is a handover destination,
  447. * otherwise returns 0.
  448. */
  449. static int register_snapshot(struct dm_snapshot *snap)
  450. {
  451. struct origin *o, *new_o = NULL;
  452. struct block_device *bdev = snap->origin->bdev;
  453. int r = 0;
  454. new_o = kmalloc(sizeof(*new_o), GFP_KERNEL);
  455. if (!new_o)
  456. return -ENOMEM;
  457. down_write(&_origins_lock);
  458. r = __validate_exception_handover(snap);
  459. if (r < 0) {
  460. kfree(new_o);
  461. goto out;
  462. }
  463. o = __lookup_origin(bdev);
  464. if (o)
  465. kfree(new_o);
  466. else {
  467. /* New origin */
  468. o = new_o;
  469. /* Initialise the struct */
  470. INIT_LIST_HEAD(&o->snapshots);
  471. o->bdev = bdev;
  472. __insert_origin(o);
  473. }
  474. __insert_snapshot(o, snap);
  475. out:
  476. up_write(&_origins_lock);
  477. return r;
  478. }
  479. /*
  480. * Move snapshot to correct place in list according to chunk size.
  481. */
  482. static void reregister_snapshot(struct dm_snapshot *s)
  483. {
  484. struct block_device *bdev = s->origin->bdev;
  485. down_write(&_origins_lock);
  486. list_del(&s->list);
  487. __insert_snapshot(__lookup_origin(bdev), s);
  488. up_write(&_origins_lock);
  489. }
  490. static void unregister_snapshot(struct dm_snapshot *s)
  491. {
  492. struct origin *o;
  493. down_write(&_origins_lock);
  494. o = __lookup_origin(s->origin->bdev);
  495. list_del(&s->list);
  496. if (o && list_empty(&o->snapshots)) {
  497. list_del(&o->hash_list);
  498. kfree(o);
  499. }
  500. up_write(&_origins_lock);
  501. }
  502. /*
  503. * Implementation of the exception hash tables.
  504. * The lowest hash_shift bits of the chunk number are ignored, allowing
  505. * some consecutive chunks to be grouped together.
  506. */
  507. static uint32_t exception_hash(struct dm_exception_table *et, chunk_t chunk);
  508. /* Lock to protect access to the completed and pending exception hash tables. */
  509. struct dm_exception_table_lock {
  510. struct hlist_bl_head *complete_slot;
  511. struct hlist_bl_head *pending_slot;
  512. };
  513. static void dm_exception_table_lock_init(struct dm_snapshot *s, chunk_t chunk,
  514. struct dm_exception_table_lock *lock)
  515. {
  516. struct dm_exception_table *complete = &s->complete;
  517. struct dm_exception_table *pending = &s->pending;
  518. lock->complete_slot = &complete->table[exception_hash(complete, chunk)];
  519. lock->pending_slot = &pending->table[exception_hash(pending, chunk)];
  520. }
  521. static void dm_exception_table_lock(struct dm_exception_table_lock *lock)
  522. {
  523. hlist_bl_lock(lock->complete_slot);
  524. hlist_bl_lock(lock->pending_slot);
  525. }
  526. static void dm_exception_table_unlock(struct dm_exception_table_lock *lock)
  527. {
  528. hlist_bl_unlock(lock->pending_slot);
  529. hlist_bl_unlock(lock->complete_slot);
  530. }
  531. static int dm_exception_table_init(struct dm_exception_table *et,
  532. uint32_t size, unsigned hash_shift)
  533. {
  534. unsigned int i;
  535. et->hash_shift = hash_shift;
  536. et->hash_mask = size - 1;
  537. et->table = dm_vcalloc(size, sizeof(struct hlist_bl_head));
  538. if (!et->table)
  539. return -ENOMEM;
  540. for (i = 0; i < size; i++)
  541. INIT_HLIST_BL_HEAD(et->table + i);
  542. return 0;
  543. }
  544. static void dm_exception_table_exit(struct dm_exception_table *et,
  545. struct kmem_cache *mem)
  546. {
  547. struct hlist_bl_head *slot;
  548. struct dm_exception *ex;
  549. struct hlist_bl_node *pos, *n;
  550. int i, size;
  551. size = et->hash_mask + 1;
  552. for (i = 0; i < size; i++) {
  553. slot = et->table + i;
  554. hlist_bl_for_each_entry_safe(ex, pos, n, slot, hash_list)
  555. kmem_cache_free(mem, ex);
  556. }
  557. vfree(et->table);
  558. }
  559. static uint32_t exception_hash(struct dm_exception_table *et, chunk_t chunk)
  560. {
  561. return (chunk >> et->hash_shift) & et->hash_mask;
  562. }
  563. static void dm_remove_exception(struct dm_exception *e)
  564. {
  565. hlist_bl_del(&e->hash_list);
  566. }
  567. /*
  568. * Return the exception data for a sector, or NULL if not
  569. * remapped.
  570. */
  571. static struct dm_exception *dm_lookup_exception(struct dm_exception_table *et,
  572. chunk_t chunk)
  573. {
  574. struct hlist_bl_head *slot;
  575. struct hlist_bl_node *pos;
  576. struct dm_exception *e;
  577. slot = &et->table[exception_hash(et, chunk)];
  578. hlist_bl_for_each_entry(e, pos, slot, hash_list)
  579. if (chunk >= e->old_chunk &&
  580. chunk <= e->old_chunk + dm_consecutive_chunk_count(e))
  581. return e;
  582. return NULL;
  583. }
  584. static struct dm_exception *alloc_completed_exception(gfp_t gfp)
  585. {
  586. struct dm_exception *e;
  587. e = kmem_cache_alloc(exception_cache, gfp);
  588. if (!e && gfp == GFP_NOIO)
  589. e = kmem_cache_alloc(exception_cache, GFP_ATOMIC);
  590. return e;
  591. }
  592. static void free_completed_exception(struct dm_exception *e)
  593. {
  594. kmem_cache_free(exception_cache, e);
  595. }
  596. static struct dm_snap_pending_exception *alloc_pending_exception(struct dm_snapshot *s)
  597. {
  598. struct dm_snap_pending_exception *pe = mempool_alloc(&s->pending_pool,
  599. GFP_NOIO);
  600. atomic_inc(&s->pending_exceptions_count);
  601. pe->snap = s;
  602. return pe;
  603. }
  604. static void free_pending_exception(struct dm_snap_pending_exception *pe)
  605. {
  606. struct dm_snapshot *s = pe->snap;
  607. mempool_free(pe, &s->pending_pool);
  608. smp_mb__before_atomic();
  609. atomic_dec(&s->pending_exceptions_count);
  610. }
  611. static void dm_insert_exception(struct dm_exception_table *eh,
  612. struct dm_exception *new_e)
  613. {
  614. struct hlist_bl_head *l;
  615. struct hlist_bl_node *pos;
  616. struct dm_exception *e = NULL;
  617. l = &eh->table[exception_hash(eh, new_e->old_chunk)];
  618. /* Add immediately if this table doesn't support consecutive chunks */
  619. if (!eh->hash_shift)
  620. goto out;
  621. /* List is ordered by old_chunk */
  622. hlist_bl_for_each_entry(e, pos, l, hash_list) {
  623. /* Insert after an existing chunk? */
  624. if (new_e->old_chunk == (e->old_chunk +
  625. dm_consecutive_chunk_count(e) + 1) &&
  626. new_e->new_chunk == (dm_chunk_number(e->new_chunk) +
  627. dm_consecutive_chunk_count(e) + 1)) {
  628. dm_consecutive_chunk_count_inc(e);
  629. free_completed_exception(new_e);
  630. return;
  631. }
  632. /* Insert before an existing chunk? */
  633. if (new_e->old_chunk == (e->old_chunk - 1) &&
  634. new_e->new_chunk == (dm_chunk_number(e->new_chunk) - 1)) {
  635. dm_consecutive_chunk_count_inc(e);
  636. e->old_chunk--;
  637. e->new_chunk--;
  638. free_completed_exception(new_e);
  639. return;
  640. }
  641. if (new_e->old_chunk < e->old_chunk)
  642. break;
  643. }
  644. out:
  645. if (!e) {
  646. /*
  647. * Either the table doesn't support consecutive chunks or slot
  648. * l is empty.
  649. */
  650. hlist_bl_add_head(&new_e->hash_list, l);
  651. } else if (new_e->old_chunk < e->old_chunk) {
  652. /* Add before an existing exception */
  653. hlist_bl_add_before(&new_e->hash_list, &e->hash_list);
  654. } else {
  655. /* Add to l's tail: e is the last exception in this slot */
  656. hlist_bl_add_behind(&new_e->hash_list, &e->hash_list);
  657. }
  658. }
  659. /*
  660. * Callback used by the exception stores to load exceptions when
  661. * initialising.
  662. */
  663. static int dm_add_exception(void *context, chunk_t old, chunk_t new)
  664. {
  665. struct dm_exception_table_lock lock;
  666. struct dm_snapshot *s = context;
  667. struct dm_exception *e;
  668. e = alloc_completed_exception(GFP_KERNEL);
  669. if (!e)
  670. return -ENOMEM;
  671. e->old_chunk = old;
  672. /* Consecutive_count is implicitly initialised to zero */
  673. e->new_chunk = new;
  674. /*
  675. * Although there is no need to lock access to the exception tables
  676. * here, if we don't then hlist_bl_add_head(), called by
  677. * dm_insert_exception(), will complain about accessing the
  678. * corresponding list without locking it first.
  679. */
  680. dm_exception_table_lock_init(s, old, &lock);
  681. dm_exception_table_lock(&lock);
  682. dm_insert_exception(&s->complete, e);
  683. dm_exception_table_unlock(&lock);
  684. return 0;
  685. }
  686. /*
  687. * Return a minimum chunk size of all snapshots that have the specified origin.
  688. * Return zero if the origin has no snapshots.
  689. */
  690. static uint32_t __minimum_chunk_size(struct origin *o)
  691. {
  692. struct dm_snapshot *snap;
  693. unsigned chunk_size = rounddown_pow_of_two(UINT_MAX);
  694. if (o)
  695. list_for_each_entry(snap, &o->snapshots, list)
  696. chunk_size = min_not_zero(chunk_size,
  697. snap->store->chunk_size);
  698. return (uint32_t) chunk_size;
  699. }
  700. /*
  701. * Hard coded magic.
  702. */
  703. static int calc_max_buckets(void)
  704. {
  705. /* use a fixed size of 2MB */
  706. unsigned long mem = 2 * 1024 * 1024;
  707. mem /= sizeof(struct hlist_bl_head);
  708. return mem;
  709. }
  710. /*
  711. * Allocate room for a suitable hash table.
  712. */
  713. static int init_hash_tables(struct dm_snapshot *s)
  714. {
  715. sector_t hash_size, cow_dev_size, max_buckets;
  716. /*
  717. * Calculate based on the size of the original volume or
  718. * the COW volume...
  719. */
  720. cow_dev_size = get_dev_size(s->cow->bdev);
  721. max_buckets = calc_max_buckets();
  722. hash_size = cow_dev_size >> s->store->chunk_shift;
  723. hash_size = min(hash_size, max_buckets);
  724. if (hash_size < 64)
  725. hash_size = 64;
  726. hash_size = rounddown_pow_of_two(hash_size);
  727. if (dm_exception_table_init(&s->complete, hash_size,
  728. DM_CHUNK_CONSECUTIVE_BITS))
  729. return -ENOMEM;
  730. /*
  731. * Allocate hash table for in-flight exceptions
  732. * Make this smaller than the real hash table
  733. */
  734. hash_size >>= 3;
  735. if (hash_size < 64)
  736. hash_size = 64;
  737. if (dm_exception_table_init(&s->pending, hash_size, 0)) {
  738. dm_exception_table_exit(&s->complete, exception_cache);
  739. return -ENOMEM;
  740. }
  741. return 0;
  742. }
  743. static void merge_shutdown(struct dm_snapshot *s)
  744. {
  745. clear_bit_unlock(RUNNING_MERGE, &s->state_bits);
  746. smp_mb__after_atomic();
  747. wake_up_bit(&s->state_bits, RUNNING_MERGE);
  748. }
  749. static struct bio *__release_queued_bios_after_merge(struct dm_snapshot *s)
  750. {
  751. s->first_merging_chunk = 0;
  752. s->num_merging_chunks = 0;
  753. return bio_list_get(&s->bios_queued_during_merge);
  754. }
  755. /*
  756. * Remove one chunk from the index of completed exceptions.
  757. */
  758. static int __remove_single_exception_chunk(struct dm_snapshot *s,
  759. chunk_t old_chunk)
  760. {
  761. struct dm_exception *e;
  762. e = dm_lookup_exception(&s->complete, old_chunk);
  763. if (!e) {
  764. DMERR("Corruption detected: exception for block %llu is "
  765. "on disk but not in memory",
  766. (unsigned long long)old_chunk);
  767. return -EINVAL;
  768. }
  769. /*
  770. * If this is the only chunk using this exception, remove exception.
  771. */
  772. if (!dm_consecutive_chunk_count(e)) {
  773. dm_remove_exception(e);
  774. free_completed_exception(e);
  775. return 0;
  776. }
  777. /*
  778. * The chunk may be either at the beginning or the end of a
  779. * group of consecutive chunks - never in the middle. We are
  780. * removing chunks in the opposite order to that in which they
  781. * were added, so this should always be true.
  782. * Decrement the consecutive chunk counter and adjust the
  783. * starting point if necessary.
  784. */
  785. if (old_chunk == e->old_chunk) {
  786. e->old_chunk++;
  787. e->new_chunk++;
  788. } else if (old_chunk != e->old_chunk +
  789. dm_consecutive_chunk_count(e)) {
  790. DMERR("Attempt to merge block %llu from the "
  791. "middle of a chunk range [%llu - %llu]",
  792. (unsigned long long)old_chunk,
  793. (unsigned long long)e->old_chunk,
  794. (unsigned long long)
  795. e->old_chunk + dm_consecutive_chunk_count(e));
  796. return -EINVAL;
  797. }
  798. dm_consecutive_chunk_count_dec(e);
  799. return 0;
  800. }
  801. static void flush_bios(struct bio *bio);
  802. static int remove_single_exception_chunk(struct dm_snapshot *s)
  803. {
  804. struct bio *b = NULL;
  805. int r;
  806. chunk_t old_chunk = s->first_merging_chunk + s->num_merging_chunks - 1;
  807. down_write(&s->lock);
  808. /*
  809. * Process chunks (and associated exceptions) in reverse order
  810. * so that dm_consecutive_chunk_count_dec() accounting works.
  811. */
  812. do {
  813. r = __remove_single_exception_chunk(s, old_chunk);
  814. if (r)
  815. goto out;
  816. } while (old_chunk-- > s->first_merging_chunk);
  817. b = __release_queued_bios_after_merge(s);
  818. out:
  819. up_write(&s->lock);
  820. if (b)
  821. flush_bios(b);
  822. return r;
  823. }
  824. static int origin_write_extent(struct dm_snapshot *merging_snap,
  825. sector_t sector, unsigned chunk_size);
  826. static void merge_callback(int read_err, unsigned long write_err,
  827. void *context);
  828. static uint64_t read_pending_exceptions_done_count(void)
  829. {
  830. uint64_t pending_exceptions_done;
  831. spin_lock(&_pending_exceptions_done_spinlock);
  832. pending_exceptions_done = _pending_exceptions_done_count;
  833. spin_unlock(&_pending_exceptions_done_spinlock);
  834. return pending_exceptions_done;
  835. }
  836. static void increment_pending_exceptions_done_count(void)
  837. {
  838. spin_lock(&_pending_exceptions_done_spinlock);
  839. _pending_exceptions_done_count++;
  840. spin_unlock(&_pending_exceptions_done_spinlock);
  841. wake_up_all(&_pending_exceptions_done);
  842. }
  843. static void snapshot_merge_next_chunks(struct dm_snapshot *s)
  844. {
  845. int i, linear_chunks;
  846. chunk_t old_chunk, new_chunk;
  847. struct dm_io_region src, dest;
  848. sector_t io_size;
  849. uint64_t previous_count;
  850. BUG_ON(!test_bit(RUNNING_MERGE, &s->state_bits));
  851. if (unlikely(test_bit(SHUTDOWN_MERGE, &s->state_bits)))
  852. goto shut;
  853. /*
  854. * valid flag never changes during merge, so no lock required.
  855. */
  856. if (!s->valid) {
  857. DMERR("Snapshot is invalid: can't merge");
  858. goto shut;
  859. }
  860. linear_chunks = s->store->type->prepare_merge(s->store, &old_chunk,
  861. &new_chunk);
  862. if (linear_chunks <= 0) {
  863. if (linear_chunks < 0) {
  864. DMERR("Read error in exception store: "
  865. "shutting down merge");
  866. down_write(&s->lock);
  867. s->merge_failed = 1;
  868. up_write(&s->lock);
  869. }
  870. goto shut;
  871. }
  872. /* Adjust old_chunk and new_chunk to reflect start of linear region */
  873. old_chunk = old_chunk + 1 - linear_chunks;
  874. new_chunk = new_chunk + 1 - linear_chunks;
  875. /*
  876. * Use one (potentially large) I/O to copy all 'linear_chunks'
  877. * from the exception store to the origin
  878. */
  879. io_size = linear_chunks * s->store->chunk_size;
  880. dest.bdev = s->origin->bdev;
  881. dest.sector = chunk_to_sector(s->store, old_chunk);
  882. dest.count = min(io_size, get_dev_size(dest.bdev) - dest.sector);
  883. src.bdev = s->cow->bdev;
  884. src.sector = chunk_to_sector(s->store, new_chunk);
  885. src.count = dest.count;
  886. /*
  887. * Reallocate any exceptions needed in other snapshots then
  888. * wait for the pending exceptions to complete.
  889. * Each time any pending exception (globally on the system)
  890. * completes we are woken and repeat the process to find out
  891. * if we can proceed. While this may not seem a particularly
  892. * efficient algorithm, it is not expected to have any
  893. * significant impact on performance.
  894. */
  895. previous_count = read_pending_exceptions_done_count();
  896. while (origin_write_extent(s, dest.sector, io_size)) {
  897. wait_event(_pending_exceptions_done,
  898. (read_pending_exceptions_done_count() !=
  899. previous_count));
  900. /* Retry after the wait, until all exceptions are done. */
  901. previous_count = read_pending_exceptions_done_count();
  902. }
  903. down_write(&s->lock);
  904. s->first_merging_chunk = old_chunk;
  905. s->num_merging_chunks = linear_chunks;
  906. up_write(&s->lock);
  907. /* Wait until writes to all 'linear_chunks' drain */
  908. for (i = 0; i < linear_chunks; i++)
  909. __check_for_conflicting_io(s, old_chunk + i);
  910. dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, merge_callback, s);
  911. return;
  912. shut:
  913. merge_shutdown(s);
  914. }
  915. static void error_bios(struct bio *bio);
  916. static int flush_data(struct dm_snapshot *s)
  917. {
  918. struct bio *flush_bio = &s->flush_bio;
  919. bio_reset(flush_bio);
  920. bio_set_dev(flush_bio, s->origin->bdev);
  921. flush_bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
  922. return submit_bio_wait(flush_bio);
  923. }
  924. static void merge_callback(int read_err, unsigned long write_err, void *context)
  925. {
  926. struct dm_snapshot *s = context;
  927. struct bio *b = NULL;
  928. if (read_err || write_err) {
  929. if (read_err)
  930. DMERR("Read error: shutting down merge.");
  931. else
  932. DMERR("Write error: shutting down merge.");
  933. goto shut;
  934. }
  935. if (flush_data(s) < 0) {
  936. DMERR("Flush after merge failed: shutting down merge");
  937. goto shut;
  938. }
  939. if (s->store->type->commit_merge(s->store,
  940. s->num_merging_chunks) < 0) {
  941. DMERR("Write error in exception store: shutting down merge");
  942. goto shut;
  943. }
  944. if (remove_single_exception_chunk(s) < 0)
  945. goto shut;
  946. snapshot_merge_next_chunks(s);
  947. return;
  948. shut:
  949. down_write(&s->lock);
  950. s->merge_failed = 1;
  951. b = __release_queued_bios_after_merge(s);
  952. up_write(&s->lock);
  953. error_bios(b);
  954. merge_shutdown(s);
  955. }
  956. static void start_merge(struct dm_snapshot *s)
  957. {
  958. if (!test_and_set_bit(RUNNING_MERGE, &s->state_bits))
  959. snapshot_merge_next_chunks(s);
  960. }
  961. /*
  962. * Stop the merging process and wait until it finishes.
  963. */
  964. static void stop_merge(struct dm_snapshot *s)
  965. {
  966. set_bit(SHUTDOWN_MERGE, &s->state_bits);
  967. wait_on_bit(&s->state_bits, RUNNING_MERGE, TASK_UNINTERRUPTIBLE);
  968. clear_bit(SHUTDOWN_MERGE, &s->state_bits);
  969. }
  970. static int parse_snapshot_features(struct dm_arg_set *as, struct dm_snapshot *s,
  971. struct dm_target *ti)
  972. {
  973. int r;
  974. unsigned argc;
  975. const char *arg_name;
  976. static const struct dm_arg _args[] = {
  977. {0, 2, "Invalid number of feature arguments"},
  978. };
  979. /*
  980. * No feature arguments supplied.
  981. */
  982. if (!as->argc)
  983. return 0;
  984. r = dm_read_arg_group(_args, as, &argc, &ti->error);
  985. if (r)
  986. return -EINVAL;
  987. while (argc && !r) {
  988. arg_name = dm_shift_arg(as);
  989. argc--;
  990. if (!strcasecmp(arg_name, "discard_zeroes_cow"))
  991. s->discard_zeroes_cow = true;
  992. else if (!strcasecmp(arg_name, "discard_passdown_origin"))
  993. s->discard_passdown_origin = true;
  994. else {
  995. ti->error = "Unrecognised feature requested";
  996. r = -EINVAL;
  997. break;
  998. }
  999. }
  1000. if (!s->discard_zeroes_cow && s->discard_passdown_origin) {
  1001. /*
  1002. * TODO: really these are disjoint.. but ti->num_discard_bios
  1003. * and dm_bio_get_target_bio_nr() require rigid constraints.
  1004. */
  1005. ti->error = "discard_passdown_origin feature depends on discard_zeroes_cow";
  1006. r = -EINVAL;
  1007. }
  1008. return r;
  1009. }
  1010. /*
  1011. * Construct a snapshot mapping:
  1012. * <origin_dev> <COW-dev> <p|po|n> <chunk-size> [<# feature args> [<arg>]*]
  1013. */
  1014. static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv)
  1015. {
  1016. struct dm_snapshot *s;
  1017. struct dm_arg_set as;
  1018. int i;
  1019. int r = -EINVAL;
  1020. char *origin_path, *cow_path;
  1021. dev_t origin_dev, cow_dev;
  1022. unsigned args_used, num_flush_bios = 1;
  1023. fmode_t origin_mode = FMODE_READ;
  1024. if (argc < 4) {
  1025. ti->error = "requires 4 or more arguments";
  1026. r = -EINVAL;
  1027. goto bad;
  1028. }
  1029. if (dm_target_is_snapshot_merge(ti)) {
  1030. num_flush_bios = 2;
  1031. origin_mode = FMODE_WRITE;
  1032. }
  1033. s = kzalloc(sizeof(*s), GFP_KERNEL);
  1034. if (!s) {
  1035. ti->error = "Cannot allocate private snapshot structure";
  1036. r = -ENOMEM;
  1037. goto bad;
  1038. }
  1039. as.argc = argc;
  1040. as.argv = argv;
  1041. dm_consume_args(&as, 4);
  1042. r = parse_snapshot_features(&as, s, ti);
  1043. if (r)
  1044. goto bad_features;
  1045. origin_path = argv[0];
  1046. argv++;
  1047. argc--;
  1048. r = dm_get_device(ti, origin_path, origin_mode, &s->origin);
  1049. if (r) {
  1050. ti->error = "Cannot get origin device";
  1051. goto bad_origin;
  1052. }
  1053. origin_dev = s->origin->bdev->bd_dev;
  1054. cow_path = argv[0];
  1055. argv++;
  1056. argc--;
  1057. cow_dev = dm_get_dev_t(cow_path);
  1058. if (cow_dev && cow_dev == origin_dev) {
  1059. ti->error = "COW device cannot be the same as origin device";
  1060. r = -EINVAL;
  1061. goto bad_cow;
  1062. }
  1063. r = dm_get_device(ti, cow_path, dm_table_get_mode(ti->table), &s->cow);
  1064. if (r) {
  1065. ti->error = "Cannot get COW device";
  1066. goto bad_cow;
  1067. }
  1068. r = dm_exception_store_create(ti, argc, argv, s, &args_used, &s->store);
  1069. if (r) {
  1070. ti->error = "Couldn't create exception store";
  1071. r = -EINVAL;
  1072. goto bad_store;
  1073. }
  1074. argv += args_used;
  1075. argc -= args_used;
  1076. s->ti = ti;
  1077. s->valid = 1;
  1078. s->snapshot_overflowed = 0;
  1079. s->active = 0;
  1080. atomic_set(&s->pending_exceptions_count, 0);
  1081. spin_lock_init(&s->pe_allocation_lock);
  1082. s->exception_start_sequence = 0;
  1083. s->exception_complete_sequence = 0;
  1084. s->out_of_order_tree = RB_ROOT;
  1085. init_rwsem(&s->lock);
  1086. INIT_LIST_HEAD(&s->list);
  1087. spin_lock_init(&s->pe_lock);
  1088. s->state_bits = 0;
  1089. s->merge_failed = 0;
  1090. s->first_merging_chunk = 0;
  1091. s->num_merging_chunks = 0;
  1092. bio_list_init(&s->bios_queued_during_merge);
  1093. bio_init(&s->flush_bio, NULL, 0);
  1094. /* Allocate hash table for COW data */
  1095. if (init_hash_tables(s)) {
  1096. ti->error = "Unable to allocate hash table space";
  1097. r = -ENOMEM;
  1098. goto bad_hash_tables;
  1099. }
  1100. init_waitqueue_head(&s->in_progress_wait);
  1101. s->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle);
  1102. if (IS_ERR(s->kcopyd_client)) {
  1103. r = PTR_ERR(s->kcopyd_client);
  1104. ti->error = "Could not create kcopyd client";
  1105. goto bad_kcopyd;
  1106. }
  1107. r = mempool_init_slab_pool(&s->pending_pool, MIN_IOS, pending_cache);
  1108. if (r) {
  1109. ti->error = "Could not allocate mempool for pending exceptions";
  1110. goto bad_pending_pool;
  1111. }
  1112. for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++)
  1113. INIT_HLIST_HEAD(&s->tracked_chunk_hash[i]);
  1114. spin_lock_init(&s->tracked_chunk_lock);
  1115. ti->private = s;
  1116. ti->num_flush_bios = num_flush_bios;
  1117. if (s->discard_zeroes_cow)
  1118. ti->num_discard_bios = (s->discard_passdown_origin ? 2 : 1);
  1119. ti->per_io_data_size = sizeof(struct dm_snap_tracked_chunk);
  1120. /* Add snapshot to the list of snapshots for this origin */
  1121. /* Exceptions aren't triggered till snapshot_resume() is called */
  1122. r = register_snapshot(s);
  1123. if (r == -ENOMEM) {
  1124. ti->error = "Snapshot origin struct allocation failed";
  1125. goto bad_load_and_register;
  1126. } else if (r < 0) {
  1127. /* invalid handover, register_snapshot has set ti->error */
  1128. goto bad_load_and_register;
  1129. }
  1130. /*
  1131. * Metadata must only be loaded into one table at once, so skip this
  1132. * if metadata will be handed over during resume.
  1133. * Chunk size will be set during the handover - set it to zero to
  1134. * ensure it's ignored.
  1135. */
  1136. if (r > 0) {
  1137. s->store->chunk_size = 0;
  1138. return 0;
  1139. }
  1140. r = s->store->type->read_metadata(s->store, dm_add_exception,
  1141. (void *)s);
  1142. if (r < 0) {
  1143. ti->error = "Failed to read snapshot metadata";
  1144. goto bad_read_metadata;
  1145. } else if (r > 0) {
  1146. s->valid = 0;
  1147. DMWARN("Snapshot is marked invalid.");
  1148. }
  1149. if (!s->store->chunk_size) {
  1150. ti->error = "Chunk size not set";
  1151. r = -EINVAL;
  1152. goto bad_read_metadata;
  1153. }
  1154. r = dm_set_target_max_io_len(ti, s->store->chunk_size);
  1155. if (r)
  1156. goto bad_read_metadata;
  1157. return 0;
  1158. bad_read_metadata:
  1159. unregister_snapshot(s);
  1160. bad_load_and_register:
  1161. mempool_exit(&s->pending_pool);
  1162. bad_pending_pool:
  1163. dm_kcopyd_client_destroy(s->kcopyd_client);
  1164. bad_kcopyd:
  1165. dm_exception_table_exit(&s->pending, pending_cache);
  1166. dm_exception_table_exit(&s->complete, exception_cache);
  1167. bad_hash_tables:
  1168. dm_exception_store_destroy(s->store);
  1169. bad_store:
  1170. dm_put_device(ti, s->cow);
  1171. bad_cow:
  1172. dm_put_device(ti, s->origin);
  1173. bad_origin:
  1174. bad_features:
  1175. kfree(s);
  1176. bad:
  1177. return r;
  1178. }
  1179. static void __free_exceptions(struct dm_snapshot *s)
  1180. {
  1181. dm_kcopyd_client_destroy(s->kcopyd_client);
  1182. s->kcopyd_client = NULL;
  1183. dm_exception_table_exit(&s->pending, pending_cache);
  1184. dm_exception_table_exit(&s->complete, exception_cache);
  1185. }
  1186. static void __handover_exceptions(struct dm_snapshot *snap_src,
  1187. struct dm_snapshot *snap_dest)
  1188. {
  1189. union {
  1190. struct dm_exception_table table_swap;
  1191. struct dm_exception_store *store_swap;
  1192. } u;
  1193. /*
  1194. * Swap all snapshot context information between the two instances.
  1195. */
  1196. u.table_swap = snap_dest->complete;
  1197. snap_dest->complete = snap_src->complete;
  1198. snap_src->complete = u.table_swap;
  1199. u.store_swap = snap_dest->store;
  1200. snap_dest->store = snap_src->store;
  1201. snap_dest->store->userspace_supports_overflow = u.store_swap->userspace_supports_overflow;
  1202. snap_src->store = u.store_swap;
  1203. snap_dest->store->snap = snap_dest;
  1204. snap_src->store->snap = snap_src;
  1205. snap_dest->ti->max_io_len = snap_dest->store->chunk_size;
  1206. snap_dest->valid = snap_src->valid;
  1207. snap_dest->snapshot_overflowed = snap_src->snapshot_overflowed;
  1208. /*
  1209. * Set source invalid to ensure it receives no further I/O.
  1210. */
  1211. snap_src->valid = 0;
  1212. }
  1213. static void snapshot_dtr(struct dm_target *ti)
  1214. {
  1215. #ifdef CONFIG_DM_DEBUG
  1216. int i;
  1217. #endif
  1218. struct dm_snapshot *s = ti->private;
  1219. struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
  1220. down_read(&_origins_lock);
  1221. /* Check whether exception handover must be cancelled */
  1222. (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
  1223. if (snap_src && snap_dest && (s == snap_src)) {
  1224. down_write(&snap_dest->lock);
  1225. snap_dest->valid = 0;
  1226. up_write(&snap_dest->lock);
  1227. DMERR("Cancelling snapshot handover.");
  1228. }
  1229. up_read(&_origins_lock);
  1230. if (dm_target_is_snapshot_merge(ti))
  1231. stop_merge(s);
  1232. /* Prevent further origin writes from using this snapshot. */
  1233. /* After this returns there can be no new kcopyd jobs. */
  1234. unregister_snapshot(s);
  1235. while (atomic_read(&s->pending_exceptions_count))
  1236. msleep(1);
  1237. /*
  1238. * Ensure instructions in mempool_exit aren't reordered
  1239. * before atomic_read.
  1240. */
  1241. smp_mb();
  1242. #ifdef CONFIG_DM_DEBUG
  1243. for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++)
  1244. BUG_ON(!hlist_empty(&s->tracked_chunk_hash[i]));
  1245. #endif
  1246. __free_exceptions(s);
  1247. mempool_exit(&s->pending_pool);
  1248. dm_exception_store_destroy(s->store);
  1249. bio_uninit(&s->flush_bio);
  1250. dm_put_device(ti, s->cow);
  1251. dm_put_device(ti, s->origin);
  1252. WARN_ON(s->in_progress);
  1253. kfree(s);
  1254. }
  1255. static void account_start_copy(struct dm_snapshot *s)
  1256. {
  1257. spin_lock(&s->in_progress_wait.lock);
  1258. s->in_progress++;
  1259. spin_unlock(&s->in_progress_wait.lock);
  1260. }
  1261. static void account_end_copy(struct dm_snapshot *s)
  1262. {
  1263. spin_lock(&s->in_progress_wait.lock);
  1264. BUG_ON(!s->in_progress);
  1265. s->in_progress--;
  1266. if (likely(s->in_progress <= cow_threshold) &&
  1267. unlikely(waitqueue_active(&s->in_progress_wait)))
  1268. wake_up_locked(&s->in_progress_wait);
  1269. spin_unlock(&s->in_progress_wait.lock);
  1270. }
  1271. static bool wait_for_in_progress(struct dm_snapshot *s, bool unlock_origins)
  1272. {
  1273. if (unlikely(s->in_progress > cow_threshold)) {
  1274. spin_lock(&s->in_progress_wait.lock);
  1275. if (likely(s->in_progress > cow_threshold)) {
  1276. /*
  1277. * NOTE: this throttle doesn't account for whether
  1278. * the caller is servicing an IO that will trigger a COW
  1279. * so excess throttling may result for chunks not required
  1280. * to be COW'd. But if cow_threshold was reached, extra
  1281. * throttling is unlikely to negatively impact performance.
  1282. */
  1283. DECLARE_WAITQUEUE(wait, current);
  1284. __add_wait_queue(&s->in_progress_wait, &wait);
  1285. __set_current_state(TASK_UNINTERRUPTIBLE);
  1286. spin_unlock(&s->in_progress_wait.lock);
  1287. if (unlock_origins)
  1288. up_read(&_origins_lock);
  1289. io_schedule();
  1290. remove_wait_queue(&s->in_progress_wait, &wait);
  1291. return false;
  1292. }
  1293. spin_unlock(&s->in_progress_wait.lock);
  1294. }
  1295. return true;
  1296. }
  1297. /*
  1298. * Flush a list of buffers.
  1299. */
  1300. static void flush_bios(struct bio *bio)
  1301. {
  1302. struct bio *n;
  1303. while (bio) {
  1304. n = bio->bi_next;
  1305. bio->bi_next = NULL;
  1306. generic_make_request(bio);
  1307. bio = n;
  1308. }
  1309. }
  1310. static int do_origin(struct dm_dev *origin, struct bio *bio, bool limit);
  1311. /*
  1312. * Flush a list of buffers.
  1313. */
  1314. static void retry_origin_bios(struct dm_snapshot *s, struct bio *bio)
  1315. {
  1316. struct bio *n;
  1317. int r;
  1318. while (bio) {
  1319. n = bio->bi_next;
  1320. bio->bi_next = NULL;
  1321. r = do_origin(s->origin, bio, false);
  1322. if (r == DM_MAPIO_REMAPPED)
  1323. generic_make_request(bio);
  1324. bio = n;
  1325. }
  1326. }
  1327. /*
  1328. * Error a list of buffers.
  1329. */
  1330. static void error_bios(struct bio *bio)
  1331. {
  1332. struct bio *n;
  1333. while (bio) {
  1334. n = bio->bi_next;
  1335. bio->bi_next = NULL;
  1336. bio_io_error(bio);
  1337. bio = n;
  1338. }
  1339. }
  1340. static void __invalidate_snapshot(struct dm_snapshot *s, int err)
  1341. {
  1342. if (!s->valid)
  1343. return;
  1344. if (err == -EIO)
  1345. DMERR("Invalidating snapshot: Error reading/writing.");
  1346. else if (err == -ENOMEM)
  1347. DMERR("Invalidating snapshot: Unable to allocate exception.");
  1348. if (s->store->type->drop_snapshot)
  1349. s->store->type->drop_snapshot(s->store);
  1350. s->valid = 0;
  1351. dm_table_event(s->ti->table);
  1352. }
  1353. static void invalidate_snapshot(struct dm_snapshot *s, int err)
  1354. {
  1355. down_write(&s->lock);
  1356. __invalidate_snapshot(s, err);
  1357. up_write(&s->lock);
  1358. }
  1359. static void pending_complete(void *context, int success)
  1360. {
  1361. struct dm_snap_pending_exception *pe = context;
  1362. struct dm_exception *e;
  1363. struct dm_snapshot *s = pe->snap;
  1364. struct bio *origin_bios = NULL;
  1365. struct bio *snapshot_bios = NULL;
  1366. struct bio *full_bio = NULL;
  1367. struct dm_exception_table_lock lock;
  1368. int error = 0;
  1369. dm_exception_table_lock_init(s, pe->e.old_chunk, &lock);
  1370. if (!success) {
  1371. /* Read/write error - snapshot is unusable */
  1372. invalidate_snapshot(s, -EIO);
  1373. error = 1;
  1374. dm_exception_table_lock(&lock);
  1375. goto out;
  1376. }
  1377. e = alloc_completed_exception(GFP_NOIO);
  1378. if (!e) {
  1379. invalidate_snapshot(s, -ENOMEM);
  1380. error = 1;
  1381. dm_exception_table_lock(&lock);
  1382. goto out;
  1383. }
  1384. *e = pe->e;
  1385. down_read(&s->lock);
  1386. dm_exception_table_lock(&lock);
  1387. if (!s->valid) {
  1388. up_read(&s->lock);
  1389. free_completed_exception(e);
  1390. error = 1;
  1391. goto out;
  1392. }
  1393. /*
  1394. * Add a proper exception. After inserting the completed exception all
  1395. * subsequent snapshot reads to this chunk will be redirected to the
  1396. * COW device. This ensures that we do not starve. Moreover, as long
  1397. * as the pending exception exists, neither origin writes nor snapshot
  1398. * merging can overwrite the chunk in origin.
  1399. */
  1400. dm_insert_exception(&s->complete, e);
  1401. up_read(&s->lock);
  1402. /* Wait for conflicting reads to drain */
  1403. if (__chunk_is_tracked(s, pe->e.old_chunk)) {
  1404. dm_exception_table_unlock(&lock);
  1405. __check_for_conflicting_io(s, pe->e.old_chunk);
  1406. dm_exception_table_lock(&lock);
  1407. }
  1408. out:
  1409. /* Remove the in-flight exception from the list */
  1410. dm_remove_exception(&pe->e);
  1411. dm_exception_table_unlock(&lock);
  1412. snapshot_bios = bio_list_get(&pe->snapshot_bios);
  1413. origin_bios = bio_list_get(&pe->origin_bios);
  1414. full_bio = pe->full_bio;
  1415. if (full_bio)
  1416. full_bio->bi_end_io = pe->full_bio_end_io;
  1417. increment_pending_exceptions_done_count();
  1418. /* Submit any pending write bios */
  1419. if (error) {
  1420. if (full_bio)
  1421. bio_io_error(full_bio);
  1422. error_bios(snapshot_bios);
  1423. } else {
  1424. if (full_bio)
  1425. bio_endio(full_bio);
  1426. flush_bios(snapshot_bios);
  1427. }
  1428. retry_origin_bios(s, origin_bios);
  1429. free_pending_exception(pe);
  1430. }
  1431. static void complete_exception(struct dm_snap_pending_exception *pe)
  1432. {
  1433. struct dm_snapshot *s = pe->snap;
  1434. /* Update the metadata if we are persistent */
  1435. s->store->type->commit_exception(s->store, &pe->e, !pe->copy_error,
  1436. pending_complete, pe);
  1437. }
  1438. /*
  1439. * Called when the copy I/O has finished. kcopyd actually runs
  1440. * this code so don't block.
  1441. */
  1442. static void copy_callback(int read_err, unsigned long write_err, void *context)
  1443. {
  1444. struct dm_snap_pending_exception *pe = context;
  1445. struct dm_snapshot *s = pe->snap;
  1446. pe->copy_error = read_err || write_err;
  1447. if (pe->exception_sequence == s->exception_complete_sequence) {
  1448. struct rb_node *next;
  1449. s->exception_complete_sequence++;
  1450. complete_exception(pe);
  1451. next = rb_first(&s->out_of_order_tree);
  1452. while (next) {
  1453. pe = rb_entry(next, struct dm_snap_pending_exception,
  1454. out_of_order_node);
  1455. if (pe->exception_sequence != s->exception_complete_sequence)
  1456. break;
  1457. next = rb_next(next);
  1458. s->exception_complete_sequence++;
  1459. rb_erase(&pe->out_of_order_node, &s->out_of_order_tree);
  1460. complete_exception(pe);
  1461. cond_resched();
  1462. }
  1463. } else {
  1464. struct rb_node *parent = NULL;
  1465. struct rb_node **p = &s->out_of_order_tree.rb_node;
  1466. struct dm_snap_pending_exception *pe2;
  1467. while (*p) {
  1468. pe2 = rb_entry(*p, struct dm_snap_pending_exception, out_of_order_node);
  1469. parent = *p;
  1470. BUG_ON(pe->exception_sequence == pe2->exception_sequence);
  1471. if (pe->exception_sequence < pe2->exception_sequence)
  1472. p = &((*p)->rb_left);
  1473. else
  1474. p = &((*p)->rb_right);
  1475. }
  1476. rb_link_node(&pe->out_of_order_node, parent, p);
  1477. rb_insert_color(&pe->out_of_order_node, &s->out_of_order_tree);
  1478. }
  1479. account_end_copy(s);
  1480. }
  1481. /*
  1482. * Dispatches the copy operation to kcopyd.
  1483. */
  1484. static void start_copy(struct dm_snap_pending_exception *pe)
  1485. {
  1486. struct dm_snapshot *s = pe->snap;
  1487. struct dm_io_region src, dest;
  1488. struct block_device *bdev = s->origin->bdev;
  1489. sector_t dev_size;
  1490. dev_size = get_dev_size(bdev);
  1491. src.bdev = bdev;
  1492. src.sector = chunk_to_sector(s->store, pe->e.old_chunk);
  1493. src.count = min((sector_t)s->store->chunk_size, dev_size - src.sector);
  1494. dest.bdev = s->cow->bdev;
  1495. dest.sector = chunk_to_sector(s->store, pe->e.new_chunk);
  1496. dest.count = src.count;
  1497. /* Hand over to kcopyd */
  1498. account_start_copy(s);
  1499. dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe);
  1500. }
  1501. static void full_bio_end_io(struct bio *bio)
  1502. {
  1503. void *callback_data = bio->bi_private;
  1504. dm_kcopyd_do_callback(callback_data, 0, bio->bi_status ? 1 : 0);
  1505. }
  1506. static void start_full_bio(struct dm_snap_pending_exception *pe,
  1507. struct bio *bio)
  1508. {
  1509. struct dm_snapshot *s = pe->snap;
  1510. void *callback_data;
  1511. pe->full_bio = bio;
  1512. pe->full_bio_end_io = bio->bi_end_io;
  1513. account_start_copy(s);
  1514. callback_data = dm_kcopyd_prepare_callback(s->kcopyd_client,
  1515. copy_callback, pe);
  1516. bio->bi_end_io = full_bio_end_io;
  1517. bio->bi_private = callback_data;
  1518. generic_make_request(bio);
  1519. }
  1520. static struct dm_snap_pending_exception *
  1521. __lookup_pending_exception(struct dm_snapshot *s, chunk_t chunk)
  1522. {
  1523. struct dm_exception *e = dm_lookup_exception(&s->pending, chunk);
  1524. if (!e)
  1525. return NULL;
  1526. return container_of(e, struct dm_snap_pending_exception, e);
  1527. }
  1528. /*
  1529. * Inserts a pending exception into the pending table.
  1530. *
  1531. * NOTE: a write lock must be held on the chunk's pending exception table slot
  1532. * before calling this.
  1533. */
  1534. static struct dm_snap_pending_exception *
  1535. __insert_pending_exception(struct dm_snapshot *s,
  1536. struct dm_snap_pending_exception *pe, chunk_t chunk)
  1537. {
  1538. pe->e.old_chunk = chunk;
  1539. bio_list_init(&pe->origin_bios);
  1540. bio_list_init(&pe->snapshot_bios);
  1541. pe->started = 0;
  1542. pe->full_bio = NULL;
  1543. spin_lock(&s->pe_allocation_lock);
  1544. if (s->store->type->prepare_exception(s->store, &pe->e)) {
  1545. spin_unlock(&s->pe_allocation_lock);
  1546. free_pending_exception(pe);
  1547. return NULL;
  1548. }
  1549. pe->exception_sequence = s->exception_start_sequence++;
  1550. spin_unlock(&s->pe_allocation_lock);
  1551. dm_insert_exception(&s->pending, &pe->e);
  1552. return pe;
  1553. }
  1554. /*
  1555. * Looks to see if this snapshot already has a pending exception
  1556. * for this chunk, otherwise it allocates a new one and inserts
  1557. * it into the pending table.
  1558. *
  1559. * NOTE: a write lock must be held on the chunk's pending exception table slot
  1560. * before calling this.
  1561. */
  1562. static struct dm_snap_pending_exception *
  1563. __find_pending_exception(struct dm_snapshot *s,
  1564. struct dm_snap_pending_exception *pe, chunk_t chunk)
  1565. {
  1566. struct dm_snap_pending_exception *pe2;
  1567. pe2 = __lookup_pending_exception(s, chunk);
  1568. if (pe2) {
  1569. free_pending_exception(pe);
  1570. return pe2;
  1571. }
  1572. return __insert_pending_exception(s, pe, chunk);
  1573. }
  1574. static void remap_exception(struct dm_snapshot *s, struct dm_exception *e,
  1575. struct bio *bio, chunk_t chunk)
  1576. {
  1577. bio_set_dev(bio, s->cow->bdev);
  1578. bio->bi_iter.bi_sector =
  1579. chunk_to_sector(s->store, dm_chunk_number(e->new_chunk) +
  1580. (chunk - e->old_chunk)) +
  1581. (bio->bi_iter.bi_sector & s->store->chunk_mask);
  1582. }
  1583. static void zero_callback(int read_err, unsigned long write_err, void *context)
  1584. {
  1585. struct bio *bio = context;
  1586. struct dm_snapshot *s = bio->bi_private;
  1587. account_end_copy(s);
  1588. bio->bi_status = write_err ? BLK_STS_IOERR : 0;
  1589. bio_endio(bio);
  1590. }
  1591. static void zero_exception(struct dm_snapshot *s, struct dm_exception *e,
  1592. struct bio *bio, chunk_t chunk)
  1593. {
  1594. struct dm_io_region dest;
  1595. dest.bdev = s->cow->bdev;
  1596. dest.sector = bio->bi_iter.bi_sector;
  1597. dest.count = s->store->chunk_size;
  1598. account_start_copy(s);
  1599. WARN_ON_ONCE(bio->bi_private);
  1600. bio->bi_private = s;
  1601. dm_kcopyd_zero(s->kcopyd_client, 1, &dest, 0, zero_callback, bio);
  1602. }
  1603. static bool io_overlaps_chunk(struct dm_snapshot *s, struct bio *bio)
  1604. {
  1605. return bio->bi_iter.bi_size ==
  1606. (s->store->chunk_size << SECTOR_SHIFT);
  1607. }
  1608. static int snapshot_map(struct dm_target *ti, struct bio *bio)
  1609. {
  1610. struct dm_exception *e;
  1611. struct dm_snapshot *s = ti->private;
  1612. int r = DM_MAPIO_REMAPPED;
  1613. chunk_t chunk;
  1614. struct dm_snap_pending_exception *pe = NULL;
  1615. struct dm_exception_table_lock lock;
  1616. init_tracked_chunk(bio);
  1617. if (bio->bi_opf & REQ_PREFLUSH) {
  1618. bio_set_dev(bio, s->cow->bdev);
  1619. return DM_MAPIO_REMAPPED;
  1620. }
  1621. chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector);
  1622. dm_exception_table_lock_init(s, chunk, &lock);
  1623. /* Full snapshots are not usable */
  1624. /* To get here the table must be live so s->active is always set. */
  1625. if (!s->valid)
  1626. return DM_MAPIO_KILL;
  1627. if (bio_data_dir(bio) == WRITE) {
  1628. while (unlikely(!wait_for_in_progress(s, false)))
  1629. ; /* wait_for_in_progress() has slept */
  1630. }
  1631. down_read(&s->lock);
  1632. dm_exception_table_lock(&lock);
  1633. if (!s->valid || (unlikely(s->snapshot_overflowed) &&
  1634. bio_data_dir(bio) == WRITE)) {
  1635. r = DM_MAPIO_KILL;
  1636. goto out_unlock;
  1637. }
  1638. if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) {
  1639. if (s->discard_passdown_origin && dm_bio_get_target_bio_nr(bio)) {
  1640. /*
  1641. * passdown discard to origin (without triggering
  1642. * snapshot exceptions via do_origin; doing so would
  1643. * defeat the goal of freeing space in origin that is
  1644. * implied by the "discard_passdown_origin" feature)
  1645. */
  1646. bio_set_dev(bio, s->origin->bdev);
  1647. track_chunk(s, bio, chunk);
  1648. goto out_unlock;
  1649. }
  1650. /* discard to snapshot (target_bio_nr == 0) zeroes exceptions */
  1651. }
  1652. /* If the block is already remapped - use that, else remap it */
  1653. e = dm_lookup_exception(&s->complete, chunk);
  1654. if (e) {
  1655. remap_exception(s, e, bio, chunk);
  1656. if (unlikely(bio_op(bio) == REQ_OP_DISCARD) &&
  1657. io_overlaps_chunk(s, bio)) {
  1658. dm_exception_table_unlock(&lock);
  1659. up_read(&s->lock);
  1660. zero_exception(s, e, bio, chunk);
  1661. r = DM_MAPIO_SUBMITTED; /* discard is not issued */
  1662. goto out;
  1663. }
  1664. goto out_unlock;
  1665. }
  1666. if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) {
  1667. /*
  1668. * If no exception exists, complete discard immediately
  1669. * otherwise it'll trigger copy-out.
  1670. */
  1671. bio_endio(bio);
  1672. r = DM_MAPIO_SUBMITTED;
  1673. goto out_unlock;
  1674. }
  1675. /*
  1676. * Write to snapshot - higher level takes care of RW/RO
  1677. * flags so we should only get this if we are
  1678. * writeable.
  1679. */
  1680. if (bio_data_dir(bio) == WRITE) {
  1681. pe = __lookup_pending_exception(s, chunk);
  1682. if (!pe) {
  1683. dm_exception_table_unlock(&lock);
  1684. pe = alloc_pending_exception(s);
  1685. dm_exception_table_lock(&lock);
  1686. e = dm_lookup_exception(&s->complete, chunk);
  1687. if (e) {
  1688. free_pending_exception(pe);
  1689. remap_exception(s, e, bio, chunk);
  1690. goto out_unlock;
  1691. }
  1692. pe = __find_pending_exception(s, pe, chunk);
  1693. if (!pe) {
  1694. dm_exception_table_unlock(&lock);
  1695. up_read(&s->lock);
  1696. down_write(&s->lock);
  1697. if (s->store->userspace_supports_overflow) {
  1698. if (s->valid && !s->snapshot_overflowed) {
  1699. s->snapshot_overflowed = 1;
  1700. DMERR("Snapshot overflowed: Unable to allocate exception.");
  1701. }
  1702. } else
  1703. __invalidate_snapshot(s, -ENOMEM);
  1704. up_write(&s->lock);
  1705. r = DM_MAPIO_KILL;
  1706. goto out;
  1707. }
  1708. }
  1709. remap_exception(s, &pe->e, bio, chunk);
  1710. r = DM_MAPIO_SUBMITTED;
  1711. if (!pe->started && io_overlaps_chunk(s, bio)) {
  1712. pe->started = 1;
  1713. dm_exception_table_unlock(&lock);
  1714. up_read(&s->lock);
  1715. start_full_bio(pe, bio);
  1716. goto out;
  1717. }
  1718. bio_list_add(&pe->snapshot_bios, bio);
  1719. if (!pe->started) {
  1720. /* this is protected by the exception table lock */
  1721. pe->started = 1;
  1722. dm_exception_table_unlock(&lock);
  1723. up_read(&s->lock);
  1724. start_copy(pe);
  1725. goto out;
  1726. }
  1727. } else {
  1728. bio_set_dev(bio, s->origin->bdev);
  1729. track_chunk(s, bio, chunk);
  1730. }
  1731. out_unlock:
  1732. dm_exception_table_unlock(&lock);
  1733. up_read(&s->lock);
  1734. out:
  1735. return r;
  1736. }
  1737. /*
  1738. * A snapshot-merge target behaves like a combination of a snapshot
  1739. * target and a snapshot-origin target. It only generates new
  1740. * exceptions in other snapshots and not in the one that is being
  1741. * merged.
  1742. *
  1743. * For each chunk, if there is an existing exception, it is used to
  1744. * redirect I/O to the cow device. Otherwise I/O is sent to the origin,
  1745. * which in turn might generate exceptions in other snapshots.
  1746. * If merging is currently taking place on the chunk in question, the
  1747. * I/O is deferred by adding it to s->bios_queued_during_merge.
  1748. */
  1749. static int snapshot_merge_map(struct dm_target *ti, struct bio *bio)
  1750. {
  1751. struct dm_exception *e;
  1752. struct dm_snapshot *s = ti->private;
  1753. int r = DM_MAPIO_REMAPPED;
  1754. chunk_t chunk;
  1755. init_tracked_chunk(bio);
  1756. if (bio->bi_opf & REQ_PREFLUSH) {
  1757. if (!dm_bio_get_target_bio_nr(bio))
  1758. bio_set_dev(bio, s->origin->bdev);
  1759. else
  1760. bio_set_dev(bio, s->cow->bdev);
  1761. return DM_MAPIO_REMAPPED;
  1762. }
  1763. if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) {
  1764. /* Once merging, discards no longer effect change */
  1765. bio_endio(bio);
  1766. return DM_MAPIO_SUBMITTED;
  1767. }
  1768. chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector);
  1769. down_write(&s->lock);
  1770. /* Full merging snapshots are redirected to the origin */
  1771. if (!s->valid)
  1772. goto redirect_to_origin;
  1773. /* If the block is already remapped - use that */
  1774. e = dm_lookup_exception(&s->complete, chunk);
  1775. if (e) {
  1776. /* Queue writes overlapping with chunks being merged */
  1777. if (bio_data_dir(bio) == WRITE &&
  1778. chunk >= s->first_merging_chunk &&
  1779. chunk < (s->first_merging_chunk +
  1780. s->num_merging_chunks)) {
  1781. bio_set_dev(bio, s->origin->bdev);
  1782. bio_list_add(&s->bios_queued_during_merge, bio);
  1783. r = DM_MAPIO_SUBMITTED;
  1784. goto out_unlock;
  1785. }
  1786. remap_exception(s, e, bio, chunk);
  1787. if (bio_data_dir(bio) == WRITE)
  1788. track_chunk(s, bio, chunk);
  1789. goto out_unlock;
  1790. }
  1791. redirect_to_origin:
  1792. bio_set_dev(bio, s->origin->bdev);
  1793. if (bio_data_dir(bio) == WRITE) {
  1794. up_write(&s->lock);
  1795. return do_origin(s->origin, bio, false);
  1796. }
  1797. out_unlock:
  1798. up_write(&s->lock);
  1799. return r;
  1800. }
  1801. static int snapshot_end_io(struct dm_target *ti, struct bio *bio,
  1802. blk_status_t *error)
  1803. {
  1804. struct dm_snapshot *s = ti->private;
  1805. if (is_bio_tracked(bio))
  1806. stop_tracking_chunk(s, bio);
  1807. return DM_ENDIO_DONE;
  1808. }
  1809. static void snapshot_merge_presuspend(struct dm_target *ti)
  1810. {
  1811. struct dm_snapshot *s = ti->private;
  1812. stop_merge(s);
  1813. }
  1814. static int snapshot_preresume(struct dm_target *ti)
  1815. {
  1816. int r = 0;
  1817. struct dm_snapshot *s = ti->private;
  1818. struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
  1819. down_read(&_origins_lock);
  1820. (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
  1821. if (snap_src && snap_dest) {
  1822. down_read(&snap_src->lock);
  1823. if (s == snap_src) {
  1824. DMERR("Unable to resume snapshot source until "
  1825. "handover completes.");
  1826. r = -EINVAL;
  1827. } else if (!dm_suspended(snap_src->ti)) {
  1828. DMERR("Unable to perform snapshot handover until "
  1829. "source is suspended.");
  1830. r = -EINVAL;
  1831. }
  1832. up_read(&snap_src->lock);
  1833. }
  1834. up_read(&_origins_lock);
  1835. return r;
  1836. }
  1837. static void snapshot_resume(struct dm_target *ti)
  1838. {
  1839. struct dm_snapshot *s = ti->private;
  1840. struct dm_snapshot *snap_src = NULL, *snap_dest = NULL, *snap_merging = NULL;
  1841. struct dm_origin *o;
  1842. struct mapped_device *origin_md = NULL;
  1843. bool must_restart_merging = false;
  1844. down_read(&_origins_lock);
  1845. o = __lookup_dm_origin(s->origin->bdev);
  1846. if (o)
  1847. origin_md = dm_table_get_md(o->ti->table);
  1848. if (!origin_md) {
  1849. (void) __find_snapshots_sharing_cow(s, NULL, NULL, &snap_merging);
  1850. if (snap_merging)
  1851. origin_md = dm_table_get_md(snap_merging->ti->table);
  1852. }
  1853. if (origin_md == dm_table_get_md(ti->table))
  1854. origin_md = NULL;
  1855. if (origin_md) {
  1856. if (dm_hold(origin_md))
  1857. origin_md = NULL;
  1858. }
  1859. up_read(&_origins_lock);
  1860. if (origin_md) {
  1861. dm_internal_suspend_fast(origin_md);
  1862. if (snap_merging && test_bit(RUNNING_MERGE, &snap_merging->state_bits)) {
  1863. must_restart_merging = true;
  1864. stop_merge(snap_merging);
  1865. }
  1866. }
  1867. down_read(&_origins_lock);
  1868. (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
  1869. if (snap_src && snap_dest) {
  1870. down_write(&snap_src->lock);
  1871. down_write_nested(&snap_dest->lock, SINGLE_DEPTH_NESTING);
  1872. __handover_exceptions(snap_src, snap_dest);
  1873. up_write(&snap_dest->lock);
  1874. up_write(&snap_src->lock);
  1875. }
  1876. up_read(&_origins_lock);
  1877. if (origin_md) {
  1878. if (must_restart_merging)
  1879. start_merge(snap_merging);
  1880. dm_internal_resume_fast(origin_md);
  1881. dm_put(origin_md);
  1882. }
  1883. /* Now we have correct chunk size, reregister */
  1884. reregister_snapshot(s);
  1885. down_write(&s->lock);
  1886. s->active = 1;
  1887. up_write(&s->lock);
  1888. }
  1889. static uint32_t get_origin_minimum_chunksize(struct block_device *bdev)
  1890. {
  1891. uint32_t min_chunksize;
  1892. down_read(&_origins_lock);
  1893. min_chunksize = __minimum_chunk_size(__lookup_origin(bdev));
  1894. up_read(&_origins_lock);
  1895. return min_chunksize;
  1896. }
  1897. static void snapshot_merge_resume(struct dm_target *ti)
  1898. {
  1899. struct dm_snapshot *s = ti->private;
  1900. /*
  1901. * Handover exceptions from existing snapshot.
  1902. */
  1903. snapshot_resume(ti);
  1904. /*
  1905. * snapshot-merge acts as an origin, so set ti->max_io_len
  1906. */
  1907. ti->max_io_len = get_origin_minimum_chunksize(s->origin->bdev);
  1908. start_merge(s);
  1909. }
  1910. static void snapshot_status(struct dm_target *ti, status_type_t type,
  1911. unsigned status_flags, char *result, unsigned maxlen)
  1912. {
  1913. unsigned sz = 0;
  1914. struct dm_snapshot *snap = ti->private;
  1915. unsigned num_features;
  1916. switch (type) {
  1917. case STATUSTYPE_INFO:
  1918. down_write(&snap->lock);
  1919. if (!snap->valid)
  1920. DMEMIT("Invalid");
  1921. else if (snap->merge_failed)
  1922. DMEMIT("Merge failed");
  1923. else if (snap->snapshot_overflowed)
  1924. DMEMIT("Overflow");
  1925. else {
  1926. if (snap->store->type->usage) {
  1927. sector_t total_sectors, sectors_allocated,
  1928. metadata_sectors;
  1929. snap->store->type->usage(snap->store,
  1930. &total_sectors,
  1931. &sectors_allocated,
  1932. &metadata_sectors);
  1933. DMEMIT("%llu/%llu %llu",
  1934. (unsigned long long)sectors_allocated,
  1935. (unsigned long long)total_sectors,
  1936. (unsigned long long)metadata_sectors);
  1937. }
  1938. else
  1939. DMEMIT("Unknown");
  1940. }
  1941. up_write(&snap->lock);
  1942. break;
  1943. case STATUSTYPE_TABLE:
  1944. /*
  1945. * kdevname returns a static pointer so we need
  1946. * to make private copies if the output is to
  1947. * make sense.
  1948. */
  1949. DMEMIT("%s %s", snap->origin->name, snap->cow->name);
  1950. sz += snap->store->type->status(snap->store, type, result + sz,
  1951. maxlen - sz);
  1952. num_features = snap->discard_zeroes_cow + snap->discard_passdown_origin;
  1953. if (num_features) {
  1954. DMEMIT(" %u", num_features);
  1955. if (snap->discard_zeroes_cow)
  1956. DMEMIT(" discard_zeroes_cow");
  1957. if (snap->discard_passdown_origin)
  1958. DMEMIT(" discard_passdown_origin");
  1959. }
  1960. break;
  1961. }
  1962. }
  1963. static int snapshot_iterate_devices(struct dm_target *ti,
  1964. iterate_devices_callout_fn fn, void *data)
  1965. {
  1966. struct dm_snapshot *snap = ti->private;
  1967. int r;
  1968. r = fn(ti, snap->origin, 0, ti->len, data);
  1969. if (!r)
  1970. r = fn(ti, snap->cow, 0, get_dev_size(snap->cow->bdev), data);
  1971. return r;
  1972. }
  1973. static void snapshot_io_hints(struct dm_target *ti, struct queue_limits *limits)
  1974. {
  1975. struct dm_snapshot *snap = ti->private;
  1976. if (snap->discard_zeroes_cow) {
  1977. struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
  1978. down_read(&_origins_lock);
  1979. (void) __find_snapshots_sharing_cow(snap, &snap_src, &snap_dest, NULL);
  1980. if (snap_src && snap_dest)
  1981. snap = snap_src;
  1982. /* All discards are split on chunk_size boundary */
  1983. limits->discard_granularity = snap->store->chunk_size;
  1984. limits->max_discard_sectors = snap->store->chunk_size;
  1985. up_read(&_origins_lock);
  1986. }
  1987. }
  1988. /*-----------------------------------------------------------------
  1989. * Origin methods
  1990. *---------------------------------------------------------------*/
  1991. /*
  1992. * If no exceptions need creating, DM_MAPIO_REMAPPED is returned and any
  1993. * supplied bio was ignored. The caller may submit it immediately.
  1994. * (No remapping actually occurs as the origin is always a direct linear
  1995. * map.)
  1996. *
  1997. * If further exceptions are required, DM_MAPIO_SUBMITTED is returned
  1998. * and any supplied bio is added to a list to be submitted once all
  1999. * the necessary exceptions exist.
  2000. */
  2001. static int __origin_write(struct list_head *snapshots, sector_t sector,
  2002. struct bio *bio)
  2003. {
  2004. int r = DM_MAPIO_REMAPPED;
  2005. struct dm_snapshot *snap;
  2006. struct dm_exception *e;
  2007. struct dm_snap_pending_exception *pe, *pe2;
  2008. struct dm_snap_pending_exception *pe_to_start_now = NULL;
  2009. struct dm_snap_pending_exception *pe_to_start_last = NULL;
  2010. struct dm_exception_table_lock lock;
  2011. chunk_t chunk;
  2012. /* Do all the snapshots on this origin */
  2013. list_for_each_entry (snap, snapshots, list) {
  2014. /*
  2015. * Don't make new exceptions in a merging snapshot
  2016. * because it has effectively been deleted
  2017. */
  2018. if (dm_target_is_snapshot_merge(snap->ti))
  2019. continue;
  2020. /* Nothing to do if writing beyond end of snapshot */
  2021. if (sector >= dm_table_get_size(snap->ti->table))
  2022. continue;
  2023. /*
  2024. * Remember, different snapshots can have
  2025. * different chunk sizes.
  2026. */
  2027. chunk = sector_to_chunk(snap->store, sector);
  2028. dm_exception_table_lock_init(snap, chunk, &lock);
  2029. down_read(&snap->lock);
  2030. dm_exception_table_lock(&lock);
  2031. /* Only deal with valid and active snapshots */
  2032. if (!snap->valid || !snap->active)
  2033. goto next_snapshot;
  2034. pe = __lookup_pending_exception(snap, chunk);
  2035. if (!pe) {
  2036. /*
  2037. * Check exception table to see if block is already
  2038. * remapped in this snapshot and trigger an exception
  2039. * if not.
  2040. */
  2041. e = dm_lookup_exception(&snap->complete, chunk);
  2042. if (e)
  2043. goto next_snapshot;
  2044. dm_exception_table_unlock(&lock);
  2045. pe = alloc_pending_exception(snap);
  2046. dm_exception_table_lock(&lock);
  2047. pe2 = __lookup_pending_exception(snap, chunk);
  2048. if (!pe2) {
  2049. e = dm_lookup_exception(&snap->complete, chunk);
  2050. if (e) {
  2051. free_pending_exception(pe);
  2052. goto next_snapshot;
  2053. }
  2054. pe = __insert_pending_exception(snap, pe, chunk);
  2055. if (!pe) {
  2056. dm_exception_table_unlock(&lock);
  2057. up_read(&snap->lock);
  2058. invalidate_snapshot(snap, -ENOMEM);
  2059. continue;
  2060. }
  2061. } else {
  2062. free_pending_exception(pe);
  2063. pe = pe2;
  2064. }
  2065. }
  2066. r = DM_MAPIO_SUBMITTED;
  2067. /*
  2068. * If an origin bio was supplied, queue it to wait for the
  2069. * completion of this exception, and start this one last,
  2070. * at the end of the function.
  2071. */
  2072. if (bio) {
  2073. bio_list_add(&pe->origin_bios, bio);
  2074. bio = NULL;
  2075. if (!pe->started) {
  2076. pe->started = 1;
  2077. pe_to_start_last = pe;
  2078. }
  2079. }
  2080. if (!pe->started) {
  2081. pe->started = 1;
  2082. pe_to_start_now = pe;
  2083. }
  2084. next_snapshot:
  2085. dm_exception_table_unlock(&lock);
  2086. up_read(&snap->lock);
  2087. if (pe_to_start_now) {
  2088. start_copy(pe_to_start_now);
  2089. pe_to_start_now = NULL;
  2090. }
  2091. }
  2092. /*
  2093. * Submit the exception against which the bio is queued last,
  2094. * to give the other exceptions a head start.
  2095. */
  2096. if (pe_to_start_last)
  2097. start_copy(pe_to_start_last);
  2098. return r;
  2099. }
  2100. /*
  2101. * Called on a write from the origin driver.
  2102. */
  2103. static int do_origin(struct dm_dev *origin, struct bio *bio, bool limit)
  2104. {
  2105. struct origin *o;
  2106. int r = DM_MAPIO_REMAPPED;
  2107. again:
  2108. down_read(&_origins_lock);
  2109. o = __lookup_origin(origin->bdev);
  2110. if (o) {
  2111. if (limit) {
  2112. struct dm_snapshot *s;
  2113. list_for_each_entry(s, &o->snapshots, list)
  2114. if (unlikely(!wait_for_in_progress(s, true)))
  2115. goto again;
  2116. }
  2117. r = __origin_write(&o->snapshots, bio->bi_iter.bi_sector, bio);
  2118. }
  2119. up_read(&_origins_lock);
  2120. return r;
  2121. }
  2122. /*
  2123. * Trigger exceptions in all non-merging snapshots.
  2124. *
  2125. * The chunk size of the merging snapshot may be larger than the chunk
  2126. * size of some other snapshot so we may need to reallocate multiple
  2127. * chunks in other snapshots.
  2128. *
  2129. * We scan all the overlapping exceptions in the other snapshots.
  2130. * Returns 1 if anything was reallocated and must be waited for,
  2131. * otherwise returns 0.
  2132. *
  2133. * size must be a multiple of merging_snap's chunk_size.
  2134. */
  2135. static int origin_write_extent(struct dm_snapshot *merging_snap,
  2136. sector_t sector, unsigned size)
  2137. {
  2138. int must_wait = 0;
  2139. sector_t n;
  2140. struct origin *o;
  2141. /*
  2142. * The origin's __minimum_chunk_size() got stored in max_io_len
  2143. * by snapshot_merge_resume().
  2144. */
  2145. down_read(&_origins_lock);
  2146. o = __lookup_origin(merging_snap->origin->bdev);
  2147. for (n = 0; n < size; n += merging_snap->ti->max_io_len)
  2148. if (__origin_write(&o->snapshots, sector + n, NULL) ==
  2149. DM_MAPIO_SUBMITTED)
  2150. must_wait = 1;
  2151. up_read(&_origins_lock);
  2152. return must_wait;
  2153. }
  2154. /*
  2155. * Origin: maps a linear range of a device, with hooks for snapshotting.
  2156. */
  2157. /*
  2158. * Construct an origin mapping: <dev_path>
  2159. * The context for an origin is merely a 'struct dm_dev *'
  2160. * pointing to the real device.
  2161. */
  2162. static int origin_ctr(struct dm_target *ti, unsigned int argc, char **argv)
  2163. {
  2164. int r;
  2165. struct dm_origin *o;
  2166. if (argc != 1) {
  2167. ti->error = "origin: incorrect number of arguments";
  2168. return -EINVAL;
  2169. }
  2170. o = kmalloc(sizeof(struct dm_origin), GFP_KERNEL);
  2171. if (!o) {
  2172. ti->error = "Cannot allocate private origin structure";
  2173. r = -ENOMEM;
  2174. goto bad_alloc;
  2175. }
  2176. r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &o->dev);
  2177. if (r) {
  2178. ti->error = "Cannot get target device";
  2179. goto bad_open;
  2180. }
  2181. o->ti = ti;
  2182. ti->private = o;
  2183. ti->num_flush_bios = 1;
  2184. return 0;
  2185. bad_open:
  2186. kfree(o);
  2187. bad_alloc:
  2188. return r;
  2189. }
  2190. static void origin_dtr(struct dm_target *ti)
  2191. {
  2192. struct dm_origin *o = ti->private;
  2193. dm_put_device(ti, o->dev);
  2194. kfree(o);
  2195. }
  2196. static int origin_map(struct dm_target *ti, struct bio *bio)
  2197. {
  2198. struct dm_origin *o = ti->private;
  2199. unsigned available_sectors;
  2200. bio_set_dev(bio, o->dev->bdev);
  2201. if (unlikely(bio->bi_opf & REQ_PREFLUSH))
  2202. return DM_MAPIO_REMAPPED;
  2203. if (bio_data_dir(bio) != WRITE)
  2204. return DM_MAPIO_REMAPPED;
  2205. available_sectors = o->split_boundary -
  2206. ((unsigned)bio->bi_iter.bi_sector & (o->split_boundary - 1));
  2207. if (bio_sectors(bio) > available_sectors)
  2208. dm_accept_partial_bio(bio, available_sectors);
  2209. /* Only tell snapshots if this is a write */
  2210. return do_origin(o->dev, bio, true);
  2211. }
  2212. /*
  2213. * Set the target "max_io_len" field to the minimum of all the snapshots'
  2214. * chunk sizes.
  2215. */
  2216. static void origin_resume(struct dm_target *ti)
  2217. {
  2218. struct dm_origin *o = ti->private;
  2219. o->split_boundary = get_origin_minimum_chunksize(o->dev->bdev);
  2220. down_write(&_origins_lock);
  2221. __insert_dm_origin(o);
  2222. up_write(&_origins_lock);
  2223. }
  2224. static void origin_postsuspend(struct dm_target *ti)
  2225. {
  2226. struct dm_origin *o = ti->private;
  2227. down_write(&_origins_lock);
  2228. __remove_dm_origin(o);
  2229. up_write(&_origins_lock);
  2230. }
  2231. static void origin_status(struct dm_target *ti, status_type_t type,
  2232. unsigned status_flags, char *result, unsigned maxlen)
  2233. {
  2234. struct dm_origin *o = ti->private;
  2235. switch (type) {
  2236. case STATUSTYPE_INFO:
  2237. result[0] = '\0';
  2238. break;
  2239. case STATUSTYPE_TABLE:
  2240. snprintf(result, maxlen, "%s", o->dev->name);
  2241. break;
  2242. }
  2243. }
  2244. static int origin_iterate_devices(struct dm_target *ti,
  2245. iterate_devices_callout_fn fn, void *data)
  2246. {
  2247. struct dm_origin *o = ti->private;
  2248. return fn(ti, o->dev, 0, ti->len, data);
  2249. }
  2250. static struct target_type origin_target = {
  2251. .name = "snapshot-origin",
  2252. .version = {1, 9, 0},
  2253. .module = THIS_MODULE,
  2254. .ctr = origin_ctr,
  2255. .dtr = origin_dtr,
  2256. .map = origin_map,
  2257. .resume = origin_resume,
  2258. .postsuspend = origin_postsuspend,
  2259. .status = origin_status,
  2260. .iterate_devices = origin_iterate_devices,
  2261. };
  2262. static struct target_type snapshot_target = {
  2263. .name = "snapshot",
  2264. .version = {1, 16, 0},
  2265. .module = THIS_MODULE,
  2266. .ctr = snapshot_ctr,
  2267. .dtr = snapshot_dtr,
  2268. .map = snapshot_map,
  2269. .end_io = snapshot_end_io,
  2270. .preresume = snapshot_preresume,
  2271. .resume = snapshot_resume,
  2272. .status = snapshot_status,
  2273. .iterate_devices = snapshot_iterate_devices,
  2274. .io_hints = snapshot_io_hints,
  2275. };
  2276. static struct target_type merge_target = {
  2277. .name = dm_snapshot_merge_target_name,
  2278. .version = {1, 5, 0},
  2279. .module = THIS_MODULE,
  2280. .ctr = snapshot_ctr,
  2281. .dtr = snapshot_dtr,
  2282. .map = snapshot_merge_map,
  2283. .end_io = snapshot_end_io,
  2284. .presuspend = snapshot_merge_presuspend,
  2285. .preresume = snapshot_preresume,
  2286. .resume = snapshot_merge_resume,
  2287. .status = snapshot_status,
  2288. .iterate_devices = snapshot_iterate_devices,
  2289. .io_hints = snapshot_io_hints,
  2290. };
  2291. static int __init dm_snapshot_init(void)
  2292. {
  2293. int r;
  2294. r = dm_exception_store_init();
  2295. if (r) {
  2296. DMERR("Failed to initialize exception stores");
  2297. return r;
  2298. }
  2299. r = init_origin_hash();
  2300. if (r) {
  2301. DMERR("init_origin_hash failed.");
  2302. goto bad_origin_hash;
  2303. }
  2304. exception_cache = KMEM_CACHE(dm_exception, 0);
  2305. if (!exception_cache) {
  2306. DMERR("Couldn't create exception cache.");
  2307. r = -ENOMEM;
  2308. goto bad_exception_cache;
  2309. }
  2310. pending_cache = KMEM_CACHE(dm_snap_pending_exception, 0);
  2311. if (!pending_cache) {
  2312. DMERR("Couldn't create pending cache.");
  2313. r = -ENOMEM;
  2314. goto bad_pending_cache;
  2315. }
  2316. r = dm_register_target(&snapshot_target);
  2317. if (r < 0) {
  2318. DMERR("snapshot target register failed %d", r);
  2319. goto bad_register_snapshot_target;
  2320. }
  2321. r = dm_register_target(&origin_target);
  2322. if (r < 0) {
  2323. DMERR("Origin target register failed %d", r);
  2324. goto bad_register_origin_target;
  2325. }
  2326. r = dm_register_target(&merge_target);
  2327. if (r < 0) {
  2328. DMERR("Merge target register failed %d", r);
  2329. goto bad_register_merge_target;
  2330. }
  2331. return 0;
  2332. bad_register_merge_target:
  2333. dm_unregister_target(&origin_target);
  2334. bad_register_origin_target:
  2335. dm_unregister_target(&snapshot_target);
  2336. bad_register_snapshot_target:
  2337. kmem_cache_destroy(pending_cache);
  2338. bad_pending_cache:
  2339. kmem_cache_destroy(exception_cache);
  2340. bad_exception_cache:
  2341. exit_origin_hash();
  2342. bad_origin_hash:
  2343. dm_exception_store_exit();
  2344. return r;
  2345. }
  2346. static void __exit dm_snapshot_exit(void)
  2347. {
  2348. dm_unregister_target(&snapshot_target);
  2349. dm_unregister_target(&origin_target);
  2350. dm_unregister_target(&merge_target);
  2351. exit_origin_hash();
  2352. kmem_cache_destroy(pending_cache);
  2353. kmem_cache_destroy(exception_cache);
  2354. dm_exception_store_exit();
  2355. }
  2356. /* Module hooks */
  2357. module_init(dm_snapshot_init);
  2358. module_exit(dm_snapshot_exit);
  2359. MODULE_DESCRIPTION(DM_NAME " snapshot target");
  2360. MODULE_AUTHOR("Joe Thornber");
  2361. MODULE_LICENSE("GPL");
  2362. MODULE_ALIAS("dm-snapshot-origin");
  2363. MODULE_ALIAS("dm-snapshot-merge");