dm-bufio.c 49 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038
  1. /*
  2. * Copyright (C) 2009-2011 Red Hat, Inc.
  3. *
  4. * Author: Mikulas Patocka <mpatocka@redhat.com>
  5. *
  6. * This file is released under the GPL.
  7. */
  8. #include <linux/dm-bufio.h>
  9. #include <linux/device-mapper.h>
  10. #include <linux/dm-io.h>
  11. #include <linux/slab.h>
  12. #include <linux/sched/mm.h>
  13. #include <linux/jiffies.h>
  14. #include <linux/vmalloc.h>
  15. #include <linux/shrinker.h>
  16. #include <linux/module.h>
  17. #include <linux/rbtree.h>
  18. #include <linux/stacktrace.h>
  19. #define DM_MSG_PREFIX "bufio"
  20. /*
  21. * Memory management policy:
  22. * Limit the number of buffers to DM_BUFIO_MEMORY_PERCENT of main memory
  23. * or DM_BUFIO_VMALLOC_PERCENT of vmalloc memory (whichever is lower).
  24. * Always allocate at least DM_BUFIO_MIN_BUFFERS buffers.
  25. * Start background writeback when there are DM_BUFIO_WRITEBACK_PERCENT
  26. * dirty buffers.
  27. */
  28. #define DM_BUFIO_MIN_BUFFERS 8
  29. #define DM_BUFIO_MEMORY_PERCENT 2
  30. #define DM_BUFIO_VMALLOC_PERCENT 25
  31. #define DM_BUFIO_WRITEBACK_RATIO 3
  32. #define DM_BUFIO_LOW_WATERMARK_RATIO 16
  33. /*
  34. * Check buffer ages in this interval (seconds)
  35. */
  36. #define DM_BUFIO_WORK_TIMER_SECS 30
  37. /*
  38. * Free buffers when they are older than this (seconds)
  39. */
  40. #define DM_BUFIO_DEFAULT_AGE_SECS 300
  41. /*
  42. * The nr of bytes of cached data to keep around.
  43. */
  44. #define DM_BUFIO_DEFAULT_RETAIN_BYTES (256 * 1024)
  45. /*
  46. * Align buffer writes to this boundary.
  47. * Tests show that SSDs have the highest IOPS when using 4k writes.
  48. */
  49. #define DM_BUFIO_WRITE_ALIGN 4096
  50. /*
  51. * dm_buffer->list_mode
  52. */
  53. #define LIST_CLEAN 0
  54. #define LIST_DIRTY 1
  55. #define LIST_SIZE 2
  56. /*
  57. * Linking of buffers:
  58. * All buffers are linked to buffer_tree with their node field.
  59. *
  60. * Clean buffers that are not being written (B_WRITING not set)
  61. * are linked to lru[LIST_CLEAN] with their lru_list field.
  62. *
  63. * Dirty and clean buffers that are being written are linked to
  64. * lru[LIST_DIRTY] with their lru_list field. When the write
  65. * finishes, the buffer cannot be relinked immediately (because we
  66. * are in an interrupt context and relinking requires process
  67. * context), so some clean-not-writing buffers can be held on
  68. * dirty_lru too. They are later added to lru in the process
  69. * context.
  70. */
  71. struct dm_bufio_client {
  72. struct mutex lock;
  73. struct list_head lru[LIST_SIZE];
  74. unsigned long n_buffers[LIST_SIZE];
  75. struct block_device *bdev;
  76. unsigned block_size;
  77. s8 sectors_per_block_bits;
  78. void (*alloc_callback)(struct dm_buffer *);
  79. void (*write_callback)(struct dm_buffer *);
  80. struct kmem_cache *slab_buffer;
  81. struct kmem_cache *slab_cache;
  82. struct dm_io_client *dm_io;
  83. struct list_head reserved_buffers;
  84. unsigned need_reserved_buffers;
  85. unsigned minimum_buffers;
  86. struct rb_root buffer_tree;
  87. wait_queue_head_t free_buffer_wait;
  88. sector_t start;
  89. int async_write_error;
  90. struct list_head client_list;
  91. struct shrinker shrinker;
  92. };
  93. /*
  94. * Buffer state bits.
  95. */
  96. #define B_READING 0
  97. #define B_WRITING 1
  98. #define B_DIRTY 2
  99. /*
  100. * Describes how the block was allocated:
  101. * kmem_cache_alloc(), __get_free_pages() or vmalloc().
  102. * See the comment at alloc_buffer_data.
  103. */
  104. enum data_mode {
  105. DATA_MODE_SLAB = 0,
  106. DATA_MODE_GET_FREE_PAGES = 1,
  107. DATA_MODE_VMALLOC = 2,
  108. DATA_MODE_LIMIT = 3
  109. };
  110. struct dm_buffer {
  111. struct rb_node node;
  112. struct list_head lru_list;
  113. struct list_head global_list;
  114. sector_t block;
  115. void *data;
  116. unsigned char data_mode; /* DATA_MODE_* */
  117. unsigned char list_mode; /* LIST_* */
  118. blk_status_t read_error;
  119. blk_status_t write_error;
  120. unsigned accessed;
  121. unsigned hold_count;
  122. unsigned long state;
  123. unsigned long last_accessed;
  124. unsigned dirty_start;
  125. unsigned dirty_end;
  126. unsigned write_start;
  127. unsigned write_end;
  128. struct dm_bufio_client *c;
  129. struct list_head write_list;
  130. void (*end_io)(struct dm_buffer *, blk_status_t);
  131. #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
  132. #define MAX_STACK 10
  133. unsigned int stack_len;
  134. unsigned long stack_entries[MAX_STACK];
  135. #endif
  136. };
  137. /*----------------------------------------------------------------*/
  138. #define dm_bufio_in_request() (!!current->bio_list)
  139. static void dm_bufio_lock(struct dm_bufio_client *c)
  140. {
  141. mutex_lock_nested(&c->lock, dm_bufio_in_request());
  142. }
  143. static int dm_bufio_trylock(struct dm_bufio_client *c)
  144. {
  145. return mutex_trylock(&c->lock);
  146. }
  147. static void dm_bufio_unlock(struct dm_bufio_client *c)
  148. {
  149. mutex_unlock(&c->lock);
  150. }
  151. /*----------------------------------------------------------------*/
  152. /*
  153. * Default cache size: available memory divided by the ratio.
  154. */
  155. static unsigned long dm_bufio_default_cache_size;
  156. /*
  157. * Total cache size set by the user.
  158. */
  159. static unsigned long dm_bufio_cache_size;
  160. /*
  161. * A copy of dm_bufio_cache_size because dm_bufio_cache_size can change
  162. * at any time. If it disagrees, the user has changed cache size.
  163. */
  164. static unsigned long dm_bufio_cache_size_latch;
  165. static DEFINE_SPINLOCK(global_spinlock);
  166. static LIST_HEAD(global_queue);
  167. static unsigned long global_num = 0;
  168. /*
  169. * Buffers are freed after this timeout
  170. */
  171. static unsigned dm_bufio_max_age = DM_BUFIO_DEFAULT_AGE_SECS;
  172. static unsigned long dm_bufio_retain_bytes = DM_BUFIO_DEFAULT_RETAIN_BYTES;
  173. static unsigned long dm_bufio_peak_allocated;
  174. static unsigned long dm_bufio_allocated_kmem_cache;
  175. static unsigned long dm_bufio_allocated_get_free_pages;
  176. static unsigned long dm_bufio_allocated_vmalloc;
  177. static unsigned long dm_bufio_current_allocated;
  178. /*----------------------------------------------------------------*/
  179. /*
  180. * The current number of clients.
  181. */
  182. static int dm_bufio_client_count;
  183. /*
  184. * The list of all clients.
  185. */
  186. static LIST_HEAD(dm_bufio_all_clients);
  187. /*
  188. * This mutex protects dm_bufio_cache_size_latch and dm_bufio_client_count
  189. */
  190. static DEFINE_MUTEX(dm_bufio_clients_lock);
  191. static struct workqueue_struct *dm_bufio_wq;
  192. static struct delayed_work dm_bufio_cleanup_old_work;
  193. static struct work_struct dm_bufio_replacement_work;
  194. #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
  195. static void buffer_record_stack(struct dm_buffer *b)
  196. {
  197. b->stack_len = stack_trace_save(b->stack_entries, MAX_STACK, 2);
  198. }
  199. #endif
  200. /*----------------------------------------------------------------
  201. * A red/black tree acts as an index for all the buffers.
  202. *--------------------------------------------------------------*/
  203. static struct dm_buffer *__find(struct dm_bufio_client *c, sector_t block)
  204. {
  205. struct rb_node *n = c->buffer_tree.rb_node;
  206. struct dm_buffer *b;
  207. while (n) {
  208. b = container_of(n, struct dm_buffer, node);
  209. if (b->block == block)
  210. return b;
  211. n = (b->block < block) ? n->rb_left : n->rb_right;
  212. }
  213. return NULL;
  214. }
  215. static void __insert(struct dm_bufio_client *c, struct dm_buffer *b)
  216. {
  217. struct rb_node **new = &c->buffer_tree.rb_node, *parent = NULL;
  218. struct dm_buffer *found;
  219. while (*new) {
  220. found = container_of(*new, struct dm_buffer, node);
  221. if (found->block == b->block) {
  222. BUG_ON(found != b);
  223. return;
  224. }
  225. parent = *new;
  226. new = (found->block < b->block) ?
  227. &((*new)->rb_left) : &((*new)->rb_right);
  228. }
  229. rb_link_node(&b->node, parent, new);
  230. rb_insert_color(&b->node, &c->buffer_tree);
  231. }
  232. static void __remove(struct dm_bufio_client *c, struct dm_buffer *b)
  233. {
  234. rb_erase(&b->node, &c->buffer_tree);
  235. }
  236. /*----------------------------------------------------------------*/
  237. static void adjust_total_allocated(struct dm_buffer *b, bool unlink)
  238. {
  239. unsigned char data_mode;
  240. long diff;
  241. static unsigned long * const class_ptr[DATA_MODE_LIMIT] = {
  242. &dm_bufio_allocated_kmem_cache,
  243. &dm_bufio_allocated_get_free_pages,
  244. &dm_bufio_allocated_vmalloc,
  245. };
  246. data_mode = b->data_mode;
  247. diff = (long)b->c->block_size;
  248. if (unlink)
  249. diff = -diff;
  250. spin_lock(&global_spinlock);
  251. *class_ptr[data_mode] += diff;
  252. dm_bufio_current_allocated += diff;
  253. if (dm_bufio_current_allocated > dm_bufio_peak_allocated)
  254. dm_bufio_peak_allocated = dm_bufio_current_allocated;
  255. b->accessed = 1;
  256. if (!unlink) {
  257. list_add(&b->global_list, &global_queue);
  258. global_num++;
  259. if (dm_bufio_current_allocated > dm_bufio_cache_size)
  260. queue_work(dm_bufio_wq, &dm_bufio_replacement_work);
  261. } else {
  262. list_del(&b->global_list);
  263. global_num--;
  264. }
  265. spin_unlock(&global_spinlock);
  266. }
  267. /*
  268. * Change the number of clients and recalculate per-client limit.
  269. */
  270. static void __cache_size_refresh(void)
  271. {
  272. BUG_ON(!mutex_is_locked(&dm_bufio_clients_lock));
  273. BUG_ON(dm_bufio_client_count < 0);
  274. dm_bufio_cache_size_latch = READ_ONCE(dm_bufio_cache_size);
  275. /*
  276. * Use default if set to 0 and report the actual cache size used.
  277. */
  278. if (!dm_bufio_cache_size_latch) {
  279. (void)cmpxchg(&dm_bufio_cache_size, 0,
  280. dm_bufio_default_cache_size);
  281. dm_bufio_cache_size_latch = dm_bufio_default_cache_size;
  282. }
  283. }
  284. /*
  285. * Allocating buffer data.
  286. *
  287. * Small buffers are allocated with kmem_cache, to use space optimally.
  288. *
  289. * For large buffers, we choose between get_free_pages and vmalloc.
  290. * Each has advantages and disadvantages.
  291. *
  292. * __get_free_pages can randomly fail if the memory is fragmented.
  293. * __vmalloc won't randomly fail, but vmalloc space is limited (it may be
  294. * as low as 128M) so using it for caching is not appropriate.
  295. *
  296. * If the allocation may fail we use __get_free_pages. Memory fragmentation
  297. * won't have a fatal effect here, but it just causes flushes of some other
  298. * buffers and more I/O will be performed. Don't use __get_free_pages if it
  299. * always fails (i.e. order >= MAX_ORDER).
  300. *
  301. * If the allocation shouldn't fail we use __vmalloc. This is only for the
  302. * initial reserve allocation, so there's no risk of wasting all vmalloc
  303. * space.
  304. */
  305. static void *alloc_buffer_data(struct dm_bufio_client *c, gfp_t gfp_mask,
  306. unsigned char *data_mode)
  307. {
  308. if (unlikely(c->slab_cache != NULL)) {
  309. *data_mode = DATA_MODE_SLAB;
  310. return kmem_cache_alloc(c->slab_cache, gfp_mask);
  311. }
  312. if (c->block_size <= KMALLOC_MAX_SIZE &&
  313. gfp_mask & __GFP_NORETRY) {
  314. *data_mode = DATA_MODE_GET_FREE_PAGES;
  315. return (void *)__get_free_pages(gfp_mask,
  316. c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT));
  317. }
  318. *data_mode = DATA_MODE_VMALLOC;
  319. /*
  320. * __vmalloc allocates the data pages and auxiliary structures with
  321. * gfp_flags that were specified, but pagetables are always allocated
  322. * with GFP_KERNEL, no matter what was specified as gfp_mask.
  323. *
  324. * Consequently, we must set per-process flag PF_MEMALLOC_NOIO so that
  325. * all allocations done by this process (including pagetables) are done
  326. * as if GFP_NOIO was specified.
  327. */
  328. if (gfp_mask & __GFP_NORETRY) {
  329. unsigned noio_flag = memalloc_noio_save();
  330. void *ptr = __vmalloc(c->block_size, gfp_mask, PAGE_KERNEL);
  331. memalloc_noio_restore(noio_flag);
  332. return ptr;
  333. }
  334. return __vmalloc(c->block_size, gfp_mask, PAGE_KERNEL);
  335. }
  336. /*
  337. * Free buffer's data.
  338. */
  339. static void free_buffer_data(struct dm_bufio_client *c,
  340. void *data, unsigned char data_mode)
  341. {
  342. switch (data_mode) {
  343. case DATA_MODE_SLAB:
  344. kmem_cache_free(c->slab_cache, data);
  345. break;
  346. case DATA_MODE_GET_FREE_PAGES:
  347. free_pages((unsigned long)data,
  348. c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT));
  349. break;
  350. case DATA_MODE_VMALLOC:
  351. vfree(data);
  352. break;
  353. default:
  354. DMCRIT("dm_bufio_free_buffer_data: bad data mode: %d",
  355. data_mode);
  356. BUG();
  357. }
  358. }
  359. /*
  360. * Allocate buffer and its data.
  361. */
  362. static struct dm_buffer *alloc_buffer(struct dm_bufio_client *c, gfp_t gfp_mask)
  363. {
  364. struct dm_buffer *b = kmem_cache_alloc(c->slab_buffer, gfp_mask);
  365. if (!b)
  366. return NULL;
  367. b->c = c;
  368. b->data = alloc_buffer_data(c, gfp_mask, &b->data_mode);
  369. if (!b->data) {
  370. kmem_cache_free(c->slab_buffer, b);
  371. return NULL;
  372. }
  373. #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
  374. b->stack_len = 0;
  375. #endif
  376. return b;
  377. }
  378. /*
  379. * Free buffer and its data.
  380. */
  381. static void free_buffer(struct dm_buffer *b)
  382. {
  383. struct dm_bufio_client *c = b->c;
  384. free_buffer_data(c, b->data, b->data_mode);
  385. kmem_cache_free(c->slab_buffer, b);
  386. }
  387. /*
  388. * Link buffer to the buffer tree and clean or dirty queue.
  389. */
  390. static void __link_buffer(struct dm_buffer *b, sector_t block, int dirty)
  391. {
  392. struct dm_bufio_client *c = b->c;
  393. c->n_buffers[dirty]++;
  394. b->block = block;
  395. b->list_mode = dirty;
  396. list_add(&b->lru_list, &c->lru[dirty]);
  397. __insert(b->c, b);
  398. b->last_accessed = jiffies;
  399. adjust_total_allocated(b, false);
  400. }
  401. /*
  402. * Unlink buffer from the buffer tree and dirty or clean queue.
  403. */
  404. static void __unlink_buffer(struct dm_buffer *b)
  405. {
  406. struct dm_bufio_client *c = b->c;
  407. BUG_ON(!c->n_buffers[b->list_mode]);
  408. c->n_buffers[b->list_mode]--;
  409. __remove(b->c, b);
  410. list_del(&b->lru_list);
  411. adjust_total_allocated(b, true);
  412. }
  413. /*
  414. * Place the buffer to the head of dirty or clean LRU queue.
  415. */
  416. static void __relink_lru(struct dm_buffer *b, int dirty)
  417. {
  418. struct dm_bufio_client *c = b->c;
  419. b->accessed = 1;
  420. BUG_ON(!c->n_buffers[b->list_mode]);
  421. c->n_buffers[b->list_mode]--;
  422. c->n_buffers[dirty]++;
  423. b->list_mode = dirty;
  424. list_move(&b->lru_list, &c->lru[dirty]);
  425. b->last_accessed = jiffies;
  426. }
  427. /*----------------------------------------------------------------
  428. * Submit I/O on the buffer.
  429. *
  430. * Bio interface is faster but it has some problems:
  431. * the vector list is limited (increasing this limit increases
  432. * memory-consumption per buffer, so it is not viable);
  433. *
  434. * the memory must be direct-mapped, not vmalloced;
  435. *
  436. * If the buffer is small enough (up to DM_BUFIO_INLINE_VECS pages) and
  437. * it is not vmalloced, try using the bio interface.
  438. *
  439. * If the buffer is big, if it is vmalloced or if the underlying device
  440. * rejects the bio because it is too large, use dm-io layer to do the I/O.
  441. * The dm-io layer splits the I/O into multiple requests, avoiding the above
  442. * shortcomings.
  443. *--------------------------------------------------------------*/
  444. /*
  445. * dm-io completion routine. It just calls b->bio.bi_end_io, pretending
  446. * that the request was handled directly with bio interface.
  447. */
  448. static void dmio_complete(unsigned long error, void *context)
  449. {
  450. struct dm_buffer *b = context;
  451. b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : 0);
  452. }
  453. static void use_dmio(struct dm_buffer *b, int rw, sector_t sector,
  454. unsigned n_sectors, unsigned offset)
  455. {
  456. int r;
  457. struct dm_io_request io_req = {
  458. .bi_op = rw,
  459. .bi_op_flags = 0,
  460. .notify.fn = dmio_complete,
  461. .notify.context = b,
  462. .client = b->c->dm_io,
  463. };
  464. struct dm_io_region region = {
  465. .bdev = b->c->bdev,
  466. .sector = sector,
  467. .count = n_sectors,
  468. };
  469. if (b->data_mode != DATA_MODE_VMALLOC) {
  470. io_req.mem.type = DM_IO_KMEM;
  471. io_req.mem.ptr.addr = (char *)b->data + offset;
  472. } else {
  473. io_req.mem.type = DM_IO_VMA;
  474. io_req.mem.ptr.vma = (char *)b->data + offset;
  475. }
  476. r = dm_io(&io_req, 1, &region, NULL);
  477. if (unlikely(r))
  478. b->end_io(b, errno_to_blk_status(r));
  479. }
  480. static void bio_complete(struct bio *bio)
  481. {
  482. struct dm_buffer *b = bio->bi_private;
  483. blk_status_t status = bio->bi_status;
  484. bio_put(bio);
  485. b->end_io(b, status);
  486. }
  487. static void use_bio(struct dm_buffer *b, int rw, sector_t sector,
  488. unsigned n_sectors, unsigned offset)
  489. {
  490. struct bio *bio;
  491. char *ptr;
  492. unsigned vec_size, len;
  493. vec_size = b->c->block_size >> PAGE_SHIFT;
  494. if (unlikely(b->c->sectors_per_block_bits < PAGE_SHIFT - SECTOR_SHIFT))
  495. vec_size += 2;
  496. bio = bio_kmalloc(GFP_NOWAIT | __GFP_NORETRY | __GFP_NOWARN, vec_size);
  497. if (!bio) {
  498. dmio:
  499. use_dmio(b, rw, sector, n_sectors, offset);
  500. return;
  501. }
  502. bio->bi_iter.bi_sector = sector;
  503. bio_set_dev(bio, b->c->bdev);
  504. bio_set_op_attrs(bio, rw, 0);
  505. bio->bi_end_io = bio_complete;
  506. bio->bi_private = b;
  507. ptr = (char *)b->data + offset;
  508. len = n_sectors << SECTOR_SHIFT;
  509. do {
  510. unsigned this_step = min((unsigned)(PAGE_SIZE - offset_in_page(ptr)), len);
  511. if (!bio_add_page(bio, virt_to_page(ptr), this_step,
  512. offset_in_page(ptr))) {
  513. bio_put(bio);
  514. goto dmio;
  515. }
  516. len -= this_step;
  517. ptr += this_step;
  518. } while (len > 0);
  519. submit_bio(bio);
  520. }
  521. static void submit_io(struct dm_buffer *b, int rw, void (*end_io)(struct dm_buffer *, blk_status_t))
  522. {
  523. unsigned n_sectors;
  524. sector_t sector;
  525. unsigned offset, end;
  526. b->end_io = end_io;
  527. if (likely(b->c->sectors_per_block_bits >= 0))
  528. sector = b->block << b->c->sectors_per_block_bits;
  529. else
  530. sector = b->block * (b->c->block_size >> SECTOR_SHIFT);
  531. sector += b->c->start;
  532. if (rw != REQ_OP_WRITE) {
  533. n_sectors = b->c->block_size >> SECTOR_SHIFT;
  534. offset = 0;
  535. } else {
  536. if (b->c->write_callback)
  537. b->c->write_callback(b);
  538. offset = b->write_start;
  539. end = b->write_end;
  540. offset &= -DM_BUFIO_WRITE_ALIGN;
  541. end += DM_BUFIO_WRITE_ALIGN - 1;
  542. end &= -DM_BUFIO_WRITE_ALIGN;
  543. if (unlikely(end > b->c->block_size))
  544. end = b->c->block_size;
  545. sector += offset >> SECTOR_SHIFT;
  546. n_sectors = (end - offset) >> SECTOR_SHIFT;
  547. }
  548. if (b->data_mode != DATA_MODE_VMALLOC)
  549. use_bio(b, rw, sector, n_sectors, offset);
  550. else
  551. use_dmio(b, rw, sector, n_sectors, offset);
  552. }
  553. /*----------------------------------------------------------------
  554. * Writing dirty buffers
  555. *--------------------------------------------------------------*/
  556. /*
  557. * The endio routine for write.
  558. *
  559. * Set the error, clear B_WRITING bit and wake anyone who was waiting on
  560. * it.
  561. */
  562. static void write_endio(struct dm_buffer *b, blk_status_t status)
  563. {
  564. b->write_error = status;
  565. if (unlikely(status)) {
  566. struct dm_bufio_client *c = b->c;
  567. (void)cmpxchg(&c->async_write_error, 0,
  568. blk_status_to_errno(status));
  569. }
  570. BUG_ON(!test_bit(B_WRITING, &b->state));
  571. smp_mb__before_atomic();
  572. clear_bit(B_WRITING, &b->state);
  573. smp_mb__after_atomic();
  574. wake_up_bit(&b->state, B_WRITING);
  575. }
  576. /*
  577. * Initiate a write on a dirty buffer, but don't wait for it.
  578. *
  579. * - If the buffer is not dirty, exit.
  580. * - If there some previous write going on, wait for it to finish (we can't
  581. * have two writes on the same buffer simultaneously).
  582. * - Submit our write and don't wait on it. We set B_WRITING indicating
  583. * that there is a write in progress.
  584. */
  585. static void __write_dirty_buffer(struct dm_buffer *b,
  586. struct list_head *write_list)
  587. {
  588. if (!test_bit(B_DIRTY, &b->state))
  589. return;
  590. clear_bit(B_DIRTY, &b->state);
  591. wait_on_bit_lock_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE);
  592. b->write_start = b->dirty_start;
  593. b->write_end = b->dirty_end;
  594. if (!write_list)
  595. submit_io(b, REQ_OP_WRITE, write_endio);
  596. else
  597. list_add_tail(&b->write_list, write_list);
  598. }
  599. static void __flush_write_list(struct list_head *write_list)
  600. {
  601. struct blk_plug plug;
  602. blk_start_plug(&plug);
  603. while (!list_empty(write_list)) {
  604. struct dm_buffer *b =
  605. list_entry(write_list->next, struct dm_buffer, write_list);
  606. list_del(&b->write_list);
  607. submit_io(b, REQ_OP_WRITE, write_endio);
  608. cond_resched();
  609. }
  610. blk_finish_plug(&plug);
  611. }
  612. /*
  613. * Wait until any activity on the buffer finishes. Possibly write the
  614. * buffer if it is dirty. When this function finishes, there is no I/O
  615. * running on the buffer and the buffer is not dirty.
  616. */
  617. static void __make_buffer_clean(struct dm_buffer *b)
  618. {
  619. BUG_ON(b->hold_count);
  620. if (!b->state) /* fast case */
  621. return;
  622. wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE);
  623. __write_dirty_buffer(b, NULL);
  624. wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE);
  625. }
  626. /*
  627. * Find some buffer that is not held by anybody, clean it, unlink it and
  628. * return it.
  629. */
  630. static struct dm_buffer *__get_unclaimed_buffer(struct dm_bufio_client *c)
  631. {
  632. struct dm_buffer *b;
  633. list_for_each_entry_reverse(b, &c->lru[LIST_CLEAN], lru_list) {
  634. BUG_ON(test_bit(B_WRITING, &b->state));
  635. BUG_ON(test_bit(B_DIRTY, &b->state));
  636. if (!b->hold_count) {
  637. __make_buffer_clean(b);
  638. __unlink_buffer(b);
  639. return b;
  640. }
  641. cond_resched();
  642. }
  643. list_for_each_entry_reverse(b, &c->lru[LIST_DIRTY], lru_list) {
  644. BUG_ON(test_bit(B_READING, &b->state));
  645. if (!b->hold_count) {
  646. __make_buffer_clean(b);
  647. __unlink_buffer(b);
  648. return b;
  649. }
  650. cond_resched();
  651. }
  652. return NULL;
  653. }
  654. /*
  655. * Wait until some other threads free some buffer or release hold count on
  656. * some buffer.
  657. *
  658. * This function is entered with c->lock held, drops it and regains it
  659. * before exiting.
  660. */
  661. static void __wait_for_free_buffer(struct dm_bufio_client *c)
  662. {
  663. DECLARE_WAITQUEUE(wait, current);
  664. add_wait_queue(&c->free_buffer_wait, &wait);
  665. set_current_state(TASK_UNINTERRUPTIBLE);
  666. dm_bufio_unlock(c);
  667. io_schedule();
  668. remove_wait_queue(&c->free_buffer_wait, &wait);
  669. dm_bufio_lock(c);
  670. }
  671. enum new_flag {
  672. NF_FRESH = 0,
  673. NF_READ = 1,
  674. NF_GET = 2,
  675. NF_PREFETCH = 3
  676. };
  677. /*
  678. * Allocate a new buffer. If the allocation is not possible, wait until
  679. * some other thread frees a buffer.
  680. *
  681. * May drop the lock and regain it.
  682. */
  683. static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c, enum new_flag nf)
  684. {
  685. struct dm_buffer *b;
  686. bool tried_noio_alloc = false;
  687. /*
  688. * dm-bufio is resistant to allocation failures (it just keeps
  689. * one buffer reserved in cases all the allocations fail).
  690. * So set flags to not try too hard:
  691. * GFP_NOWAIT: don't wait; if we need to sleep we'll release our
  692. * mutex and wait ourselves.
  693. * __GFP_NORETRY: don't retry and rather return failure
  694. * __GFP_NOMEMALLOC: don't use emergency reserves
  695. * __GFP_NOWARN: don't print a warning in case of failure
  696. *
  697. * For debugging, if we set the cache size to 1, no new buffers will
  698. * be allocated.
  699. */
  700. while (1) {
  701. if (dm_bufio_cache_size_latch != 1) {
  702. b = alloc_buffer(c, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
  703. if (b)
  704. return b;
  705. }
  706. if (nf == NF_PREFETCH)
  707. return NULL;
  708. if (dm_bufio_cache_size_latch != 1 && !tried_noio_alloc) {
  709. dm_bufio_unlock(c);
  710. b = alloc_buffer(c, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
  711. dm_bufio_lock(c);
  712. if (b)
  713. return b;
  714. tried_noio_alloc = true;
  715. }
  716. if (!list_empty(&c->reserved_buffers)) {
  717. b = list_entry(c->reserved_buffers.next,
  718. struct dm_buffer, lru_list);
  719. list_del(&b->lru_list);
  720. c->need_reserved_buffers++;
  721. return b;
  722. }
  723. b = __get_unclaimed_buffer(c);
  724. if (b)
  725. return b;
  726. __wait_for_free_buffer(c);
  727. }
  728. }
  729. static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c, enum new_flag nf)
  730. {
  731. struct dm_buffer *b = __alloc_buffer_wait_no_callback(c, nf);
  732. if (!b)
  733. return NULL;
  734. if (c->alloc_callback)
  735. c->alloc_callback(b);
  736. return b;
  737. }
  738. /*
  739. * Free a buffer and wake other threads waiting for free buffers.
  740. */
  741. static void __free_buffer_wake(struct dm_buffer *b)
  742. {
  743. struct dm_bufio_client *c = b->c;
  744. if (!c->need_reserved_buffers)
  745. free_buffer(b);
  746. else {
  747. list_add(&b->lru_list, &c->reserved_buffers);
  748. c->need_reserved_buffers--;
  749. }
  750. wake_up(&c->free_buffer_wait);
  751. }
  752. static void __write_dirty_buffers_async(struct dm_bufio_client *c, int no_wait,
  753. struct list_head *write_list)
  754. {
  755. struct dm_buffer *b, *tmp;
  756. list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_DIRTY], lru_list) {
  757. BUG_ON(test_bit(B_READING, &b->state));
  758. if (!test_bit(B_DIRTY, &b->state) &&
  759. !test_bit(B_WRITING, &b->state)) {
  760. __relink_lru(b, LIST_CLEAN);
  761. continue;
  762. }
  763. if (no_wait && test_bit(B_WRITING, &b->state))
  764. return;
  765. __write_dirty_buffer(b, write_list);
  766. cond_resched();
  767. }
  768. }
  769. /*
  770. * Check if we're over watermark.
  771. * If we are over threshold_buffers, start freeing buffers.
  772. * If we're over "limit_buffers", block until we get under the limit.
  773. */
  774. static void __check_watermark(struct dm_bufio_client *c,
  775. struct list_head *write_list)
  776. {
  777. if (c->n_buffers[LIST_DIRTY] > c->n_buffers[LIST_CLEAN] * DM_BUFIO_WRITEBACK_RATIO)
  778. __write_dirty_buffers_async(c, 1, write_list);
  779. }
  780. /*----------------------------------------------------------------
  781. * Getting a buffer
  782. *--------------------------------------------------------------*/
  783. static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block,
  784. enum new_flag nf, int *need_submit,
  785. struct list_head *write_list)
  786. {
  787. struct dm_buffer *b, *new_b = NULL;
  788. *need_submit = 0;
  789. b = __find(c, block);
  790. if (b)
  791. goto found_buffer;
  792. if (nf == NF_GET)
  793. return NULL;
  794. new_b = __alloc_buffer_wait(c, nf);
  795. if (!new_b)
  796. return NULL;
  797. /*
  798. * We've had a period where the mutex was unlocked, so need to
  799. * recheck the buffer tree.
  800. */
  801. b = __find(c, block);
  802. if (b) {
  803. __free_buffer_wake(new_b);
  804. goto found_buffer;
  805. }
  806. __check_watermark(c, write_list);
  807. b = new_b;
  808. b->hold_count = 1;
  809. b->read_error = 0;
  810. b->write_error = 0;
  811. __link_buffer(b, block, LIST_CLEAN);
  812. if (nf == NF_FRESH) {
  813. b->state = 0;
  814. return b;
  815. }
  816. b->state = 1 << B_READING;
  817. *need_submit = 1;
  818. return b;
  819. found_buffer:
  820. if (nf == NF_PREFETCH)
  821. return NULL;
  822. /*
  823. * Note: it is essential that we don't wait for the buffer to be
  824. * read if dm_bufio_get function is used. Both dm_bufio_get and
  825. * dm_bufio_prefetch can be used in the driver request routine.
  826. * If the user called both dm_bufio_prefetch and dm_bufio_get on
  827. * the same buffer, it would deadlock if we waited.
  828. */
  829. if (nf == NF_GET && unlikely(test_bit(B_READING, &b->state)))
  830. return NULL;
  831. b->hold_count++;
  832. __relink_lru(b, test_bit(B_DIRTY, &b->state) ||
  833. test_bit(B_WRITING, &b->state));
  834. return b;
  835. }
  836. /*
  837. * The endio routine for reading: set the error, clear the bit and wake up
  838. * anyone waiting on the buffer.
  839. */
  840. static void read_endio(struct dm_buffer *b, blk_status_t status)
  841. {
  842. b->read_error = status;
  843. BUG_ON(!test_bit(B_READING, &b->state));
  844. smp_mb__before_atomic();
  845. clear_bit(B_READING, &b->state);
  846. smp_mb__after_atomic();
  847. wake_up_bit(&b->state, B_READING);
  848. }
  849. /*
  850. * A common routine for dm_bufio_new and dm_bufio_read. Operation of these
  851. * functions is similar except that dm_bufio_new doesn't read the
  852. * buffer from the disk (assuming that the caller overwrites all the data
  853. * and uses dm_bufio_mark_buffer_dirty to write new data back).
  854. */
  855. static void *new_read(struct dm_bufio_client *c, sector_t block,
  856. enum new_flag nf, struct dm_buffer **bp)
  857. {
  858. int need_submit;
  859. struct dm_buffer *b;
  860. LIST_HEAD(write_list);
  861. dm_bufio_lock(c);
  862. b = __bufio_new(c, block, nf, &need_submit, &write_list);
  863. #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
  864. if (b && b->hold_count == 1)
  865. buffer_record_stack(b);
  866. #endif
  867. dm_bufio_unlock(c);
  868. __flush_write_list(&write_list);
  869. if (!b)
  870. return NULL;
  871. if (need_submit)
  872. submit_io(b, REQ_OP_READ, read_endio);
  873. wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE);
  874. if (b->read_error) {
  875. int error = blk_status_to_errno(b->read_error);
  876. dm_bufio_release(b);
  877. return ERR_PTR(error);
  878. }
  879. *bp = b;
  880. return b->data;
  881. }
  882. void *dm_bufio_get(struct dm_bufio_client *c, sector_t block,
  883. struct dm_buffer **bp)
  884. {
  885. return new_read(c, block, NF_GET, bp);
  886. }
  887. EXPORT_SYMBOL_GPL(dm_bufio_get);
  888. void *dm_bufio_read(struct dm_bufio_client *c, sector_t block,
  889. struct dm_buffer **bp)
  890. {
  891. BUG_ON(dm_bufio_in_request());
  892. return new_read(c, block, NF_READ, bp);
  893. }
  894. EXPORT_SYMBOL_GPL(dm_bufio_read);
  895. void *dm_bufio_new(struct dm_bufio_client *c, sector_t block,
  896. struct dm_buffer **bp)
  897. {
  898. BUG_ON(dm_bufio_in_request());
  899. return new_read(c, block, NF_FRESH, bp);
  900. }
  901. EXPORT_SYMBOL_GPL(dm_bufio_new);
  902. void dm_bufio_prefetch(struct dm_bufio_client *c,
  903. sector_t block, unsigned n_blocks)
  904. {
  905. struct blk_plug plug;
  906. LIST_HEAD(write_list);
  907. BUG_ON(dm_bufio_in_request());
  908. blk_start_plug(&plug);
  909. dm_bufio_lock(c);
  910. for (; n_blocks--; block++) {
  911. int need_submit;
  912. struct dm_buffer *b;
  913. b = __bufio_new(c, block, NF_PREFETCH, &need_submit,
  914. &write_list);
  915. if (unlikely(!list_empty(&write_list))) {
  916. dm_bufio_unlock(c);
  917. blk_finish_plug(&plug);
  918. __flush_write_list(&write_list);
  919. blk_start_plug(&plug);
  920. dm_bufio_lock(c);
  921. }
  922. if (unlikely(b != NULL)) {
  923. dm_bufio_unlock(c);
  924. if (need_submit)
  925. submit_io(b, REQ_OP_READ, read_endio);
  926. dm_bufio_release(b);
  927. cond_resched();
  928. if (!n_blocks)
  929. goto flush_plug;
  930. dm_bufio_lock(c);
  931. }
  932. }
  933. dm_bufio_unlock(c);
  934. flush_plug:
  935. blk_finish_plug(&plug);
  936. }
  937. EXPORT_SYMBOL_GPL(dm_bufio_prefetch);
  938. void dm_bufio_release(struct dm_buffer *b)
  939. {
  940. struct dm_bufio_client *c = b->c;
  941. dm_bufio_lock(c);
  942. BUG_ON(!b->hold_count);
  943. b->hold_count--;
  944. if (!b->hold_count) {
  945. wake_up(&c->free_buffer_wait);
  946. /*
  947. * If there were errors on the buffer, and the buffer is not
  948. * to be written, free the buffer. There is no point in caching
  949. * invalid buffer.
  950. */
  951. if ((b->read_error || b->write_error) &&
  952. !test_bit(B_READING, &b->state) &&
  953. !test_bit(B_WRITING, &b->state) &&
  954. !test_bit(B_DIRTY, &b->state)) {
  955. __unlink_buffer(b);
  956. __free_buffer_wake(b);
  957. }
  958. }
  959. dm_bufio_unlock(c);
  960. }
  961. EXPORT_SYMBOL_GPL(dm_bufio_release);
  962. void dm_bufio_mark_partial_buffer_dirty(struct dm_buffer *b,
  963. unsigned start, unsigned end)
  964. {
  965. struct dm_bufio_client *c = b->c;
  966. BUG_ON(start >= end);
  967. BUG_ON(end > b->c->block_size);
  968. dm_bufio_lock(c);
  969. BUG_ON(test_bit(B_READING, &b->state));
  970. if (!test_and_set_bit(B_DIRTY, &b->state)) {
  971. b->dirty_start = start;
  972. b->dirty_end = end;
  973. __relink_lru(b, LIST_DIRTY);
  974. } else {
  975. if (start < b->dirty_start)
  976. b->dirty_start = start;
  977. if (end > b->dirty_end)
  978. b->dirty_end = end;
  979. }
  980. dm_bufio_unlock(c);
  981. }
  982. EXPORT_SYMBOL_GPL(dm_bufio_mark_partial_buffer_dirty);
  983. void dm_bufio_mark_buffer_dirty(struct dm_buffer *b)
  984. {
  985. dm_bufio_mark_partial_buffer_dirty(b, 0, b->c->block_size);
  986. }
  987. EXPORT_SYMBOL_GPL(dm_bufio_mark_buffer_dirty);
  988. void dm_bufio_write_dirty_buffers_async(struct dm_bufio_client *c)
  989. {
  990. LIST_HEAD(write_list);
  991. BUG_ON(dm_bufio_in_request());
  992. dm_bufio_lock(c);
  993. __write_dirty_buffers_async(c, 0, &write_list);
  994. dm_bufio_unlock(c);
  995. __flush_write_list(&write_list);
  996. }
  997. EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers_async);
  998. /*
  999. * For performance, it is essential that the buffers are written asynchronously
  1000. * and simultaneously (so that the block layer can merge the writes) and then
  1001. * waited upon.
  1002. *
  1003. * Finally, we flush hardware disk cache.
  1004. */
  1005. int dm_bufio_write_dirty_buffers(struct dm_bufio_client *c)
  1006. {
  1007. int a, f;
  1008. unsigned long buffers_processed = 0;
  1009. struct dm_buffer *b, *tmp;
  1010. LIST_HEAD(write_list);
  1011. dm_bufio_lock(c);
  1012. __write_dirty_buffers_async(c, 0, &write_list);
  1013. dm_bufio_unlock(c);
  1014. __flush_write_list(&write_list);
  1015. dm_bufio_lock(c);
  1016. again:
  1017. list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_DIRTY], lru_list) {
  1018. int dropped_lock = 0;
  1019. if (buffers_processed < c->n_buffers[LIST_DIRTY])
  1020. buffers_processed++;
  1021. BUG_ON(test_bit(B_READING, &b->state));
  1022. if (test_bit(B_WRITING, &b->state)) {
  1023. if (buffers_processed < c->n_buffers[LIST_DIRTY]) {
  1024. dropped_lock = 1;
  1025. b->hold_count++;
  1026. dm_bufio_unlock(c);
  1027. wait_on_bit_io(&b->state, B_WRITING,
  1028. TASK_UNINTERRUPTIBLE);
  1029. dm_bufio_lock(c);
  1030. b->hold_count--;
  1031. } else
  1032. wait_on_bit_io(&b->state, B_WRITING,
  1033. TASK_UNINTERRUPTIBLE);
  1034. }
  1035. if (!test_bit(B_DIRTY, &b->state) &&
  1036. !test_bit(B_WRITING, &b->state))
  1037. __relink_lru(b, LIST_CLEAN);
  1038. cond_resched();
  1039. /*
  1040. * If we dropped the lock, the list is no longer consistent,
  1041. * so we must restart the search.
  1042. *
  1043. * In the most common case, the buffer just processed is
  1044. * relinked to the clean list, so we won't loop scanning the
  1045. * same buffer again and again.
  1046. *
  1047. * This may livelock if there is another thread simultaneously
  1048. * dirtying buffers, so we count the number of buffers walked
  1049. * and if it exceeds the total number of buffers, it means that
  1050. * someone is doing some writes simultaneously with us. In
  1051. * this case, stop, dropping the lock.
  1052. */
  1053. if (dropped_lock)
  1054. goto again;
  1055. }
  1056. wake_up(&c->free_buffer_wait);
  1057. dm_bufio_unlock(c);
  1058. a = xchg(&c->async_write_error, 0);
  1059. f = dm_bufio_issue_flush(c);
  1060. if (a)
  1061. return a;
  1062. return f;
  1063. }
  1064. EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers);
  1065. /*
  1066. * Use dm-io to send an empty barrier to flush the device.
  1067. */
  1068. int dm_bufio_issue_flush(struct dm_bufio_client *c)
  1069. {
  1070. struct dm_io_request io_req = {
  1071. .bi_op = REQ_OP_WRITE,
  1072. .bi_op_flags = REQ_PREFLUSH | REQ_SYNC,
  1073. .mem.type = DM_IO_KMEM,
  1074. .mem.ptr.addr = NULL,
  1075. .client = c->dm_io,
  1076. };
  1077. struct dm_io_region io_reg = {
  1078. .bdev = c->bdev,
  1079. .sector = 0,
  1080. .count = 0,
  1081. };
  1082. BUG_ON(dm_bufio_in_request());
  1083. return dm_io(&io_req, 1, &io_reg, NULL);
  1084. }
  1085. EXPORT_SYMBOL_GPL(dm_bufio_issue_flush);
  1086. /*
  1087. * We first delete any other buffer that may be at that new location.
  1088. *
  1089. * Then, we write the buffer to the original location if it was dirty.
  1090. *
  1091. * Then, if we are the only one who is holding the buffer, relink the buffer
  1092. * in the buffer tree for the new location.
  1093. *
  1094. * If there was someone else holding the buffer, we write it to the new
  1095. * location but not relink it, because that other user needs to have the buffer
  1096. * at the same place.
  1097. */
  1098. void dm_bufio_release_move(struct dm_buffer *b, sector_t new_block)
  1099. {
  1100. struct dm_bufio_client *c = b->c;
  1101. struct dm_buffer *new;
  1102. BUG_ON(dm_bufio_in_request());
  1103. dm_bufio_lock(c);
  1104. retry:
  1105. new = __find(c, new_block);
  1106. if (new) {
  1107. if (new->hold_count) {
  1108. __wait_for_free_buffer(c);
  1109. goto retry;
  1110. }
  1111. /*
  1112. * FIXME: Is there any point waiting for a write that's going
  1113. * to be overwritten in a bit?
  1114. */
  1115. __make_buffer_clean(new);
  1116. __unlink_buffer(new);
  1117. __free_buffer_wake(new);
  1118. }
  1119. BUG_ON(!b->hold_count);
  1120. BUG_ON(test_bit(B_READING, &b->state));
  1121. __write_dirty_buffer(b, NULL);
  1122. if (b->hold_count == 1) {
  1123. wait_on_bit_io(&b->state, B_WRITING,
  1124. TASK_UNINTERRUPTIBLE);
  1125. set_bit(B_DIRTY, &b->state);
  1126. b->dirty_start = 0;
  1127. b->dirty_end = c->block_size;
  1128. __unlink_buffer(b);
  1129. __link_buffer(b, new_block, LIST_DIRTY);
  1130. } else {
  1131. sector_t old_block;
  1132. wait_on_bit_lock_io(&b->state, B_WRITING,
  1133. TASK_UNINTERRUPTIBLE);
  1134. /*
  1135. * Relink buffer to "new_block" so that write_callback
  1136. * sees "new_block" as a block number.
  1137. * After the write, link the buffer back to old_block.
  1138. * All this must be done in bufio lock, so that block number
  1139. * change isn't visible to other threads.
  1140. */
  1141. old_block = b->block;
  1142. __unlink_buffer(b);
  1143. __link_buffer(b, new_block, b->list_mode);
  1144. submit_io(b, REQ_OP_WRITE, write_endio);
  1145. wait_on_bit_io(&b->state, B_WRITING,
  1146. TASK_UNINTERRUPTIBLE);
  1147. __unlink_buffer(b);
  1148. __link_buffer(b, old_block, b->list_mode);
  1149. }
  1150. dm_bufio_unlock(c);
  1151. dm_bufio_release(b);
  1152. }
  1153. EXPORT_SYMBOL_GPL(dm_bufio_release_move);
  1154. /*
  1155. * Free the given buffer.
  1156. *
  1157. * This is just a hint, if the buffer is in use or dirty, this function
  1158. * does nothing.
  1159. */
  1160. void dm_bufio_forget(struct dm_bufio_client *c, sector_t block)
  1161. {
  1162. struct dm_buffer *b;
  1163. dm_bufio_lock(c);
  1164. b = __find(c, block);
  1165. if (b && likely(!b->hold_count) && likely(!b->state)) {
  1166. __unlink_buffer(b);
  1167. __free_buffer_wake(b);
  1168. }
  1169. dm_bufio_unlock(c);
  1170. }
  1171. EXPORT_SYMBOL_GPL(dm_bufio_forget);
  1172. void dm_bufio_set_minimum_buffers(struct dm_bufio_client *c, unsigned n)
  1173. {
  1174. c->minimum_buffers = n;
  1175. }
  1176. EXPORT_SYMBOL_GPL(dm_bufio_set_minimum_buffers);
  1177. unsigned dm_bufio_get_block_size(struct dm_bufio_client *c)
  1178. {
  1179. return c->block_size;
  1180. }
  1181. EXPORT_SYMBOL_GPL(dm_bufio_get_block_size);
  1182. sector_t dm_bufio_get_device_size(struct dm_bufio_client *c)
  1183. {
  1184. sector_t s = i_size_read(c->bdev->bd_inode) >> SECTOR_SHIFT;
  1185. if (s >= c->start)
  1186. s -= c->start;
  1187. else
  1188. s = 0;
  1189. if (likely(c->sectors_per_block_bits >= 0))
  1190. s >>= c->sectors_per_block_bits;
  1191. else
  1192. sector_div(s, c->block_size >> SECTOR_SHIFT);
  1193. return s;
  1194. }
  1195. EXPORT_SYMBOL_GPL(dm_bufio_get_device_size);
  1196. struct dm_io_client *dm_bufio_get_dm_io_client(struct dm_bufio_client *c)
  1197. {
  1198. return c->dm_io;
  1199. }
  1200. EXPORT_SYMBOL_GPL(dm_bufio_get_dm_io_client);
  1201. sector_t dm_bufio_get_block_number(struct dm_buffer *b)
  1202. {
  1203. return b->block;
  1204. }
  1205. EXPORT_SYMBOL_GPL(dm_bufio_get_block_number);
  1206. void *dm_bufio_get_block_data(struct dm_buffer *b)
  1207. {
  1208. return b->data;
  1209. }
  1210. EXPORT_SYMBOL_GPL(dm_bufio_get_block_data);
  1211. void *dm_bufio_get_aux_data(struct dm_buffer *b)
  1212. {
  1213. return b + 1;
  1214. }
  1215. EXPORT_SYMBOL_GPL(dm_bufio_get_aux_data);
  1216. struct dm_bufio_client *dm_bufio_get_client(struct dm_buffer *b)
  1217. {
  1218. return b->c;
  1219. }
  1220. EXPORT_SYMBOL_GPL(dm_bufio_get_client);
  1221. static void drop_buffers(struct dm_bufio_client *c)
  1222. {
  1223. struct dm_buffer *b;
  1224. int i;
  1225. bool warned = false;
  1226. BUG_ON(dm_bufio_in_request());
  1227. /*
  1228. * An optimization so that the buffers are not written one-by-one.
  1229. */
  1230. dm_bufio_write_dirty_buffers_async(c);
  1231. dm_bufio_lock(c);
  1232. while ((b = __get_unclaimed_buffer(c)))
  1233. __free_buffer_wake(b);
  1234. for (i = 0; i < LIST_SIZE; i++)
  1235. list_for_each_entry(b, &c->lru[i], lru_list) {
  1236. WARN_ON(!warned);
  1237. warned = true;
  1238. DMERR("leaked buffer %llx, hold count %u, list %d",
  1239. (unsigned long long)b->block, b->hold_count, i);
  1240. #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
  1241. stack_trace_print(b->stack_entries, b->stack_len, 1);
  1242. /* mark unclaimed to avoid BUG_ON below */
  1243. b->hold_count = 0;
  1244. #endif
  1245. }
  1246. #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
  1247. while ((b = __get_unclaimed_buffer(c)))
  1248. __free_buffer_wake(b);
  1249. #endif
  1250. for (i = 0; i < LIST_SIZE; i++)
  1251. BUG_ON(!list_empty(&c->lru[i]));
  1252. dm_bufio_unlock(c);
  1253. }
  1254. /*
  1255. * We may not be able to evict this buffer if IO pending or the client
  1256. * is still using it. Caller is expected to know buffer is too old.
  1257. *
  1258. * And if GFP_NOFS is used, we must not do any I/O because we hold
  1259. * dm_bufio_clients_lock and we would risk deadlock if the I/O gets
  1260. * rerouted to different bufio client.
  1261. */
  1262. static bool __try_evict_buffer(struct dm_buffer *b, gfp_t gfp)
  1263. {
  1264. if (!(gfp & __GFP_FS)) {
  1265. if (test_bit(B_READING, &b->state) ||
  1266. test_bit(B_WRITING, &b->state) ||
  1267. test_bit(B_DIRTY, &b->state))
  1268. return false;
  1269. }
  1270. if (b->hold_count)
  1271. return false;
  1272. __make_buffer_clean(b);
  1273. __unlink_buffer(b);
  1274. __free_buffer_wake(b);
  1275. return true;
  1276. }
  1277. static unsigned long get_retain_buffers(struct dm_bufio_client *c)
  1278. {
  1279. unsigned long retain_bytes = READ_ONCE(dm_bufio_retain_bytes);
  1280. if (likely(c->sectors_per_block_bits >= 0))
  1281. retain_bytes >>= c->sectors_per_block_bits + SECTOR_SHIFT;
  1282. else
  1283. retain_bytes /= c->block_size;
  1284. return retain_bytes;
  1285. }
  1286. static unsigned long __scan(struct dm_bufio_client *c, unsigned long nr_to_scan,
  1287. gfp_t gfp_mask)
  1288. {
  1289. int l;
  1290. struct dm_buffer *b, *tmp;
  1291. unsigned long freed = 0;
  1292. unsigned long count = c->n_buffers[LIST_CLEAN] +
  1293. c->n_buffers[LIST_DIRTY];
  1294. unsigned long retain_target = get_retain_buffers(c);
  1295. for (l = 0; l < LIST_SIZE; l++) {
  1296. list_for_each_entry_safe_reverse(b, tmp, &c->lru[l], lru_list) {
  1297. if (__try_evict_buffer(b, gfp_mask))
  1298. freed++;
  1299. if (!--nr_to_scan || ((count - freed) <= retain_target))
  1300. return freed;
  1301. cond_resched();
  1302. }
  1303. }
  1304. return freed;
  1305. }
  1306. static unsigned long
  1307. dm_bufio_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
  1308. {
  1309. struct dm_bufio_client *c;
  1310. unsigned long freed;
  1311. c = container_of(shrink, struct dm_bufio_client, shrinker);
  1312. if (sc->gfp_mask & __GFP_FS)
  1313. dm_bufio_lock(c);
  1314. else if (!dm_bufio_trylock(c))
  1315. return SHRINK_STOP;
  1316. freed = __scan(c, sc->nr_to_scan, sc->gfp_mask);
  1317. dm_bufio_unlock(c);
  1318. return freed;
  1319. }
  1320. static unsigned long
  1321. dm_bufio_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
  1322. {
  1323. struct dm_bufio_client *c = container_of(shrink, struct dm_bufio_client, shrinker);
  1324. unsigned long count = READ_ONCE(c->n_buffers[LIST_CLEAN]) +
  1325. READ_ONCE(c->n_buffers[LIST_DIRTY]);
  1326. unsigned long retain_target = get_retain_buffers(c);
  1327. return (count < retain_target) ? 0 : (count - retain_target);
  1328. }
  1329. /*
  1330. * Create the buffering interface
  1331. */
  1332. struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsigned block_size,
  1333. unsigned reserved_buffers, unsigned aux_size,
  1334. void (*alloc_callback)(struct dm_buffer *),
  1335. void (*write_callback)(struct dm_buffer *))
  1336. {
  1337. int r;
  1338. struct dm_bufio_client *c;
  1339. unsigned i;
  1340. char slab_name[27];
  1341. if (!block_size || block_size & ((1 << SECTOR_SHIFT) - 1)) {
  1342. DMERR("%s: block size not specified or is not multiple of 512b", __func__);
  1343. r = -EINVAL;
  1344. goto bad_client;
  1345. }
  1346. c = kzalloc(sizeof(*c), GFP_KERNEL);
  1347. if (!c) {
  1348. r = -ENOMEM;
  1349. goto bad_client;
  1350. }
  1351. c->buffer_tree = RB_ROOT;
  1352. c->bdev = bdev;
  1353. c->block_size = block_size;
  1354. if (is_power_of_2(block_size))
  1355. c->sectors_per_block_bits = __ffs(block_size) - SECTOR_SHIFT;
  1356. else
  1357. c->sectors_per_block_bits = -1;
  1358. c->alloc_callback = alloc_callback;
  1359. c->write_callback = write_callback;
  1360. for (i = 0; i < LIST_SIZE; i++) {
  1361. INIT_LIST_HEAD(&c->lru[i]);
  1362. c->n_buffers[i] = 0;
  1363. }
  1364. mutex_init(&c->lock);
  1365. INIT_LIST_HEAD(&c->reserved_buffers);
  1366. c->need_reserved_buffers = reserved_buffers;
  1367. dm_bufio_set_minimum_buffers(c, DM_BUFIO_MIN_BUFFERS);
  1368. init_waitqueue_head(&c->free_buffer_wait);
  1369. c->async_write_error = 0;
  1370. c->dm_io = dm_io_client_create();
  1371. if (IS_ERR(c->dm_io)) {
  1372. r = PTR_ERR(c->dm_io);
  1373. goto bad_dm_io;
  1374. }
  1375. if (block_size <= KMALLOC_MAX_SIZE &&
  1376. (block_size < PAGE_SIZE || !is_power_of_2(block_size))) {
  1377. unsigned align = min(1U << __ffs(block_size), (unsigned)PAGE_SIZE);
  1378. snprintf(slab_name, sizeof slab_name, "dm_bufio_cache-%u", block_size);
  1379. c->slab_cache = kmem_cache_create(slab_name, block_size, align,
  1380. SLAB_RECLAIM_ACCOUNT, NULL);
  1381. if (!c->slab_cache) {
  1382. r = -ENOMEM;
  1383. goto bad;
  1384. }
  1385. }
  1386. if (aux_size)
  1387. snprintf(slab_name, sizeof slab_name, "dm_bufio_buffer-%u", aux_size);
  1388. else
  1389. snprintf(slab_name, sizeof slab_name, "dm_bufio_buffer");
  1390. c->slab_buffer = kmem_cache_create(slab_name, sizeof(struct dm_buffer) + aux_size,
  1391. 0, SLAB_RECLAIM_ACCOUNT, NULL);
  1392. if (!c->slab_buffer) {
  1393. r = -ENOMEM;
  1394. goto bad;
  1395. }
  1396. while (c->need_reserved_buffers) {
  1397. struct dm_buffer *b = alloc_buffer(c, GFP_KERNEL);
  1398. if (!b) {
  1399. r = -ENOMEM;
  1400. goto bad;
  1401. }
  1402. __free_buffer_wake(b);
  1403. }
  1404. c->shrinker.count_objects = dm_bufio_shrink_count;
  1405. c->shrinker.scan_objects = dm_bufio_shrink_scan;
  1406. c->shrinker.seeks = 1;
  1407. c->shrinker.batch = 0;
  1408. r = register_shrinker(&c->shrinker);
  1409. if (r)
  1410. goto bad;
  1411. mutex_lock(&dm_bufio_clients_lock);
  1412. dm_bufio_client_count++;
  1413. list_add(&c->client_list, &dm_bufio_all_clients);
  1414. __cache_size_refresh();
  1415. mutex_unlock(&dm_bufio_clients_lock);
  1416. return c;
  1417. bad:
  1418. while (!list_empty(&c->reserved_buffers)) {
  1419. struct dm_buffer *b = list_entry(c->reserved_buffers.next,
  1420. struct dm_buffer, lru_list);
  1421. list_del(&b->lru_list);
  1422. free_buffer(b);
  1423. }
  1424. kmem_cache_destroy(c->slab_cache);
  1425. kmem_cache_destroy(c->slab_buffer);
  1426. dm_io_client_destroy(c->dm_io);
  1427. bad_dm_io:
  1428. mutex_destroy(&c->lock);
  1429. kfree(c);
  1430. bad_client:
  1431. return ERR_PTR(r);
  1432. }
  1433. EXPORT_SYMBOL_GPL(dm_bufio_client_create);
  1434. /*
  1435. * Free the buffering interface.
  1436. * It is required that there are no references on any buffers.
  1437. */
  1438. void dm_bufio_client_destroy(struct dm_bufio_client *c)
  1439. {
  1440. unsigned i;
  1441. drop_buffers(c);
  1442. unregister_shrinker(&c->shrinker);
  1443. mutex_lock(&dm_bufio_clients_lock);
  1444. list_del(&c->client_list);
  1445. dm_bufio_client_count--;
  1446. __cache_size_refresh();
  1447. mutex_unlock(&dm_bufio_clients_lock);
  1448. BUG_ON(!RB_EMPTY_ROOT(&c->buffer_tree));
  1449. BUG_ON(c->need_reserved_buffers);
  1450. while (!list_empty(&c->reserved_buffers)) {
  1451. struct dm_buffer *b = list_entry(c->reserved_buffers.next,
  1452. struct dm_buffer, lru_list);
  1453. list_del(&b->lru_list);
  1454. free_buffer(b);
  1455. }
  1456. for (i = 0; i < LIST_SIZE; i++)
  1457. if (c->n_buffers[i])
  1458. DMERR("leaked buffer count %d: %ld", i, c->n_buffers[i]);
  1459. for (i = 0; i < LIST_SIZE; i++)
  1460. BUG_ON(c->n_buffers[i]);
  1461. kmem_cache_destroy(c->slab_cache);
  1462. kmem_cache_destroy(c->slab_buffer);
  1463. dm_io_client_destroy(c->dm_io);
  1464. mutex_destroy(&c->lock);
  1465. kfree(c);
  1466. }
  1467. EXPORT_SYMBOL_GPL(dm_bufio_client_destroy);
  1468. void dm_bufio_set_sector_offset(struct dm_bufio_client *c, sector_t start)
  1469. {
  1470. c->start = start;
  1471. }
  1472. EXPORT_SYMBOL_GPL(dm_bufio_set_sector_offset);
  1473. static unsigned get_max_age_hz(void)
  1474. {
  1475. unsigned max_age = READ_ONCE(dm_bufio_max_age);
  1476. if (max_age > UINT_MAX / HZ)
  1477. max_age = UINT_MAX / HZ;
  1478. return max_age * HZ;
  1479. }
  1480. static bool older_than(struct dm_buffer *b, unsigned long age_hz)
  1481. {
  1482. return time_after_eq(jiffies, b->last_accessed + age_hz);
  1483. }
  1484. static void __evict_old_buffers(struct dm_bufio_client *c, unsigned long age_hz)
  1485. {
  1486. struct dm_buffer *b, *tmp;
  1487. unsigned long retain_target = get_retain_buffers(c);
  1488. unsigned long count;
  1489. LIST_HEAD(write_list);
  1490. dm_bufio_lock(c);
  1491. __check_watermark(c, &write_list);
  1492. if (unlikely(!list_empty(&write_list))) {
  1493. dm_bufio_unlock(c);
  1494. __flush_write_list(&write_list);
  1495. dm_bufio_lock(c);
  1496. }
  1497. count = c->n_buffers[LIST_CLEAN] + c->n_buffers[LIST_DIRTY];
  1498. list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_CLEAN], lru_list) {
  1499. if (count <= retain_target)
  1500. break;
  1501. if (!older_than(b, age_hz))
  1502. break;
  1503. if (__try_evict_buffer(b, 0))
  1504. count--;
  1505. cond_resched();
  1506. }
  1507. dm_bufio_unlock(c);
  1508. }
  1509. static void do_global_cleanup(struct work_struct *w)
  1510. {
  1511. struct dm_bufio_client *locked_client = NULL;
  1512. struct dm_bufio_client *current_client;
  1513. struct dm_buffer *b;
  1514. unsigned spinlock_hold_count;
  1515. unsigned long threshold = dm_bufio_cache_size -
  1516. dm_bufio_cache_size / DM_BUFIO_LOW_WATERMARK_RATIO;
  1517. unsigned long loops = global_num * 2;
  1518. mutex_lock(&dm_bufio_clients_lock);
  1519. while (1) {
  1520. cond_resched();
  1521. spin_lock(&global_spinlock);
  1522. if (unlikely(dm_bufio_current_allocated <= threshold))
  1523. break;
  1524. spinlock_hold_count = 0;
  1525. get_next:
  1526. if (!loops--)
  1527. break;
  1528. if (unlikely(list_empty(&global_queue)))
  1529. break;
  1530. b = list_entry(global_queue.prev, struct dm_buffer, global_list);
  1531. if (b->accessed) {
  1532. b->accessed = 0;
  1533. list_move(&b->global_list, &global_queue);
  1534. if (likely(++spinlock_hold_count < 16))
  1535. goto get_next;
  1536. spin_unlock(&global_spinlock);
  1537. continue;
  1538. }
  1539. current_client = b->c;
  1540. if (unlikely(current_client != locked_client)) {
  1541. if (locked_client)
  1542. dm_bufio_unlock(locked_client);
  1543. if (!dm_bufio_trylock(current_client)) {
  1544. spin_unlock(&global_spinlock);
  1545. dm_bufio_lock(current_client);
  1546. locked_client = current_client;
  1547. continue;
  1548. }
  1549. locked_client = current_client;
  1550. }
  1551. spin_unlock(&global_spinlock);
  1552. if (unlikely(!__try_evict_buffer(b, GFP_KERNEL))) {
  1553. spin_lock(&global_spinlock);
  1554. list_move(&b->global_list, &global_queue);
  1555. spin_unlock(&global_spinlock);
  1556. }
  1557. }
  1558. spin_unlock(&global_spinlock);
  1559. if (locked_client)
  1560. dm_bufio_unlock(locked_client);
  1561. mutex_unlock(&dm_bufio_clients_lock);
  1562. }
  1563. static void cleanup_old_buffers(void)
  1564. {
  1565. unsigned long max_age_hz = get_max_age_hz();
  1566. struct dm_bufio_client *c;
  1567. mutex_lock(&dm_bufio_clients_lock);
  1568. __cache_size_refresh();
  1569. list_for_each_entry(c, &dm_bufio_all_clients, client_list)
  1570. __evict_old_buffers(c, max_age_hz);
  1571. mutex_unlock(&dm_bufio_clients_lock);
  1572. }
  1573. static void work_fn(struct work_struct *w)
  1574. {
  1575. cleanup_old_buffers();
  1576. queue_delayed_work(dm_bufio_wq, &dm_bufio_cleanup_old_work,
  1577. DM_BUFIO_WORK_TIMER_SECS * HZ);
  1578. }
  1579. /*----------------------------------------------------------------
  1580. * Module setup
  1581. *--------------------------------------------------------------*/
  1582. /*
  1583. * This is called only once for the whole dm_bufio module.
  1584. * It initializes memory limit.
  1585. */
  1586. static int __init dm_bufio_init(void)
  1587. {
  1588. __u64 mem;
  1589. dm_bufio_allocated_kmem_cache = 0;
  1590. dm_bufio_allocated_get_free_pages = 0;
  1591. dm_bufio_allocated_vmalloc = 0;
  1592. dm_bufio_current_allocated = 0;
  1593. mem = (__u64)mult_frac(totalram_pages() - totalhigh_pages(),
  1594. DM_BUFIO_MEMORY_PERCENT, 100) << PAGE_SHIFT;
  1595. if (mem > ULONG_MAX)
  1596. mem = ULONG_MAX;
  1597. #ifdef CONFIG_MMU
  1598. if (mem > mult_frac(VMALLOC_TOTAL, DM_BUFIO_VMALLOC_PERCENT, 100))
  1599. mem = mult_frac(VMALLOC_TOTAL, DM_BUFIO_VMALLOC_PERCENT, 100);
  1600. #endif
  1601. dm_bufio_default_cache_size = mem;
  1602. mutex_lock(&dm_bufio_clients_lock);
  1603. __cache_size_refresh();
  1604. mutex_unlock(&dm_bufio_clients_lock);
  1605. dm_bufio_wq = alloc_workqueue("dm_bufio_cache", WQ_MEM_RECLAIM, 0);
  1606. if (!dm_bufio_wq)
  1607. return -ENOMEM;
  1608. INIT_DELAYED_WORK(&dm_bufio_cleanup_old_work, work_fn);
  1609. INIT_WORK(&dm_bufio_replacement_work, do_global_cleanup);
  1610. queue_delayed_work(dm_bufio_wq, &dm_bufio_cleanup_old_work,
  1611. DM_BUFIO_WORK_TIMER_SECS * HZ);
  1612. return 0;
  1613. }
  1614. /*
  1615. * This is called once when unloading the dm_bufio module.
  1616. */
  1617. static void __exit dm_bufio_exit(void)
  1618. {
  1619. int bug = 0;
  1620. cancel_delayed_work_sync(&dm_bufio_cleanup_old_work);
  1621. flush_workqueue(dm_bufio_wq);
  1622. destroy_workqueue(dm_bufio_wq);
  1623. if (dm_bufio_client_count) {
  1624. DMCRIT("%s: dm_bufio_client_count leaked: %d",
  1625. __func__, dm_bufio_client_count);
  1626. bug = 1;
  1627. }
  1628. if (dm_bufio_current_allocated) {
  1629. DMCRIT("%s: dm_bufio_current_allocated leaked: %lu",
  1630. __func__, dm_bufio_current_allocated);
  1631. bug = 1;
  1632. }
  1633. if (dm_bufio_allocated_get_free_pages) {
  1634. DMCRIT("%s: dm_bufio_allocated_get_free_pages leaked: %lu",
  1635. __func__, dm_bufio_allocated_get_free_pages);
  1636. bug = 1;
  1637. }
  1638. if (dm_bufio_allocated_vmalloc) {
  1639. DMCRIT("%s: dm_bufio_vmalloc leaked: %lu",
  1640. __func__, dm_bufio_allocated_vmalloc);
  1641. bug = 1;
  1642. }
  1643. BUG_ON(bug);
  1644. }
  1645. module_init(dm_bufio_init)
  1646. module_exit(dm_bufio_exit)
  1647. module_param_named(max_cache_size_bytes, dm_bufio_cache_size, ulong, S_IRUGO | S_IWUSR);
  1648. MODULE_PARM_DESC(max_cache_size_bytes, "Size of metadata cache");
  1649. module_param_named(max_age_seconds, dm_bufio_max_age, uint, S_IRUGO | S_IWUSR);
  1650. MODULE_PARM_DESC(max_age_seconds, "Max age of a buffer in seconds");
  1651. module_param_named(retain_bytes, dm_bufio_retain_bytes, ulong, S_IRUGO | S_IWUSR);
  1652. MODULE_PARM_DESC(retain_bytes, "Try to keep at least this many bytes cached in memory");
  1653. module_param_named(peak_allocated_bytes, dm_bufio_peak_allocated, ulong, S_IRUGO | S_IWUSR);
  1654. MODULE_PARM_DESC(peak_allocated_bytes, "Tracks the maximum allocated memory");
  1655. module_param_named(allocated_kmem_cache_bytes, dm_bufio_allocated_kmem_cache, ulong, S_IRUGO);
  1656. MODULE_PARM_DESC(allocated_kmem_cache_bytes, "Memory allocated with kmem_cache_alloc");
  1657. module_param_named(allocated_get_free_pages_bytes, dm_bufio_allocated_get_free_pages, ulong, S_IRUGO);
  1658. MODULE_PARM_DESC(allocated_get_free_pages_bytes, "Memory allocated with get_free_pages");
  1659. module_param_named(allocated_vmalloc_bytes, dm_bufio_allocated_vmalloc, ulong, S_IRUGO);
  1660. MODULE_PARM_DESC(allocated_vmalloc_bytes, "Memory allocated with vmalloc");
  1661. module_param_named(current_allocated_bytes, dm_bufio_current_allocated, ulong, S_IRUGO);
  1662. MODULE_PARM_DESC(current_allocated_bytes, "Memory currently used by the cache");
  1663. MODULE_AUTHOR("Mikulas Patocka <dm-devel@redhat.com>");
  1664. MODULE_DESCRIPTION(DM_NAME " buffered I/O library");
  1665. MODULE_LICENSE("GPL");