dm-era-target.c 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. #include "dm.h"
  3. #include "persistent-data/dm-transaction-manager.h"
  4. #include "persistent-data/dm-bitset.h"
  5. #include "persistent-data/dm-space-map.h"
  6. #include <linux/dm-io.h>
  7. #include <linux/dm-kcopyd.h>
  8. #include <linux/init.h>
  9. #include <linux/mempool.h>
  10. #include <linux/module.h>
  11. #include <linux/slab.h>
  12. #include <linux/vmalloc.h>
  13. #define DM_MSG_PREFIX "era"
  14. #define SUPERBLOCK_LOCATION 0
  15. #define SUPERBLOCK_MAGIC 2126579579
  16. #define SUPERBLOCK_CSUM_XOR 146538381
  17. #define MIN_ERA_VERSION 1
  18. #define MAX_ERA_VERSION 1
  19. #define INVALID_WRITESET_ROOT SUPERBLOCK_LOCATION
  20. #define MIN_BLOCK_SIZE 8
  21. /*----------------------------------------------------------------
  22. * Writeset
  23. *--------------------------------------------------------------*/
  24. struct writeset_metadata {
  25. uint32_t nr_bits;
  26. dm_block_t root;
  27. };
  28. struct writeset {
  29. struct writeset_metadata md;
  30. /*
  31. * An in core copy of the bits to save constantly doing look ups on
  32. * disk.
  33. */
  34. unsigned long *bits;
  35. };
  36. /*
  37. * This does not free off the on disk bitset as this will normally be done
  38. * after digesting into the era array.
  39. */
  40. static void writeset_free(struct writeset *ws)
  41. {
  42. vfree(ws->bits);
  43. ws->bits = NULL;
  44. }
  45. static int setup_on_disk_bitset(struct dm_disk_bitset *info,
  46. unsigned nr_bits, dm_block_t *root)
  47. {
  48. int r;
  49. r = dm_bitset_empty(info, root);
  50. if (r)
  51. return r;
  52. return dm_bitset_resize(info, *root, 0, nr_bits, false, root);
  53. }
  54. static size_t bitset_size(unsigned nr_bits)
  55. {
  56. return sizeof(unsigned long) * dm_div_up(nr_bits, BITS_PER_LONG);
  57. }
  58. /*
  59. * Allocates memory for the in core bitset.
  60. */
  61. static int writeset_alloc(struct writeset *ws, dm_block_t nr_blocks)
  62. {
  63. ws->bits = vzalloc(bitset_size(nr_blocks));
  64. if (!ws->bits) {
  65. DMERR("%s: couldn't allocate in memory bitset", __func__);
  66. return -ENOMEM;
  67. }
  68. return 0;
  69. }
  70. /*
  71. * Wipes the in-core bitset, and creates a new on disk bitset.
  72. */
  73. static int writeset_init(struct dm_disk_bitset *info, struct writeset *ws,
  74. dm_block_t nr_blocks)
  75. {
  76. int r;
  77. memset(ws->bits, 0, bitset_size(nr_blocks));
  78. ws->md.nr_bits = nr_blocks;
  79. r = setup_on_disk_bitset(info, ws->md.nr_bits, &ws->md.root);
  80. if (r) {
  81. DMERR("%s: setup_on_disk_bitset failed", __func__);
  82. return r;
  83. }
  84. return 0;
  85. }
  86. static bool writeset_marked(struct writeset *ws, dm_block_t block)
  87. {
  88. return test_bit(block, ws->bits);
  89. }
  90. static int writeset_marked_on_disk(struct dm_disk_bitset *info,
  91. struct writeset_metadata *m, dm_block_t block,
  92. bool *result)
  93. {
  94. dm_block_t old = m->root;
  95. /*
  96. * The bitset was flushed when it was archived, so we know there'll
  97. * be no change to the root.
  98. */
  99. int r = dm_bitset_test_bit(info, m->root, block, &m->root, result);
  100. if (r) {
  101. DMERR("%s: dm_bitset_test_bit failed", __func__);
  102. return r;
  103. }
  104. BUG_ON(m->root != old);
  105. return r;
  106. }
  107. /*
  108. * Returns < 0 on error, 0 if the bit wasn't previously set, 1 if it was.
  109. */
  110. static int writeset_test_and_set(struct dm_disk_bitset *info,
  111. struct writeset *ws, uint32_t block)
  112. {
  113. int r;
  114. if (!test_bit(block, ws->bits)) {
  115. r = dm_bitset_set_bit(info, ws->md.root, block, &ws->md.root);
  116. if (r) {
  117. /* FIXME: fail mode */
  118. return r;
  119. }
  120. return 0;
  121. }
  122. return 1;
  123. }
  124. /*----------------------------------------------------------------
  125. * On disk metadata layout
  126. *--------------------------------------------------------------*/
  127. #define SPACE_MAP_ROOT_SIZE 128
  128. #define UUID_LEN 16
  129. struct writeset_disk {
  130. __le32 nr_bits;
  131. __le64 root;
  132. } __packed;
  133. struct superblock_disk {
  134. __le32 csum;
  135. __le32 flags;
  136. __le64 blocknr;
  137. __u8 uuid[UUID_LEN];
  138. __le64 magic;
  139. __le32 version;
  140. __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
  141. __le32 data_block_size;
  142. __le32 metadata_block_size;
  143. __le32 nr_blocks;
  144. __le32 current_era;
  145. struct writeset_disk current_writeset;
  146. /*
  147. * Only these two fields are valid within the metadata snapshot.
  148. */
  149. __le64 writeset_tree_root;
  150. __le64 era_array_root;
  151. __le64 metadata_snap;
  152. } __packed;
  153. /*----------------------------------------------------------------
  154. * Superblock validation
  155. *--------------------------------------------------------------*/
  156. static void sb_prepare_for_write(struct dm_block_validator *v,
  157. struct dm_block *b,
  158. size_t sb_block_size)
  159. {
  160. struct superblock_disk *disk = dm_block_data(b);
  161. disk->blocknr = cpu_to_le64(dm_block_location(b));
  162. disk->csum = cpu_to_le32(dm_bm_checksum(&disk->flags,
  163. sb_block_size - sizeof(__le32),
  164. SUPERBLOCK_CSUM_XOR));
  165. }
  166. static int check_metadata_version(struct superblock_disk *disk)
  167. {
  168. uint32_t metadata_version = le32_to_cpu(disk->version);
  169. if (metadata_version < MIN_ERA_VERSION || metadata_version > MAX_ERA_VERSION) {
  170. DMERR("Era metadata version %u found, but only versions between %u and %u supported.",
  171. metadata_version, MIN_ERA_VERSION, MAX_ERA_VERSION);
  172. return -EINVAL;
  173. }
  174. return 0;
  175. }
  176. static int sb_check(struct dm_block_validator *v,
  177. struct dm_block *b,
  178. size_t sb_block_size)
  179. {
  180. struct superblock_disk *disk = dm_block_data(b);
  181. __le32 csum_le;
  182. if (dm_block_location(b) != le64_to_cpu(disk->blocknr)) {
  183. DMERR("sb_check failed: blocknr %llu: wanted %llu",
  184. le64_to_cpu(disk->blocknr),
  185. (unsigned long long)dm_block_location(b));
  186. return -ENOTBLK;
  187. }
  188. if (le64_to_cpu(disk->magic) != SUPERBLOCK_MAGIC) {
  189. DMERR("sb_check failed: magic %llu: wanted %llu",
  190. le64_to_cpu(disk->magic),
  191. (unsigned long long) SUPERBLOCK_MAGIC);
  192. return -EILSEQ;
  193. }
  194. csum_le = cpu_to_le32(dm_bm_checksum(&disk->flags,
  195. sb_block_size - sizeof(__le32),
  196. SUPERBLOCK_CSUM_XOR));
  197. if (csum_le != disk->csum) {
  198. DMERR("sb_check failed: csum %u: wanted %u",
  199. le32_to_cpu(csum_le), le32_to_cpu(disk->csum));
  200. return -EILSEQ;
  201. }
  202. return check_metadata_version(disk);
  203. }
  204. static struct dm_block_validator sb_validator = {
  205. .name = "superblock",
  206. .prepare_for_write = sb_prepare_for_write,
  207. .check = sb_check
  208. };
  209. /*----------------------------------------------------------------
  210. * Low level metadata handling
  211. *--------------------------------------------------------------*/
  212. #define DM_ERA_METADATA_BLOCK_SIZE 4096
  213. #define ERA_MAX_CONCURRENT_LOCKS 5
  214. struct era_metadata {
  215. struct block_device *bdev;
  216. struct dm_block_manager *bm;
  217. struct dm_space_map *sm;
  218. struct dm_transaction_manager *tm;
  219. dm_block_t block_size;
  220. uint32_t nr_blocks;
  221. uint32_t current_era;
  222. /*
  223. * We preallocate 2 writesets. When an era rolls over we
  224. * switch between them. This means the allocation is done at
  225. * preresume time, rather than on the io path.
  226. */
  227. struct writeset writesets[2];
  228. struct writeset *current_writeset;
  229. dm_block_t writeset_tree_root;
  230. dm_block_t era_array_root;
  231. struct dm_disk_bitset bitset_info;
  232. struct dm_btree_info writeset_tree_info;
  233. struct dm_array_info era_array_info;
  234. dm_block_t metadata_snap;
  235. /*
  236. * A flag that is set whenever a writeset has been archived.
  237. */
  238. bool archived_writesets;
  239. /*
  240. * Reading the space map root can fail, so we read it into this
  241. * buffer before the superblock is locked and updated.
  242. */
  243. __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
  244. };
  245. static int superblock_read_lock(struct era_metadata *md,
  246. struct dm_block **sblock)
  247. {
  248. return dm_bm_read_lock(md->bm, SUPERBLOCK_LOCATION,
  249. &sb_validator, sblock);
  250. }
  251. static int superblock_lock_zero(struct era_metadata *md,
  252. struct dm_block **sblock)
  253. {
  254. return dm_bm_write_lock_zero(md->bm, SUPERBLOCK_LOCATION,
  255. &sb_validator, sblock);
  256. }
  257. static int superblock_lock(struct era_metadata *md,
  258. struct dm_block **sblock)
  259. {
  260. return dm_bm_write_lock(md->bm, SUPERBLOCK_LOCATION,
  261. &sb_validator, sblock);
  262. }
  263. /* FIXME: duplication with cache and thin */
  264. static int superblock_all_zeroes(struct dm_block_manager *bm, bool *result)
  265. {
  266. int r;
  267. unsigned i;
  268. struct dm_block *b;
  269. __le64 *data_le, zero = cpu_to_le64(0);
  270. unsigned sb_block_size = dm_bm_block_size(bm) / sizeof(__le64);
  271. /*
  272. * We can't use a validator here - it may be all zeroes.
  273. */
  274. r = dm_bm_read_lock(bm, SUPERBLOCK_LOCATION, NULL, &b);
  275. if (r)
  276. return r;
  277. data_le = dm_block_data(b);
  278. *result = true;
  279. for (i = 0; i < sb_block_size; i++) {
  280. if (data_le[i] != zero) {
  281. *result = false;
  282. break;
  283. }
  284. }
  285. dm_bm_unlock(b);
  286. return 0;
  287. }
  288. /*----------------------------------------------------------------*/
  289. static void ws_pack(const struct writeset_metadata *core, struct writeset_disk *disk)
  290. {
  291. disk->nr_bits = cpu_to_le32(core->nr_bits);
  292. disk->root = cpu_to_le64(core->root);
  293. }
  294. static void ws_unpack(const struct writeset_disk *disk, struct writeset_metadata *core)
  295. {
  296. core->nr_bits = le32_to_cpu(disk->nr_bits);
  297. core->root = le64_to_cpu(disk->root);
  298. }
  299. static void ws_inc(void *context, const void *value)
  300. {
  301. struct era_metadata *md = context;
  302. struct writeset_disk ws_d;
  303. dm_block_t b;
  304. memcpy(&ws_d, value, sizeof(ws_d));
  305. b = le64_to_cpu(ws_d.root);
  306. dm_tm_inc(md->tm, b);
  307. }
  308. static void ws_dec(void *context, const void *value)
  309. {
  310. struct era_metadata *md = context;
  311. struct writeset_disk ws_d;
  312. dm_block_t b;
  313. memcpy(&ws_d, value, sizeof(ws_d));
  314. b = le64_to_cpu(ws_d.root);
  315. dm_bitset_del(&md->bitset_info, b);
  316. }
  317. static int ws_eq(void *context, const void *value1, const void *value2)
  318. {
  319. return !memcmp(value1, value2, sizeof(struct writeset_disk));
  320. }
  321. /*----------------------------------------------------------------*/
  322. static void setup_writeset_tree_info(struct era_metadata *md)
  323. {
  324. struct dm_btree_value_type *vt = &md->writeset_tree_info.value_type;
  325. md->writeset_tree_info.tm = md->tm;
  326. md->writeset_tree_info.levels = 1;
  327. vt->context = md;
  328. vt->size = sizeof(struct writeset_disk);
  329. vt->inc = ws_inc;
  330. vt->dec = ws_dec;
  331. vt->equal = ws_eq;
  332. }
  333. static void setup_era_array_info(struct era_metadata *md)
  334. {
  335. struct dm_btree_value_type vt;
  336. vt.context = NULL;
  337. vt.size = sizeof(__le32);
  338. vt.inc = NULL;
  339. vt.dec = NULL;
  340. vt.equal = NULL;
  341. dm_array_info_init(&md->era_array_info, md->tm, &vt);
  342. }
  343. static void setup_infos(struct era_metadata *md)
  344. {
  345. dm_disk_bitset_init(md->tm, &md->bitset_info);
  346. setup_writeset_tree_info(md);
  347. setup_era_array_info(md);
  348. }
  349. /*----------------------------------------------------------------*/
  350. static int create_fresh_metadata(struct era_metadata *md)
  351. {
  352. int r;
  353. r = dm_tm_create_with_sm(md->bm, SUPERBLOCK_LOCATION,
  354. &md->tm, &md->sm);
  355. if (r < 0) {
  356. DMERR("dm_tm_create_with_sm failed");
  357. return r;
  358. }
  359. setup_infos(md);
  360. r = dm_btree_empty(&md->writeset_tree_info, &md->writeset_tree_root);
  361. if (r) {
  362. DMERR("couldn't create new writeset tree");
  363. goto bad;
  364. }
  365. r = dm_array_empty(&md->era_array_info, &md->era_array_root);
  366. if (r) {
  367. DMERR("couldn't create era array");
  368. goto bad;
  369. }
  370. return 0;
  371. bad:
  372. dm_sm_destroy(md->sm);
  373. dm_tm_destroy(md->tm);
  374. return r;
  375. }
  376. static int save_sm_root(struct era_metadata *md)
  377. {
  378. int r;
  379. size_t metadata_len;
  380. r = dm_sm_root_size(md->sm, &metadata_len);
  381. if (r < 0)
  382. return r;
  383. return dm_sm_copy_root(md->sm, &md->metadata_space_map_root,
  384. metadata_len);
  385. }
  386. static void copy_sm_root(struct era_metadata *md, struct superblock_disk *disk)
  387. {
  388. memcpy(&disk->metadata_space_map_root,
  389. &md->metadata_space_map_root,
  390. sizeof(md->metadata_space_map_root));
  391. }
  392. /*
  393. * Writes a superblock, including the static fields that don't get updated
  394. * with every commit (possible optimisation here). 'md' should be fully
  395. * constructed when this is called.
  396. */
  397. static void prepare_superblock(struct era_metadata *md, struct superblock_disk *disk)
  398. {
  399. disk->magic = cpu_to_le64(SUPERBLOCK_MAGIC);
  400. disk->flags = cpu_to_le32(0ul);
  401. /* FIXME: can't keep blanking the uuid (uuid is currently unused though) */
  402. memset(disk->uuid, 0, sizeof(disk->uuid));
  403. disk->version = cpu_to_le32(MAX_ERA_VERSION);
  404. copy_sm_root(md, disk);
  405. disk->data_block_size = cpu_to_le32(md->block_size);
  406. disk->metadata_block_size = cpu_to_le32(DM_ERA_METADATA_BLOCK_SIZE >> SECTOR_SHIFT);
  407. disk->nr_blocks = cpu_to_le32(md->nr_blocks);
  408. disk->current_era = cpu_to_le32(md->current_era);
  409. ws_pack(&md->current_writeset->md, &disk->current_writeset);
  410. disk->writeset_tree_root = cpu_to_le64(md->writeset_tree_root);
  411. disk->era_array_root = cpu_to_le64(md->era_array_root);
  412. disk->metadata_snap = cpu_to_le64(md->metadata_snap);
  413. }
  414. static int write_superblock(struct era_metadata *md)
  415. {
  416. int r;
  417. struct dm_block *sblock;
  418. struct superblock_disk *disk;
  419. r = save_sm_root(md);
  420. if (r) {
  421. DMERR("%s: save_sm_root failed", __func__);
  422. return r;
  423. }
  424. r = superblock_lock_zero(md, &sblock);
  425. if (r)
  426. return r;
  427. disk = dm_block_data(sblock);
  428. prepare_superblock(md, disk);
  429. return dm_tm_commit(md->tm, sblock);
  430. }
  431. /*
  432. * Assumes block_size and the infos are set.
  433. */
  434. static int format_metadata(struct era_metadata *md)
  435. {
  436. int r;
  437. r = create_fresh_metadata(md);
  438. if (r)
  439. return r;
  440. r = write_superblock(md);
  441. if (r) {
  442. dm_sm_destroy(md->sm);
  443. dm_tm_destroy(md->tm);
  444. return r;
  445. }
  446. return 0;
  447. }
  448. static int open_metadata(struct era_metadata *md)
  449. {
  450. int r;
  451. struct dm_block *sblock;
  452. struct superblock_disk *disk;
  453. r = superblock_read_lock(md, &sblock);
  454. if (r) {
  455. DMERR("couldn't read_lock superblock");
  456. return r;
  457. }
  458. disk = dm_block_data(sblock);
  459. /* Verify the data block size hasn't changed */
  460. if (le32_to_cpu(disk->data_block_size) != md->block_size) {
  461. DMERR("changing the data block size (from %u to %llu) is not supported",
  462. le32_to_cpu(disk->data_block_size), md->block_size);
  463. r = -EINVAL;
  464. goto bad;
  465. }
  466. r = dm_tm_open_with_sm(md->bm, SUPERBLOCK_LOCATION,
  467. disk->metadata_space_map_root,
  468. sizeof(disk->metadata_space_map_root),
  469. &md->tm, &md->sm);
  470. if (r) {
  471. DMERR("dm_tm_open_with_sm failed");
  472. goto bad;
  473. }
  474. setup_infos(md);
  475. md->nr_blocks = le32_to_cpu(disk->nr_blocks);
  476. md->current_era = le32_to_cpu(disk->current_era);
  477. ws_unpack(&disk->current_writeset, &md->current_writeset->md);
  478. md->writeset_tree_root = le64_to_cpu(disk->writeset_tree_root);
  479. md->era_array_root = le64_to_cpu(disk->era_array_root);
  480. md->metadata_snap = le64_to_cpu(disk->metadata_snap);
  481. md->archived_writesets = true;
  482. dm_bm_unlock(sblock);
  483. return 0;
  484. bad:
  485. dm_bm_unlock(sblock);
  486. return r;
  487. }
  488. static int open_or_format_metadata(struct era_metadata *md,
  489. bool may_format)
  490. {
  491. int r;
  492. bool unformatted = false;
  493. r = superblock_all_zeroes(md->bm, &unformatted);
  494. if (r)
  495. return r;
  496. if (unformatted)
  497. return may_format ? format_metadata(md) : -EPERM;
  498. return open_metadata(md);
  499. }
  500. static int create_persistent_data_objects(struct era_metadata *md,
  501. bool may_format)
  502. {
  503. int r;
  504. md->bm = dm_block_manager_create(md->bdev, DM_ERA_METADATA_BLOCK_SIZE,
  505. ERA_MAX_CONCURRENT_LOCKS);
  506. if (IS_ERR(md->bm)) {
  507. DMERR("could not create block manager");
  508. return PTR_ERR(md->bm);
  509. }
  510. r = open_or_format_metadata(md, may_format);
  511. if (r)
  512. dm_block_manager_destroy(md->bm);
  513. return r;
  514. }
  515. static void destroy_persistent_data_objects(struct era_metadata *md)
  516. {
  517. dm_sm_destroy(md->sm);
  518. dm_tm_destroy(md->tm);
  519. dm_block_manager_destroy(md->bm);
  520. }
  521. /*
  522. * This waits until all era_map threads have picked up the new filter.
  523. */
  524. static void swap_writeset(struct era_metadata *md, struct writeset *new_writeset)
  525. {
  526. rcu_assign_pointer(md->current_writeset, new_writeset);
  527. synchronize_rcu();
  528. }
  529. /*----------------------------------------------------------------
  530. * Writesets get 'digested' into the main era array.
  531. *
  532. * We're using a coroutine here so the worker thread can do the digestion,
  533. * thus avoiding synchronisation of the metadata. Digesting a whole
  534. * writeset in one go would cause too much latency.
  535. *--------------------------------------------------------------*/
  536. struct digest {
  537. uint32_t era;
  538. unsigned nr_bits, current_bit;
  539. struct writeset_metadata writeset;
  540. __le32 value;
  541. struct dm_disk_bitset info;
  542. int (*step)(struct era_metadata *, struct digest *);
  543. };
  544. static int metadata_digest_lookup_writeset(struct era_metadata *md,
  545. struct digest *d);
  546. static int metadata_digest_remove_writeset(struct era_metadata *md,
  547. struct digest *d)
  548. {
  549. int r;
  550. uint64_t key = d->era;
  551. r = dm_btree_remove(&md->writeset_tree_info, md->writeset_tree_root,
  552. &key, &md->writeset_tree_root);
  553. if (r) {
  554. DMERR("%s: dm_btree_remove failed", __func__);
  555. return r;
  556. }
  557. d->step = metadata_digest_lookup_writeset;
  558. return 0;
  559. }
  560. #define INSERTS_PER_STEP 100
  561. static int metadata_digest_transcribe_writeset(struct era_metadata *md,
  562. struct digest *d)
  563. {
  564. int r;
  565. bool marked;
  566. unsigned b, e = min(d->current_bit + INSERTS_PER_STEP, d->nr_bits);
  567. for (b = d->current_bit; b < e; b++) {
  568. r = writeset_marked_on_disk(&d->info, &d->writeset, b, &marked);
  569. if (r) {
  570. DMERR("%s: writeset_marked_on_disk failed", __func__);
  571. return r;
  572. }
  573. if (!marked)
  574. continue;
  575. __dm_bless_for_disk(&d->value);
  576. r = dm_array_set_value(&md->era_array_info, md->era_array_root,
  577. b, &d->value, &md->era_array_root);
  578. if (r) {
  579. DMERR("%s: dm_array_set_value failed", __func__);
  580. return r;
  581. }
  582. }
  583. if (b == d->nr_bits)
  584. d->step = metadata_digest_remove_writeset;
  585. else
  586. d->current_bit = b;
  587. return 0;
  588. }
  589. static int metadata_digest_lookup_writeset(struct era_metadata *md,
  590. struct digest *d)
  591. {
  592. int r;
  593. uint64_t key;
  594. struct writeset_disk disk;
  595. r = dm_btree_find_lowest_key(&md->writeset_tree_info,
  596. md->writeset_tree_root, &key);
  597. if (r < 0)
  598. return r;
  599. d->era = key;
  600. r = dm_btree_lookup(&md->writeset_tree_info,
  601. md->writeset_tree_root, &key, &disk);
  602. if (r) {
  603. if (r == -ENODATA) {
  604. d->step = NULL;
  605. return 0;
  606. }
  607. DMERR("%s: dm_btree_lookup failed", __func__);
  608. return r;
  609. }
  610. ws_unpack(&disk, &d->writeset);
  611. d->value = cpu_to_le32(key);
  612. /*
  613. * We initialise another bitset info to avoid any caching side effects
  614. * with the previous one.
  615. */
  616. dm_disk_bitset_init(md->tm, &d->info);
  617. d->nr_bits = min(d->writeset.nr_bits, md->nr_blocks);
  618. d->current_bit = 0;
  619. d->step = metadata_digest_transcribe_writeset;
  620. return 0;
  621. }
  622. static int metadata_digest_start(struct era_metadata *md, struct digest *d)
  623. {
  624. if (d->step)
  625. return 0;
  626. memset(d, 0, sizeof(*d));
  627. d->step = metadata_digest_lookup_writeset;
  628. return 0;
  629. }
  630. /*----------------------------------------------------------------
  631. * High level metadata interface. Target methods should use these, and not
  632. * the lower level ones.
  633. *--------------------------------------------------------------*/
  634. static struct era_metadata *metadata_open(struct block_device *bdev,
  635. sector_t block_size,
  636. bool may_format)
  637. {
  638. int r;
  639. struct era_metadata *md = kzalloc(sizeof(*md), GFP_KERNEL);
  640. if (!md)
  641. return NULL;
  642. md->bdev = bdev;
  643. md->block_size = block_size;
  644. md->writesets[0].md.root = INVALID_WRITESET_ROOT;
  645. md->writesets[1].md.root = INVALID_WRITESET_ROOT;
  646. md->current_writeset = &md->writesets[0];
  647. r = create_persistent_data_objects(md, may_format);
  648. if (r) {
  649. kfree(md);
  650. return ERR_PTR(r);
  651. }
  652. return md;
  653. }
  654. static void metadata_close(struct era_metadata *md)
  655. {
  656. writeset_free(&md->writesets[0]);
  657. writeset_free(&md->writesets[1]);
  658. destroy_persistent_data_objects(md);
  659. kfree(md);
  660. }
  661. static bool valid_nr_blocks(dm_block_t n)
  662. {
  663. /*
  664. * dm_bitset restricts us to 2^32. test_bit & co. restrict us
  665. * further to 2^31 - 1
  666. */
  667. return n < (1ull << 31);
  668. }
  669. static int metadata_resize(struct era_metadata *md, void *arg)
  670. {
  671. int r;
  672. dm_block_t *new_size = arg;
  673. __le32 value;
  674. if (!valid_nr_blocks(*new_size)) {
  675. DMERR("Invalid number of origin blocks %llu",
  676. (unsigned long long) *new_size);
  677. return -EINVAL;
  678. }
  679. writeset_free(&md->writesets[0]);
  680. writeset_free(&md->writesets[1]);
  681. r = writeset_alloc(&md->writesets[0], *new_size);
  682. if (r) {
  683. DMERR("%s: writeset_alloc failed for writeset 0", __func__);
  684. return r;
  685. }
  686. r = writeset_alloc(&md->writesets[1], *new_size);
  687. if (r) {
  688. DMERR("%s: writeset_alloc failed for writeset 1", __func__);
  689. writeset_free(&md->writesets[0]);
  690. return r;
  691. }
  692. value = cpu_to_le32(0u);
  693. __dm_bless_for_disk(&value);
  694. r = dm_array_resize(&md->era_array_info, md->era_array_root,
  695. md->nr_blocks, *new_size,
  696. &value, &md->era_array_root);
  697. if (r) {
  698. DMERR("%s: dm_array_resize failed", __func__);
  699. writeset_free(&md->writesets[0]);
  700. writeset_free(&md->writesets[1]);
  701. return r;
  702. }
  703. md->nr_blocks = *new_size;
  704. return 0;
  705. }
  706. static int metadata_era_archive(struct era_metadata *md)
  707. {
  708. int r;
  709. uint64_t keys[1];
  710. struct writeset_disk value;
  711. r = dm_bitset_flush(&md->bitset_info, md->current_writeset->md.root,
  712. &md->current_writeset->md.root);
  713. if (r) {
  714. DMERR("%s: dm_bitset_flush failed", __func__);
  715. return r;
  716. }
  717. ws_pack(&md->current_writeset->md, &value);
  718. keys[0] = md->current_era;
  719. __dm_bless_for_disk(&value);
  720. r = dm_btree_insert(&md->writeset_tree_info, md->writeset_tree_root,
  721. keys, &value, &md->writeset_tree_root);
  722. if (r) {
  723. DMERR("%s: couldn't insert writeset into btree", __func__);
  724. /* FIXME: fail mode */
  725. return r;
  726. }
  727. md->current_writeset->md.root = INVALID_WRITESET_ROOT;
  728. md->archived_writesets = true;
  729. return 0;
  730. }
  731. static struct writeset *next_writeset(struct era_metadata *md)
  732. {
  733. return (md->current_writeset == &md->writesets[0]) ?
  734. &md->writesets[1] : &md->writesets[0];
  735. }
  736. static int metadata_new_era(struct era_metadata *md)
  737. {
  738. int r;
  739. struct writeset *new_writeset = next_writeset(md);
  740. r = writeset_init(&md->bitset_info, new_writeset, md->nr_blocks);
  741. if (r) {
  742. DMERR("%s: writeset_init failed", __func__);
  743. return r;
  744. }
  745. swap_writeset(md, new_writeset);
  746. md->current_era++;
  747. return 0;
  748. }
  749. static int metadata_era_rollover(struct era_metadata *md)
  750. {
  751. int r;
  752. if (md->current_writeset->md.root != INVALID_WRITESET_ROOT) {
  753. r = metadata_era_archive(md);
  754. if (r) {
  755. DMERR("%s: metadata_archive_era failed", __func__);
  756. /* FIXME: fail mode? */
  757. return r;
  758. }
  759. }
  760. r = metadata_new_era(md);
  761. if (r) {
  762. DMERR("%s: new era failed", __func__);
  763. /* FIXME: fail mode */
  764. return r;
  765. }
  766. return 0;
  767. }
  768. static bool metadata_current_marked(struct era_metadata *md, dm_block_t block)
  769. {
  770. bool r;
  771. struct writeset *ws;
  772. rcu_read_lock();
  773. ws = rcu_dereference(md->current_writeset);
  774. r = writeset_marked(ws, block);
  775. rcu_read_unlock();
  776. return r;
  777. }
  778. static int metadata_commit(struct era_metadata *md)
  779. {
  780. int r;
  781. struct dm_block *sblock;
  782. if (md->current_writeset->md.root != INVALID_WRITESET_ROOT) {
  783. r = dm_bitset_flush(&md->bitset_info, md->current_writeset->md.root,
  784. &md->current_writeset->md.root);
  785. if (r) {
  786. DMERR("%s: bitset flush failed", __func__);
  787. return r;
  788. }
  789. }
  790. r = dm_tm_pre_commit(md->tm);
  791. if (r) {
  792. DMERR("%s: pre commit failed", __func__);
  793. return r;
  794. }
  795. r = save_sm_root(md);
  796. if (r) {
  797. DMERR("%s: save_sm_root failed", __func__);
  798. return r;
  799. }
  800. r = superblock_lock(md, &sblock);
  801. if (r) {
  802. DMERR("%s: superblock lock failed", __func__);
  803. return r;
  804. }
  805. prepare_superblock(md, dm_block_data(sblock));
  806. return dm_tm_commit(md->tm, sblock);
  807. }
  808. static int metadata_checkpoint(struct era_metadata *md)
  809. {
  810. /*
  811. * For now we just rollover, but later I want to put a check in to
  812. * avoid this if the filter is still pretty fresh.
  813. */
  814. return metadata_era_rollover(md);
  815. }
  816. /*
  817. * Metadata snapshots allow userland to access era data.
  818. */
  819. static int metadata_take_snap(struct era_metadata *md)
  820. {
  821. int r, inc;
  822. struct dm_block *clone;
  823. if (md->metadata_snap != SUPERBLOCK_LOCATION) {
  824. DMERR("%s: metadata snapshot already exists", __func__);
  825. return -EINVAL;
  826. }
  827. r = metadata_era_rollover(md);
  828. if (r) {
  829. DMERR("%s: era rollover failed", __func__);
  830. return r;
  831. }
  832. r = metadata_commit(md);
  833. if (r) {
  834. DMERR("%s: pre commit failed", __func__);
  835. return r;
  836. }
  837. r = dm_sm_inc_block(md->sm, SUPERBLOCK_LOCATION);
  838. if (r) {
  839. DMERR("%s: couldn't increment superblock", __func__);
  840. return r;
  841. }
  842. r = dm_tm_shadow_block(md->tm, SUPERBLOCK_LOCATION,
  843. &sb_validator, &clone, &inc);
  844. if (r) {
  845. DMERR("%s: couldn't shadow superblock", __func__);
  846. dm_sm_dec_block(md->sm, SUPERBLOCK_LOCATION);
  847. return r;
  848. }
  849. BUG_ON(!inc);
  850. r = dm_sm_inc_block(md->sm, md->writeset_tree_root);
  851. if (r) {
  852. DMERR("%s: couldn't inc writeset tree root", __func__);
  853. dm_tm_unlock(md->tm, clone);
  854. return r;
  855. }
  856. r = dm_sm_inc_block(md->sm, md->era_array_root);
  857. if (r) {
  858. DMERR("%s: couldn't inc era tree root", __func__);
  859. dm_sm_dec_block(md->sm, md->writeset_tree_root);
  860. dm_tm_unlock(md->tm, clone);
  861. return r;
  862. }
  863. md->metadata_snap = dm_block_location(clone);
  864. dm_tm_unlock(md->tm, clone);
  865. return 0;
  866. }
  867. static int metadata_drop_snap(struct era_metadata *md)
  868. {
  869. int r;
  870. dm_block_t location;
  871. struct dm_block *clone;
  872. struct superblock_disk *disk;
  873. if (md->metadata_snap == SUPERBLOCK_LOCATION) {
  874. DMERR("%s: no snap to drop", __func__);
  875. return -EINVAL;
  876. }
  877. r = dm_tm_read_lock(md->tm, md->metadata_snap, &sb_validator, &clone);
  878. if (r) {
  879. DMERR("%s: couldn't read lock superblock clone", __func__);
  880. return r;
  881. }
  882. /*
  883. * Whatever happens now we'll commit with no record of the metadata
  884. * snap.
  885. */
  886. md->metadata_snap = SUPERBLOCK_LOCATION;
  887. disk = dm_block_data(clone);
  888. r = dm_btree_del(&md->writeset_tree_info,
  889. le64_to_cpu(disk->writeset_tree_root));
  890. if (r) {
  891. DMERR("%s: error deleting writeset tree clone", __func__);
  892. dm_tm_unlock(md->tm, clone);
  893. return r;
  894. }
  895. r = dm_array_del(&md->era_array_info, le64_to_cpu(disk->era_array_root));
  896. if (r) {
  897. DMERR("%s: error deleting era array clone", __func__);
  898. dm_tm_unlock(md->tm, clone);
  899. return r;
  900. }
  901. location = dm_block_location(clone);
  902. dm_tm_unlock(md->tm, clone);
  903. return dm_sm_dec_block(md->sm, location);
  904. }
  905. struct metadata_stats {
  906. dm_block_t used;
  907. dm_block_t total;
  908. dm_block_t snap;
  909. uint32_t era;
  910. };
  911. static int metadata_get_stats(struct era_metadata *md, void *ptr)
  912. {
  913. int r;
  914. struct metadata_stats *s = ptr;
  915. dm_block_t nr_free, nr_total;
  916. r = dm_sm_get_nr_free(md->sm, &nr_free);
  917. if (r) {
  918. DMERR("dm_sm_get_nr_free returned %d", r);
  919. return r;
  920. }
  921. r = dm_sm_get_nr_blocks(md->sm, &nr_total);
  922. if (r) {
  923. DMERR("dm_pool_get_metadata_dev_size returned %d", r);
  924. return r;
  925. }
  926. s->used = nr_total - nr_free;
  927. s->total = nr_total;
  928. s->snap = md->metadata_snap;
  929. s->era = md->current_era;
  930. return 0;
  931. }
  932. /*----------------------------------------------------------------*/
  933. struct era {
  934. struct dm_target *ti;
  935. struct dm_target_callbacks callbacks;
  936. struct dm_dev *metadata_dev;
  937. struct dm_dev *origin_dev;
  938. dm_block_t nr_blocks;
  939. uint32_t sectors_per_block;
  940. int sectors_per_block_shift;
  941. struct era_metadata *md;
  942. struct workqueue_struct *wq;
  943. struct work_struct worker;
  944. spinlock_t deferred_lock;
  945. struct bio_list deferred_bios;
  946. spinlock_t rpc_lock;
  947. struct list_head rpc_calls;
  948. struct digest digest;
  949. atomic_t suspended;
  950. };
  951. struct rpc {
  952. struct list_head list;
  953. int (*fn0)(struct era_metadata *);
  954. int (*fn1)(struct era_metadata *, void *);
  955. void *arg;
  956. int result;
  957. struct completion complete;
  958. };
  959. /*----------------------------------------------------------------
  960. * Remapping.
  961. *---------------------------------------------------------------*/
  962. static bool block_size_is_power_of_two(struct era *era)
  963. {
  964. return era->sectors_per_block_shift >= 0;
  965. }
  966. static dm_block_t get_block(struct era *era, struct bio *bio)
  967. {
  968. sector_t block_nr = bio->bi_iter.bi_sector;
  969. if (!block_size_is_power_of_two(era))
  970. (void) sector_div(block_nr, era->sectors_per_block);
  971. else
  972. block_nr >>= era->sectors_per_block_shift;
  973. return block_nr;
  974. }
  975. static void remap_to_origin(struct era *era, struct bio *bio)
  976. {
  977. bio_set_dev(bio, era->origin_dev->bdev);
  978. }
  979. /*----------------------------------------------------------------
  980. * Worker thread
  981. *--------------------------------------------------------------*/
  982. static void wake_worker(struct era *era)
  983. {
  984. if (!atomic_read(&era->suspended))
  985. queue_work(era->wq, &era->worker);
  986. }
  987. static void process_old_eras(struct era *era)
  988. {
  989. int r;
  990. if (!era->digest.step)
  991. return;
  992. r = era->digest.step(era->md, &era->digest);
  993. if (r < 0) {
  994. DMERR("%s: digest step failed, stopping digestion", __func__);
  995. era->digest.step = NULL;
  996. } else if (era->digest.step)
  997. wake_worker(era);
  998. }
  999. static void process_deferred_bios(struct era *era)
  1000. {
  1001. int r;
  1002. struct bio_list deferred_bios, marked_bios;
  1003. struct bio *bio;
  1004. struct blk_plug plug;
  1005. bool commit_needed = false;
  1006. bool failed = false;
  1007. struct writeset *ws = era->md->current_writeset;
  1008. bio_list_init(&deferred_bios);
  1009. bio_list_init(&marked_bios);
  1010. spin_lock(&era->deferred_lock);
  1011. bio_list_merge(&deferred_bios, &era->deferred_bios);
  1012. bio_list_init(&era->deferred_bios);
  1013. spin_unlock(&era->deferred_lock);
  1014. if (bio_list_empty(&deferred_bios))
  1015. return;
  1016. while ((bio = bio_list_pop(&deferred_bios))) {
  1017. r = writeset_test_and_set(&era->md->bitset_info, ws,
  1018. get_block(era, bio));
  1019. if (r < 0) {
  1020. /*
  1021. * This is bad news, we need to rollback.
  1022. * FIXME: finish.
  1023. */
  1024. failed = true;
  1025. } else if (r == 0)
  1026. commit_needed = true;
  1027. bio_list_add(&marked_bios, bio);
  1028. }
  1029. if (commit_needed) {
  1030. r = metadata_commit(era->md);
  1031. if (r)
  1032. failed = true;
  1033. }
  1034. if (failed)
  1035. while ((bio = bio_list_pop(&marked_bios)))
  1036. bio_io_error(bio);
  1037. else {
  1038. blk_start_plug(&plug);
  1039. while ((bio = bio_list_pop(&marked_bios))) {
  1040. /*
  1041. * Only update the in-core writeset if the on-disk one
  1042. * was updated too.
  1043. */
  1044. if (commit_needed)
  1045. set_bit(get_block(era, bio), ws->bits);
  1046. generic_make_request(bio);
  1047. }
  1048. blk_finish_plug(&plug);
  1049. }
  1050. }
  1051. static void process_rpc_calls(struct era *era)
  1052. {
  1053. int r;
  1054. bool need_commit = false;
  1055. struct list_head calls;
  1056. struct rpc *rpc, *tmp;
  1057. INIT_LIST_HEAD(&calls);
  1058. spin_lock(&era->rpc_lock);
  1059. list_splice_init(&era->rpc_calls, &calls);
  1060. spin_unlock(&era->rpc_lock);
  1061. list_for_each_entry_safe(rpc, tmp, &calls, list) {
  1062. rpc->result = rpc->fn0 ? rpc->fn0(era->md) : rpc->fn1(era->md, rpc->arg);
  1063. need_commit = true;
  1064. }
  1065. if (need_commit) {
  1066. r = metadata_commit(era->md);
  1067. if (r)
  1068. list_for_each_entry_safe(rpc, tmp, &calls, list)
  1069. rpc->result = r;
  1070. }
  1071. list_for_each_entry_safe(rpc, tmp, &calls, list)
  1072. complete(&rpc->complete);
  1073. }
  1074. static void kick_off_digest(struct era *era)
  1075. {
  1076. if (era->md->archived_writesets) {
  1077. era->md->archived_writesets = false;
  1078. metadata_digest_start(era->md, &era->digest);
  1079. }
  1080. }
  1081. static void do_work(struct work_struct *ws)
  1082. {
  1083. struct era *era = container_of(ws, struct era, worker);
  1084. kick_off_digest(era);
  1085. process_old_eras(era);
  1086. process_deferred_bios(era);
  1087. process_rpc_calls(era);
  1088. }
  1089. static void defer_bio(struct era *era, struct bio *bio)
  1090. {
  1091. spin_lock(&era->deferred_lock);
  1092. bio_list_add(&era->deferred_bios, bio);
  1093. spin_unlock(&era->deferred_lock);
  1094. wake_worker(era);
  1095. }
  1096. /*
  1097. * Make an rpc call to the worker to change the metadata.
  1098. */
  1099. static int perform_rpc(struct era *era, struct rpc *rpc)
  1100. {
  1101. rpc->result = 0;
  1102. init_completion(&rpc->complete);
  1103. spin_lock(&era->rpc_lock);
  1104. list_add(&rpc->list, &era->rpc_calls);
  1105. spin_unlock(&era->rpc_lock);
  1106. wake_worker(era);
  1107. wait_for_completion(&rpc->complete);
  1108. return rpc->result;
  1109. }
  1110. static int in_worker0(struct era *era, int (*fn)(struct era_metadata *))
  1111. {
  1112. struct rpc rpc;
  1113. rpc.fn0 = fn;
  1114. rpc.fn1 = NULL;
  1115. return perform_rpc(era, &rpc);
  1116. }
  1117. static int in_worker1(struct era *era,
  1118. int (*fn)(struct era_metadata *, void *), void *arg)
  1119. {
  1120. struct rpc rpc;
  1121. rpc.fn0 = NULL;
  1122. rpc.fn1 = fn;
  1123. rpc.arg = arg;
  1124. return perform_rpc(era, &rpc);
  1125. }
  1126. static void start_worker(struct era *era)
  1127. {
  1128. atomic_set(&era->suspended, 0);
  1129. }
  1130. static void stop_worker(struct era *era)
  1131. {
  1132. atomic_set(&era->suspended, 1);
  1133. flush_workqueue(era->wq);
  1134. }
  1135. /*----------------------------------------------------------------
  1136. * Target methods
  1137. *--------------------------------------------------------------*/
  1138. static int dev_is_congested(struct dm_dev *dev, int bdi_bits)
  1139. {
  1140. struct request_queue *q = bdev_get_queue(dev->bdev);
  1141. return bdi_congested(q->backing_dev_info, bdi_bits);
  1142. }
  1143. static int era_is_congested(struct dm_target_callbacks *cb, int bdi_bits)
  1144. {
  1145. struct era *era = container_of(cb, struct era, callbacks);
  1146. return dev_is_congested(era->origin_dev, bdi_bits);
  1147. }
  1148. static void era_destroy(struct era *era)
  1149. {
  1150. if (era->md)
  1151. metadata_close(era->md);
  1152. if (era->wq)
  1153. destroy_workqueue(era->wq);
  1154. if (era->origin_dev)
  1155. dm_put_device(era->ti, era->origin_dev);
  1156. if (era->metadata_dev)
  1157. dm_put_device(era->ti, era->metadata_dev);
  1158. kfree(era);
  1159. }
  1160. static dm_block_t calc_nr_blocks(struct era *era)
  1161. {
  1162. return dm_sector_div_up(era->ti->len, era->sectors_per_block);
  1163. }
  1164. static bool valid_block_size(dm_block_t block_size)
  1165. {
  1166. bool greater_than_zero = block_size > 0;
  1167. bool multiple_of_min_block_size = (block_size & (MIN_BLOCK_SIZE - 1)) == 0;
  1168. return greater_than_zero && multiple_of_min_block_size;
  1169. }
  1170. /*
  1171. * <metadata dev> <data dev> <data block size (sectors)>
  1172. */
  1173. static int era_ctr(struct dm_target *ti, unsigned argc, char **argv)
  1174. {
  1175. int r;
  1176. char dummy;
  1177. struct era *era;
  1178. struct era_metadata *md;
  1179. if (argc != 3) {
  1180. ti->error = "Invalid argument count";
  1181. return -EINVAL;
  1182. }
  1183. era = kzalloc(sizeof(*era), GFP_KERNEL);
  1184. if (!era) {
  1185. ti->error = "Error allocating era structure";
  1186. return -ENOMEM;
  1187. }
  1188. era->ti = ti;
  1189. r = dm_get_device(ti, argv[0], FMODE_READ | FMODE_WRITE, &era->metadata_dev);
  1190. if (r) {
  1191. ti->error = "Error opening metadata device";
  1192. era_destroy(era);
  1193. return -EINVAL;
  1194. }
  1195. r = dm_get_device(ti, argv[1], FMODE_READ | FMODE_WRITE, &era->origin_dev);
  1196. if (r) {
  1197. ti->error = "Error opening data device";
  1198. era_destroy(era);
  1199. return -EINVAL;
  1200. }
  1201. r = sscanf(argv[2], "%u%c", &era->sectors_per_block, &dummy);
  1202. if (r != 1) {
  1203. ti->error = "Error parsing block size";
  1204. era_destroy(era);
  1205. return -EINVAL;
  1206. }
  1207. r = dm_set_target_max_io_len(ti, era->sectors_per_block);
  1208. if (r) {
  1209. ti->error = "could not set max io len";
  1210. era_destroy(era);
  1211. return -EINVAL;
  1212. }
  1213. if (!valid_block_size(era->sectors_per_block)) {
  1214. ti->error = "Invalid block size";
  1215. era_destroy(era);
  1216. return -EINVAL;
  1217. }
  1218. if (era->sectors_per_block & (era->sectors_per_block - 1))
  1219. era->sectors_per_block_shift = -1;
  1220. else
  1221. era->sectors_per_block_shift = __ffs(era->sectors_per_block);
  1222. md = metadata_open(era->metadata_dev->bdev, era->sectors_per_block, true);
  1223. if (IS_ERR(md)) {
  1224. ti->error = "Error reading metadata";
  1225. era_destroy(era);
  1226. return PTR_ERR(md);
  1227. }
  1228. era->md = md;
  1229. era->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
  1230. if (!era->wq) {
  1231. ti->error = "could not create workqueue for metadata object";
  1232. era_destroy(era);
  1233. return -ENOMEM;
  1234. }
  1235. INIT_WORK(&era->worker, do_work);
  1236. spin_lock_init(&era->deferred_lock);
  1237. bio_list_init(&era->deferred_bios);
  1238. spin_lock_init(&era->rpc_lock);
  1239. INIT_LIST_HEAD(&era->rpc_calls);
  1240. ti->private = era;
  1241. ti->num_flush_bios = 1;
  1242. ti->flush_supported = true;
  1243. ti->num_discard_bios = 1;
  1244. era->callbacks.congested_fn = era_is_congested;
  1245. dm_table_add_target_callbacks(ti->table, &era->callbacks);
  1246. return 0;
  1247. }
  1248. static void era_dtr(struct dm_target *ti)
  1249. {
  1250. era_destroy(ti->private);
  1251. }
  1252. static int era_map(struct dm_target *ti, struct bio *bio)
  1253. {
  1254. struct era *era = ti->private;
  1255. dm_block_t block = get_block(era, bio);
  1256. /*
  1257. * All bios get remapped to the origin device. We do this now, but
  1258. * it may not get issued until later. Depending on whether the
  1259. * block is marked in this era.
  1260. */
  1261. remap_to_origin(era, bio);
  1262. /*
  1263. * REQ_PREFLUSH bios carry no data, so we're not interested in them.
  1264. */
  1265. if (!(bio->bi_opf & REQ_PREFLUSH) &&
  1266. (bio_data_dir(bio) == WRITE) &&
  1267. !metadata_current_marked(era->md, block)) {
  1268. defer_bio(era, bio);
  1269. return DM_MAPIO_SUBMITTED;
  1270. }
  1271. return DM_MAPIO_REMAPPED;
  1272. }
  1273. static void era_postsuspend(struct dm_target *ti)
  1274. {
  1275. int r;
  1276. struct era *era = ti->private;
  1277. r = in_worker0(era, metadata_era_archive);
  1278. if (r) {
  1279. DMERR("%s: couldn't archive current era", __func__);
  1280. /* FIXME: fail mode */
  1281. }
  1282. stop_worker(era);
  1283. }
  1284. static int era_preresume(struct dm_target *ti)
  1285. {
  1286. int r;
  1287. struct era *era = ti->private;
  1288. dm_block_t new_size = calc_nr_blocks(era);
  1289. if (era->nr_blocks != new_size) {
  1290. r = metadata_resize(era->md, &new_size);
  1291. if (r) {
  1292. DMERR("%s: metadata_resize failed", __func__);
  1293. return r;
  1294. }
  1295. r = metadata_commit(era->md);
  1296. if (r) {
  1297. DMERR("%s: metadata_commit failed", __func__);
  1298. return r;
  1299. }
  1300. era->nr_blocks = new_size;
  1301. }
  1302. start_worker(era);
  1303. r = in_worker0(era, metadata_era_rollover);
  1304. if (r) {
  1305. DMERR("%s: metadata_era_rollover failed", __func__);
  1306. return r;
  1307. }
  1308. return 0;
  1309. }
  1310. /*
  1311. * Status format:
  1312. *
  1313. * <metadata block size> <#used metadata blocks>/<#total metadata blocks>
  1314. * <current era> <held metadata root | '-'>
  1315. */
  1316. static void era_status(struct dm_target *ti, status_type_t type,
  1317. unsigned status_flags, char *result, unsigned maxlen)
  1318. {
  1319. int r;
  1320. struct era *era = ti->private;
  1321. ssize_t sz = 0;
  1322. struct metadata_stats stats;
  1323. char buf[BDEVNAME_SIZE];
  1324. switch (type) {
  1325. case STATUSTYPE_INFO:
  1326. r = in_worker1(era, metadata_get_stats, &stats);
  1327. if (r)
  1328. goto err;
  1329. DMEMIT("%u %llu/%llu %u",
  1330. (unsigned) (DM_ERA_METADATA_BLOCK_SIZE >> SECTOR_SHIFT),
  1331. (unsigned long long) stats.used,
  1332. (unsigned long long) stats.total,
  1333. (unsigned) stats.era);
  1334. if (stats.snap != SUPERBLOCK_LOCATION)
  1335. DMEMIT(" %llu", stats.snap);
  1336. else
  1337. DMEMIT(" -");
  1338. break;
  1339. case STATUSTYPE_TABLE:
  1340. format_dev_t(buf, era->metadata_dev->bdev->bd_dev);
  1341. DMEMIT("%s ", buf);
  1342. format_dev_t(buf, era->origin_dev->bdev->bd_dev);
  1343. DMEMIT("%s %u", buf, era->sectors_per_block);
  1344. break;
  1345. }
  1346. return;
  1347. err:
  1348. DMEMIT("Error");
  1349. }
  1350. static int era_message(struct dm_target *ti, unsigned argc, char **argv,
  1351. char *result, unsigned maxlen)
  1352. {
  1353. struct era *era = ti->private;
  1354. if (argc != 1) {
  1355. DMERR("incorrect number of message arguments");
  1356. return -EINVAL;
  1357. }
  1358. if (!strcasecmp(argv[0], "checkpoint"))
  1359. return in_worker0(era, metadata_checkpoint);
  1360. if (!strcasecmp(argv[0], "take_metadata_snap"))
  1361. return in_worker0(era, metadata_take_snap);
  1362. if (!strcasecmp(argv[0], "drop_metadata_snap"))
  1363. return in_worker0(era, metadata_drop_snap);
  1364. DMERR("unsupported message '%s'", argv[0]);
  1365. return -EINVAL;
  1366. }
  1367. static sector_t get_dev_size(struct dm_dev *dev)
  1368. {
  1369. return i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT;
  1370. }
  1371. static int era_iterate_devices(struct dm_target *ti,
  1372. iterate_devices_callout_fn fn, void *data)
  1373. {
  1374. struct era *era = ti->private;
  1375. return fn(ti, era->origin_dev, 0, get_dev_size(era->origin_dev), data);
  1376. }
  1377. static void era_io_hints(struct dm_target *ti, struct queue_limits *limits)
  1378. {
  1379. struct era *era = ti->private;
  1380. uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
  1381. /*
  1382. * If the system-determined stacked limits are compatible with the
  1383. * era device's blocksize (io_opt is a factor) do not override them.
  1384. */
  1385. if (io_opt_sectors < era->sectors_per_block ||
  1386. do_div(io_opt_sectors, era->sectors_per_block)) {
  1387. blk_limits_io_min(limits, 0);
  1388. blk_limits_io_opt(limits, era->sectors_per_block << SECTOR_SHIFT);
  1389. }
  1390. }
  1391. /*----------------------------------------------------------------*/
  1392. static struct target_type era_target = {
  1393. .name = "era",
  1394. .version = {1, 0, 0},
  1395. .module = THIS_MODULE,
  1396. .ctr = era_ctr,
  1397. .dtr = era_dtr,
  1398. .map = era_map,
  1399. .postsuspend = era_postsuspend,
  1400. .preresume = era_preresume,
  1401. .status = era_status,
  1402. .message = era_message,
  1403. .iterate_devices = era_iterate_devices,
  1404. .io_hints = era_io_hints
  1405. };
  1406. static int __init dm_era_init(void)
  1407. {
  1408. int r;
  1409. r = dm_register_target(&era_target);
  1410. if (r) {
  1411. DMERR("era target registration failed: %d", r);
  1412. return r;
  1413. }
  1414. return 0;
  1415. }
  1416. static void __exit dm_era_exit(void)
  1417. {
  1418. dm_unregister_target(&era_target);
  1419. }
  1420. module_init(dm_era_init);
  1421. module_exit(dm_era_exit);
  1422. MODULE_DESCRIPTION(DM_NAME " era target");
  1423. MODULE_AUTHOR("Joe Thornber <ejt@redhat.com>");
  1424. MODULE_LICENSE("GPL");