internal.h 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469
  1. /* Internal definitions for FS-Cache
  2. *
  3. * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved.
  4. * Written by David Howells (dhowells@redhat.com)
  5. *
  6. * This program is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU General Public License
  8. * as published by the Free Software Foundation; either version
  9. * 2 of the License, or (at your option) any later version.
  10. */
  11. /*
  12. * Lock order, in the order in which multiple locks should be obtained:
  13. * - fscache_addremove_sem
  14. * - cookie->lock
  15. * - cookie->parent->lock
  16. * - cache->object_list_lock
  17. * - object->lock
  18. * - object->parent->lock
  19. * - cookie->stores_lock
  20. * - fscache_thread_lock
  21. *
  22. */
  23. #ifdef pr_fmt
  24. #undef pr_fmt
  25. #endif
  26. #define pr_fmt(fmt) "FS-Cache: " fmt
  27. #include <linux/fscache-cache.h>
  28. #include <linux/sched.h>
  29. #define FSCACHE_MIN_THREADS 4
  30. #define FSCACHE_MAX_THREADS 32
  31. /*
  32. * cache.c
  33. */
  34. extern struct list_head fscache_cache_list;
  35. extern struct rw_semaphore fscache_addremove_sem;
  36. extern struct fscache_cache *fscache_select_cache_for_object(
  37. struct fscache_cookie *);
  38. /*
  39. * cookie.c
  40. */
  41. extern struct kmem_cache *fscache_cookie_jar;
  42. extern void fscache_cookie_init_once(void *);
  43. extern void __fscache_cookie_put(struct fscache_cookie *);
  44. /*
  45. * fsdef.c
  46. */
  47. extern struct fscache_cookie fscache_fsdef_index;
  48. extern struct fscache_cookie_def fscache_fsdef_netfs_def;
  49. /*
  50. * histogram.c
  51. */
  52. #ifdef CONFIG_FSCACHE_HISTOGRAM
  53. extern atomic_t fscache_obj_instantiate_histogram[HZ];
  54. extern atomic_t fscache_objs_histogram[HZ];
  55. extern atomic_t fscache_ops_histogram[HZ];
  56. extern atomic_t fscache_retrieval_delay_histogram[HZ];
  57. extern atomic_t fscache_retrieval_histogram[HZ];
  58. static inline void fscache_hist(atomic_t histogram[], unsigned long start_jif)
  59. {
  60. unsigned long jif = jiffies - start_jif;
  61. if (jif >= HZ)
  62. jif = HZ - 1;
  63. atomic_inc(&histogram[jif]);
  64. }
  65. extern const struct file_operations fscache_histogram_fops;
  66. #else
  67. #define fscache_hist(hist, start_jif) do {} while (0)
  68. #endif
  69. /*
  70. * main.c
  71. */
  72. extern unsigned fscache_defer_lookup;
  73. extern unsigned fscache_defer_create;
  74. extern unsigned fscache_debug;
  75. extern struct kobject *fscache_root;
  76. extern struct workqueue_struct *fscache_object_wq;
  77. extern struct workqueue_struct *fscache_op_wq;
  78. DECLARE_PER_CPU(wait_queue_head_t, fscache_object_cong_wait);
  79. static inline bool fscache_object_congested(void)
  80. {
  81. return workqueue_congested(WORK_CPU_UNBOUND, fscache_object_wq);
  82. }
  83. extern int fscache_wait_atomic_t(atomic_t *);
  84. /*
  85. * object.c
  86. */
  87. extern void fscache_enqueue_object(struct fscache_object *);
  88. /*
  89. * object-list.c
  90. */
  91. #ifdef CONFIG_FSCACHE_OBJECT_LIST
  92. extern const struct file_operations fscache_objlist_fops;
  93. extern void fscache_objlist_add(struct fscache_object *);
  94. extern void fscache_objlist_remove(struct fscache_object *);
  95. #else
  96. #define fscache_objlist_add(object) do {} while(0)
  97. #define fscache_objlist_remove(object) do {} while(0)
  98. #endif
  99. /*
  100. * operation.c
  101. */
  102. extern int fscache_submit_exclusive_op(struct fscache_object *,
  103. struct fscache_operation *);
  104. extern int fscache_submit_op(struct fscache_object *,
  105. struct fscache_operation *);
  106. extern int fscache_cancel_op(struct fscache_operation *, bool);
  107. extern void fscache_cancel_all_ops(struct fscache_object *);
  108. extern void fscache_abort_object(struct fscache_object *);
  109. extern void fscache_start_operations(struct fscache_object *);
  110. extern void fscache_operation_gc(struct work_struct *);
  111. /*
  112. * page.c
  113. */
  114. extern int fscache_wait_for_deferred_lookup(struct fscache_cookie *);
  115. extern int fscache_wait_for_operation_activation(struct fscache_object *,
  116. struct fscache_operation *,
  117. atomic_t *,
  118. atomic_t *);
  119. extern void fscache_invalidate_writes(struct fscache_cookie *);
  120. /*
  121. * proc.c
  122. */
  123. #ifdef CONFIG_PROC_FS
  124. extern int __init fscache_proc_init(void);
  125. extern void fscache_proc_cleanup(void);
  126. #else
  127. #define fscache_proc_init() (0)
  128. #define fscache_proc_cleanup() do {} while (0)
  129. #endif
  130. /*
  131. * stats.c
  132. */
  133. #ifdef CONFIG_FSCACHE_STATS
  134. extern atomic_t fscache_n_ops_processed[FSCACHE_MAX_THREADS];
  135. extern atomic_t fscache_n_objs_processed[FSCACHE_MAX_THREADS];
  136. extern atomic_t fscache_n_op_pend;
  137. extern atomic_t fscache_n_op_run;
  138. extern atomic_t fscache_n_op_enqueue;
  139. extern atomic_t fscache_n_op_deferred_release;
  140. extern atomic_t fscache_n_op_initialised;
  141. extern atomic_t fscache_n_op_release;
  142. extern atomic_t fscache_n_op_gc;
  143. extern atomic_t fscache_n_op_cancelled;
  144. extern atomic_t fscache_n_op_rejected;
  145. extern atomic_t fscache_n_attr_changed;
  146. extern atomic_t fscache_n_attr_changed_ok;
  147. extern atomic_t fscache_n_attr_changed_nobufs;
  148. extern atomic_t fscache_n_attr_changed_nomem;
  149. extern atomic_t fscache_n_attr_changed_calls;
  150. extern atomic_t fscache_n_allocs;
  151. extern atomic_t fscache_n_allocs_ok;
  152. extern atomic_t fscache_n_allocs_wait;
  153. extern atomic_t fscache_n_allocs_nobufs;
  154. extern atomic_t fscache_n_allocs_intr;
  155. extern atomic_t fscache_n_allocs_object_dead;
  156. extern atomic_t fscache_n_alloc_ops;
  157. extern atomic_t fscache_n_alloc_op_waits;
  158. extern atomic_t fscache_n_retrievals;
  159. extern atomic_t fscache_n_retrievals_ok;
  160. extern atomic_t fscache_n_retrievals_wait;
  161. extern atomic_t fscache_n_retrievals_nodata;
  162. extern atomic_t fscache_n_retrievals_nobufs;
  163. extern atomic_t fscache_n_retrievals_intr;
  164. extern atomic_t fscache_n_retrievals_nomem;
  165. extern atomic_t fscache_n_retrievals_object_dead;
  166. extern atomic_t fscache_n_retrieval_ops;
  167. extern atomic_t fscache_n_retrieval_op_waits;
  168. extern atomic_t fscache_n_stores;
  169. extern atomic_t fscache_n_stores_ok;
  170. extern atomic_t fscache_n_stores_again;
  171. extern atomic_t fscache_n_stores_nobufs;
  172. extern atomic_t fscache_n_stores_oom;
  173. extern atomic_t fscache_n_store_ops;
  174. extern atomic_t fscache_n_store_calls;
  175. extern atomic_t fscache_n_store_pages;
  176. extern atomic_t fscache_n_store_radix_deletes;
  177. extern atomic_t fscache_n_store_pages_over_limit;
  178. extern atomic_t fscache_n_store_vmscan_not_storing;
  179. extern atomic_t fscache_n_store_vmscan_gone;
  180. extern atomic_t fscache_n_store_vmscan_busy;
  181. extern atomic_t fscache_n_store_vmscan_cancelled;
  182. extern atomic_t fscache_n_store_vmscan_wait;
  183. extern atomic_t fscache_n_marks;
  184. extern atomic_t fscache_n_uncaches;
  185. extern atomic_t fscache_n_acquires;
  186. extern atomic_t fscache_n_acquires_null;
  187. extern atomic_t fscache_n_acquires_no_cache;
  188. extern atomic_t fscache_n_acquires_ok;
  189. extern atomic_t fscache_n_acquires_nobufs;
  190. extern atomic_t fscache_n_acquires_oom;
  191. extern atomic_t fscache_n_invalidates;
  192. extern atomic_t fscache_n_invalidates_run;
  193. extern atomic_t fscache_n_updates;
  194. extern atomic_t fscache_n_updates_null;
  195. extern atomic_t fscache_n_updates_run;
  196. extern atomic_t fscache_n_relinquishes;
  197. extern atomic_t fscache_n_relinquishes_null;
  198. extern atomic_t fscache_n_relinquishes_waitcrt;
  199. extern atomic_t fscache_n_relinquishes_retire;
  200. extern atomic_t fscache_n_cookie_index;
  201. extern atomic_t fscache_n_cookie_data;
  202. extern atomic_t fscache_n_cookie_special;
  203. extern atomic_t fscache_n_object_alloc;
  204. extern atomic_t fscache_n_object_no_alloc;
  205. extern atomic_t fscache_n_object_lookups;
  206. extern atomic_t fscache_n_object_lookups_negative;
  207. extern atomic_t fscache_n_object_lookups_positive;
  208. extern atomic_t fscache_n_object_lookups_timed_out;
  209. extern atomic_t fscache_n_object_created;
  210. extern atomic_t fscache_n_object_avail;
  211. extern atomic_t fscache_n_object_dead;
  212. extern atomic_t fscache_n_checkaux_none;
  213. extern atomic_t fscache_n_checkaux_okay;
  214. extern atomic_t fscache_n_checkaux_update;
  215. extern atomic_t fscache_n_checkaux_obsolete;
  216. extern atomic_t fscache_n_cop_alloc_object;
  217. extern atomic_t fscache_n_cop_lookup_object;
  218. extern atomic_t fscache_n_cop_lookup_complete;
  219. extern atomic_t fscache_n_cop_grab_object;
  220. extern atomic_t fscache_n_cop_invalidate_object;
  221. extern atomic_t fscache_n_cop_update_object;
  222. extern atomic_t fscache_n_cop_drop_object;
  223. extern atomic_t fscache_n_cop_put_object;
  224. extern atomic_t fscache_n_cop_sync_cache;
  225. extern atomic_t fscache_n_cop_attr_changed;
  226. extern atomic_t fscache_n_cop_read_or_alloc_page;
  227. extern atomic_t fscache_n_cop_read_or_alloc_pages;
  228. extern atomic_t fscache_n_cop_allocate_page;
  229. extern atomic_t fscache_n_cop_allocate_pages;
  230. extern atomic_t fscache_n_cop_write_page;
  231. extern atomic_t fscache_n_cop_uncache_page;
  232. extern atomic_t fscache_n_cop_dissociate_pages;
  233. extern atomic_t fscache_n_cache_no_space_reject;
  234. extern atomic_t fscache_n_cache_stale_objects;
  235. extern atomic_t fscache_n_cache_retired_objects;
  236. extern atomic_t fscache_n_cache_culled_objects;
  237. static inline void fscache_stat(atomic_t *stat)
  238. {
  239. atomic_inc(stat);
  240. }
  241. static inline void fscache_stat_d(atomic_t *stat)
  242. {
  243. atomic_dec(stat);
  244. }
  245. #define __fscache_stat(stat) (stat)
  246. extern const struct file_operations fscache_stats_fops;
  247. #else
  248. #define __fscache_stat(stat) (NULL)
  249. #define fscache_stat(stat) do {} while (0)
  250. #define fscache_stat_d(stat) do {} while (0)
  251. #endif
  252. /*
  253. * raise an event on an object
  254. * - if the event is not masked for that object, then the object is
  255. * queued for attention by the thread pool.
  256. */
  257. static inline void fscache_raise_event(struct fscache_object *object,
  258. unsigned event)
  259. {
  260. BUG_ON(event >= NR_FSCACHE_OBJECT_EVENTS);
  261. #if 0
  262. printk("*** fscache_raise_event(OBJ%d{%lx},%x)\n",
  263. object->debug_id, object->event_mask, (1 << event));
  264. #endif
  265. if (!test_and_set_bit(event, &object->events) &&
  266. test_bit(event, &object->event_mask))
  267. fscache_enqueue_object(object);
  268. }
  269. /*
  270. * drop a reference to a cookie
  271. */
  272. static inline void fscache_cookie_put(struct fscache_cookie *cookie)
  273. {
  274. BUG_ON(atomic_read(&cookie->usage) <= 0);
  275. if (atomic_dec_and_test(&cookie->usage))
  276. __fscache_cookie_put(cookie);
  277. }
  278. /*
  279. * get an extra reference to a netfs retrieval context
  280. */
  281. static inline
  282. void *fscache_get_context(struct fscache_cookie *cookie, void *context)
  283. {
  284. if (cookie->def->get_context)
  285. cookie->def->get_context(cookie->netfs_data, context);
  286. return context;
  287. }
  288. /*
  289. * release a reference to a netfs retrieval context
  290. */
  291. static inline
  292. void fscache_put_context(struct fscache_cookie *cookie, void *context)
  293. {
  294. if (cookie->def->put_context)
  295. cookie->def->put_context(cookie->netfs_data, context);
  296. }
  297. /*****************************************************************************/
  298. /*
  299. * debug tracing
  300. */
  301. #define dbgprintk(FMT, ...) \
  302. printk(KERN_DEBUG "[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__)
  303. #define kenter(FMT, ...) dbgprintk("==> %s("FMT")", __func__, ##__VA_ARGS__)
  304. #define kleave(FMT, ...) dbgprintk("<== %s()"FMT"", __func__, ##__VA_ARGS__)
  305. #define kdebug(FMT, ...) dbgprintk(FMT, ##__VA_ARGS__)
  306. #define kjournal(FMT, ...) no_printk(FMT, ##__VA_ARGS__)
  307. #ifdef __KDEBUG
  308. #define _enter(FMT, ...) kenter(FMT, ##__VA_ARGS__)
  309. #define _leave(FMT, ...) kleave(FMT, ##__VA_ARGS__)
  310. #define _debug(FMT, ...) kdebug(FMT, ##__VA_ARGS__)
  311. #elif defined(CONFIG_FSCACHE_DEBUG)
  312. #define _enter(FMT, ...) \
  313. do { \
  314. if (__do_kdebug(ENTER)) \
  315. kenter(FMT, ##__VA_ARGS__); \
  316. } while (0)
  317. #define _leave(FMT, ...) \
  318. do { \
  319. if (__do_kdebug(LEAVE)) \
  320. kleave(FMT, ##__VA_ARGS__); \
  321. } while (0)
  322. #define _debug(FMT, ...) \
  323. do { \
  324. if (__do_kdebug(DEBUG)) \
  325. kdebug(FMT, ##__VA_ARGS__); \
  326. } while (0)
  327. #else
  328. #define _enter(FMT, ...) no_printk("==> %s("FMT")", __func__, ##__VA_ARGS__)
  329. #define _leave(FMT, ...) no_printk("<== %s()"FMT"", __func__, ##__VA_ARGS__)
  330. #define _debug(FMT, ...) no_printk(FMT, ##__VA_ARGS__)
  331. #endif
  332. /*
  333. * determine whether a particular optional debugging point should be logged
  334. * - we need to go through three steps to persuade cpp to correctly join the
  335. * shorthand in FSCACHE_DEBUG_LEVEL with its prefix
  336. */
  337. #define ____do_kdebug(LEVEL, POINT) \
  338. unlikely((fscache_debug & \
  339. (FSCACHE_POINT_##POINT << (FSCACHE_DEBUG_ ## LEVEL * 3))))
  340. #define ___do_kdebug(LEVEL, POINT) \
  341. ____do_kdebug(LEVEL, POINT)
  342. #define __do_kdebug(POINT) \
  343. ___do_kdebug(FSCACHE_DEBUG_LEVEL, POINT)
  344. #define FSCACHE_DEBUG_CACHE 0
  345. #define FSCACHE_DEBUG_COOKIE 1
  346. #define FSCACHE_DEBUG_PAGE 2
  347. #define FSCACHE_DEBUG_OPERATION 3
  348. #define FSCACHE_POINT_ENTER 1
  349. #define FSCACHE_POINT_LEAVE 2
  350. #define FSCACHE_POINT_DEBUG 4
  351. #ifndef FSCACHE_DEBUG_LEVEL
  352. #define FSCACHE_DEBUG_LEVEL CACHE
  353. #endif
  354. /*
  355. * assertions
  356. */
  357. #if 1 /* defined(__KDEBUGALL) */
  358. #define ASSERT(X) \
  359. do { \
  360. if (unlikely(!(X))) { \
  361. pr_err("\n"); \
  362. pr_err("Assertion failed\n"); \
  363. BUG(); \
  364. } \
  365. } while (0)
  366. #define ASSERTCMP(X, OP, Y) \
  367. do { \
  368. if (unlikely(!((X) OP (Y)))) { \
  369. pr_err("\n"); \
  370. pr_err("Assertion failed\n"); \
  371. pr_err("%lx " #OP " %lx is false\n", \
  372. (unsigned long)(X), (unsigned long)(Y)); \
  373. BUG(); \
  374. } \
  375. } while (0)
  376. #define ASSERTIF(C, X) \
  377. do { \
  378. if (unlikely((C) && !(X))) { \
  379. pr_err("\n"); \
  380. pr_err("Assertion failed\n"); \
  381. BUG(); \
  382. } \
  383. } while (0)
  384. #define ASSERTIFCMP(C, X, OP, Y) \
  385. do { \
  386. if (unlikely((C) && !((X) OP (Y)))) { \
  387. pr_err("\n"); \
  388. pr_err("Assertion failed\n"); \
  389. pr_err("%lx " #OP " %lx is false\n", \
  390. (unsigned long)(X), (unsigned long)(Y)); \
  391. BUG(); \
  392. } \
  393. } while (0)
  394. #else
  395. #define ASSERT(X) do {} while (0)
  396. #define ASSERTCMP(X, OP, Y) do {} while (0)
  397. #define ASSERTIF(C, X) do {} while (0)
  398. #define ASSERTIFCMP(C, X, OP, Y) do {} while (0)
  399. #endif /* assert or not */