async_tx.c 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295
  1. /*
  2. * core routines for the asynchronous memory transfer/transform api
  3. *
  4. * Copyright © 2006, Intel Corporation.
  5. *
  6. * Dan Williams <dan.j.williams@intel.com>
  7. *
  8. * with architecture considerations by:
  9. * Neil Brown <neilb@suse.de>
  10. * Jeff Garzik <jeff@garzik.org>
  11. *
  12. * This program is free software; you can redistribute it and/or modify it
  13. * under the terms and conditions of the GNU General Public License,
  14. * version 2, as published by the Free Software Foundation.
  15. *
  16. * This program is distributed in the hope it will be useful, but WITHOUT
  17. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  18. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  19. * more details.
  20. *
  21. * You should have received a copy of the GNU General Public License along with
  22. * this program; if not, write to the Free Software Foundation, Inc.,
  23. * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  24. *
  25. */
  26. #include <linux/rculist.h>
  27. #include <linux/module.h>
  28. #include <linux/kernel.h>
  29. #include <linux/async_tx.h>
  30. #ifdef CONFIG_DMA_ENGINE
  31. static int __init async_tx_init(void)
  32. {
  33. async_dmaengine_get();
  34. printk(KERN_INFO "async_tx: api initialized (async)\n");
  35. return 0;
  36. }
  37. static void __exit async_tx_exit(void)
  38. {
  39. async_dmaengine_put();
  40. }
  41. module_init(async_tx_init);
  42. module_exit(async_tx_exit);
  43. /**
  44. * __async_tx_find_channel - find a channel to carry out the operation or let
  45. * the transaction execute synchronously
  46. * @submit: transaction dependency and submission modifiers
  47. * @tx_type: transaction type
  48. */
  49. struct dma_chan *
  50. __async_tx_find_channel(struct async_submit_ctl *submit,
  51. enum dma_transaction_type tx_type)
  52. {
  53. struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;
  54. /* see if we can keep the chain on one channel */
  55. if (depend_tx &&
  56. dma_has_cap(tx_type, depend_tx->chan->device->cap_mask))
  57. return depend_tx->chan;
  58. return async_dma_find_channel(tx_type);
  59. }
  60. EXPORT_SYMBOL_GPL(__async_tx_find_channel);
  61. #endif
  62. /**
  63. * async_tx_channel_switch - queue an interrupt descriptor with a dependency
  64. * pre-attached.
  65. * @depend_tx: the operation that must finish before the new operation runs
  66. * @tx: the new operation
  67. */
  68. static void
  69. async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
  70. struct dma_async_tx_descriptor *tx)
  71. {
  72. struct dma_chan *chan = depend_tx->chan;
  73. struct dma_device *device = chan->device;
  74. struct dma_async_tx_descriptor *intr_tx = (void *) ~0;
  75. /* first check to see if we can still append to depend_tx */
  76. txd_lock(depend_tx);
  77. if (txd_parent(depend_tx) && depend_tx->chan == tx->chan) {
  78. txd_chain(depend_tx, tx);
  79. intr_tx = NULL;
  80. }
  81. txd_unlock(depend_tx);
  82. /* attached dependency, flush the parent channel */
  83. if (!intr_tx) {
  84. device->device_issue_pending(chan);
  85. return;
  86. }
  87. /* see if we can schedule an interrupt
  88. * otherwise poll for completion
  89. */
  90. if (dma_has_cap(DMA_INTERRUPT, device->cap_mask))
  91. intr_tx = device->device_prep_dma_interrupt(chan, 0);
  92. else
  93. intr_tx = NULL;
  94. if (intr_tx) {
  95. intr_tx->callback = NULL;
  96. intr_tx->callback_param = NULL;
  97. /* safe to chain outside the lock since we know we are
  98. * not submitted yet
  99. */
  100. txd_chain(intr_tx, tx);
  101. /* check if we need to append */
  102. txd_lock(depend_tx);
  103. if (txd_parent(depend_tx)) {
  104. txd_chain(depend_tx, intr_tx);
  105. async_tx_ack(intr_tx);
  106. intr_tx = NULL;
  107. }
  108. txd_unlock(depend_tx);
  109. if (intr_tx) {
  110. txd_clear_parent(intr_tx);
  111. intr_tx->tx_submit(intr_tx);
  112. async_tx_ack(intr_tx);
  113. }
  114. device->device_issue_pending(chan);
  115. } else {
  116. if (dma_wait_for_async_tx(depend_tx) != DMA_COMPLETE)
  117. panic("%s: DMA error waiting for depend_tx\n",
  118. __func__);
  119. tx->tx_submit(tx);
  120. }
  121. }
  122. /**
  123. * submit_disposition - flags for routing an incoming operation
  124. * @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock
  125. * @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch
  126. * @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly
  127. *
  128. * while holding depend_tx->lock we must avoid submitting new operations
  129. * to prevent a circular locking dependency with drivers that already
  130. * hold a channel lock when calling async_tx_run_dependencies.
  131. */
  132. enum submit_disposition {
  133. ASYNC_TX_SUBMITTED,
  134. ASYNC_TX_CHANNEL_SWITCH,
  135. ASYNC_TX_DIRECT_SUBMIT,
  136. };
  137. void
  138. async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx,
  139. struct async_submit_ctl *submit)
  140. {
  141. struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;
  142. tx->callback = submit->cb_fn;
  143. tx->callback_param = submit->cb_param;
  144. if (depend_tx) {
  145. enum submit_disposition s;
  146. /* sanity check the dependency chain:
  147. * 1/ if ack is already set then we cannot be sure
  148. * we are referring to the correct operation
  149. * 2/ dependencies are 1:1 i.e. two transactions can
  150. * not depend on the same parent
  151. */
  152. BUG_ON(async_tx_test_ack(depend_tx) || txd_next(depend_tx) ||
  153. txd_parent(tx));
  154. /* the lock prevents async_tx_run_dependencies from missing
  155. * the setting of ->next when ->parent != NULL
  156. */
  157. txd_lock(depend_tx);
  158. if (txd_parent(depend_tx)) {
  159. /* we have a parent so we can not submit directly
  160. * if we are staying on the same channel: append
  161. * else: channel switch
  162. */
  163. if (depend_tx->chan == chan) {
  164. txd_chain(depend_tx, tx);
  165. s = ASYNC_TX_SUBMITTED;
  166. } else
  167. s = ASYNC_TX_CHANNEL_SWITCH;
  168. } else {
  169. /* we do not have a parent so we may be able to submit
  170. * directly if we are staying on the same channel
  171. */
  172. if (depend_tx->chan == chan)
  173. s = ASYNC_TX_DIRECT_SUBMIT;
  174. else
  175. s = ASYNC_TX_CHANNEL_SWITCH;
  176. }
  177. txd_unlock(depend_tx);
  178. switch (s) {
  179. case ASYNC_TX_SUBMITTED:
  180. break;
  181. case ASYNC_TX_CHANNEL_SWITCH:
  182. async_tx_channel_switch(depend_tx, tx);
  183. break;
  184. case ASYNC_TX_DIRECT_SUBMIT:
  185. txd_clear_parent(tx);
  186. tx->tx_submit(tx);
  187. break;
  188. }
  189. } else {
  190. txd_clear_parent(tx);
  191. tx->tx_submit(tx);
  192. }
  193. if (submit->flags & ASYNC_TX_ACK)
  194. async_tx_ack(tx);
  195. if (depend_tx)
  196. async_tx_ack(depend_tx);
  197. }
  198. EXPORT_SYMBOL_GPL(async_tx_submit);
  199. /**
  200. * async_trigger_callback - schedules the callback function to be run
  201. * @submit: submission and completion parameters
  202. *
  203. * honored flags: ASYNC_TX_ACK
  204. *
  205. * The callback is run after any dependent operations have completed.
  206. */
  207. struct dma_async_tx_descriptor *
  208. async_trigger_callback(struct async_submit_ctl *submit)
  209. {
  210. struct dma_chan *chan;
  211. struct dma_device *device;
  212. struct dma_async_tx_descriptor *tx;
  213. struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;
  214. if (depend_tx) {
  215. chan = depend_tx->chan;
  216. device = chan->device;
  217. /* see if we can schedule an interrupt
  218. * otherwise poll for completion
  219. */
  220. if (device && !dma_has_cap(DMA_INTERRUPT, device->cap_mask))
  221. device = NULL;
  222. tx = device ? device->device_prep_dma_interrupt(chan, 0) : NULL;
  223. } else
  224. tx = NULL;
  225. if (tx) {
  226. pr_debug("%s: (async)\n", __func__);
  227. async_tx_submit(chan, tx, submit);
  228. } else {
  229. pr_debug("%s: (sync)\n", __func__);
  230. /* wait for any prerequisite operations */
  231. async_tx_quiesce(&submit->depend_tx);
  232. async_tx_sync_epilog(submit);
  233. }
  234. return tx;
  235. }
  236. EXPORT_SYMBOL_GPL(async_trigger_callback);
  237. /**
  238. * async_tx_quiesce - ensure tx is complete and freeable upon return
  239. * @tx - transaction to quiesce
  240. */
  241. void async_tx_quiesce(struct dma_async_tx_descriptor **tx)
  242. {
  243. if (*tx) {
  244. /* if ack is already set then we cannot be sure
  245. * we are referring to the correct operation
  246. */
  247. BUG_ON(async_tx_test_ack(*tx));
  248. if (dma_wait_for_async_tx(*tx) != DMA_COMPLETE)
  249. panic("%s: DMA error waiting for transaction\n",
  250. __func__);
  251. async_tx_ack(*tx);
  252. *tx = NULL;
  253. }
  254. }
  255. EXPORT_SYMBOL_GPL(async_tx_quiesce);
  256. MODULE_AUTHOR("Intel Corporation");
  257. MODULE_DESCRIPTION("Asynchronous Bulk Memory Transactions API");
  258. MODULE_LICENSE("GPL");