mic_x100_dma.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811
  1. /*
  2. * Intel MIC Platform Software Stack (MPSS)
  3. *
  4. * Copyright(c) 2014 Intel Corporation.
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License, version 2, as
  8. * published by the Free Software Foundation.
  9. *
  10. * This program is distributed in the hope that it will be useful, but
  11. * WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  13. * General Public License for more details.
  14. *
  15. * The full GNU General Public License is included in this distribution in
  16. * the file called "COPYING".
  17. *
  18. * Intel MIC X100 DMA Driver.
  19. *
  20. * Adapted from IOAT dma driver.
  21. */
  22. #include <linux/module.h>
  23. #include <linux/io.h>
  24. #include <linux/seq_file.h>
  25. #include <linux/vmalloc.h>
  26. #include "mic_x100_dma.h"
  27. #define MIC_DMA_MAX_XFER_SIZE_CARD (1 * 1024 * 1024 -\
  28. MIC_DMA_ALIGN_BYTES)
  29. #define MIC_DMA_MAX_XFER_SIZE_HOST (1 * 1024 * 1024 >> 1)
  30. #define MIC_DMA_DESC_TYPE_SHIFT 60
  31. #define MIC_DMA_MEMCPY_LEN_SHIFT 46
  32. #define MIC_DMA_STAT_INTR_SHIFT 59
  33. /* high-water mark for pushing dma descriptors */
  34. static int mic_dma_pending_level = 4;
  35. /* Status descriptor is used to write a 64 bit value to a memory location */
  36. enum mic_dma_desc_format_type {
  37. MIC_DMA_MEMCPY = 1,
  38. MIC_DMA_STATUS,
  39. };
  40. static inline u32 mic_dma_hw_ring_inc(u32 val)
  41. {
  42. return (val + 1) % MIC_DMA_DESC_RX_SIZE;
  43. }
  44. static inline u32 mic_dma_hw_ring_dec(u32 val)
  45. {
  46. return val ? val - 1 : MIC_DMA_DESC_RX_SIZE - 1;
  47. }
  48. static inline void mic_dma_hw_ring_inc_head(struct mic_dma_chan *ch)
  49. {
  50. ch->head = mic_dma_hw_ring_inc(ch->head);
  51. }
  52. /* Prepare a memcpy desc */
  53. static inline void mic_dma_memcpy_desc(struct mic_dma_desc *desc,
  54. dma_addr_t src_phys, dma_addr_t dst_phys, u64 size)
  55. {
  56. u64 qw0, qw1;
  57. qw0 = src_phys;
  58. qw0 |= (size >> MIC_DMA_ALIGN_SHIFT) << MIC_DMA_MEMCPY_LEN_SHIFT;
  59. qw1 = MIC_DMA_MEMCPY;
  60. qw1 <<= MIC_DMA_DESC_TYPE_SHIFT;
  61. qw1 |= dst_phys;
  62. desc->qw0 = qw0;
  63. desc->qw1 = qw1;
  64. }
  65. /* Prepare a status desc. with @data to be written at @dst_phys */
  66. static inline void mic_dma_prep_status_desc(struct mic_dma_desc *desc, u64 data,
  67. dma_addr_t dst_phys, bool generate_intr)
  68. {
  69. u64 qw0, qw1;
  70. qw0 = data;
  71. qw1 = (u64) MIC_DMA_STATUS << MIC_DMA_DESC_TYPE_SHIFT | dst_phys;
  72. if (generate_intr)
  73. qw1 |= (1ULL << MIC_DMA_STAT_INTR_SHIFT);
  74. desc->qw0 = qw0;
  75. desc->qw1 = qw1;
  76. }
  77. static void mic_dma_cleanup(struct mic_dma_chan *ch)
  78. {
  79. struct dma_async_tx_descriptor *tx;
  80. u32 tail;
  81. u32 last_tail;
  82. spin_lock(&ch->cleanup_lock);
  83. tail = mic_dma_read_cmp_cnt(ch);
  84. /*
  85. * This is the barrier pair for smp_wmb() in fn.
  86. * mic_dma_tx_submit_unlock. It's required so that we read the
  87. * updated cookie value from tx->cookie.
  88. */
  89. smp_rmb();
  90. for (last_tail = ch->last_tail; tail != last_tail;) {
  91. tx = &ch->tx_array[last_tail];
  92. if (tx->cookie) {
  93. dma_cookie_complete(tx);
  94. dmaengine_desc_get_callback_invoke(tx, NULL);
  95. tx->callback = NULL;
  96. }
  97. last_tail = mic_dma_hw_ring_inc(last_tail);
  98. }
  99. /* finish all completion callbacks before incrementing tail */
  100. smp_mb();
  101. ch->last_tail = last_tail;
  102. spin_unlock(&ch->cleanup_lock);
  103. }
  104. static u32 mic_dma_ring_count(u32 head, u32 tail)
  105. {
  106. u32 count;
  107. if (head >= tail)
  108. count = (tail - 0) + (MIC_DMA_DESC_RX_SIZE - head);
  109. else
  110. count = tail - head;
  111. return count - 1;
  112. }
  113. /* Returns the num. of free descriptors on success, -ENOMEM on failure */
  114. static int mic_dma_avail_desc_ring_space(struct mic_dma_chan *ch, int required)
  115. {
  116. struct device *dev = mic_dma_ch_to_device(ch);
  117. u32 count;
  118. count = mic_dma_ring_count(ch->head, ch->last_tail);
  119. if (count < required) {
  120. mic_dma_cleanup(ch);
  121. count = mic_dma_ring_count(ch->head, ch->last_tail);
  122. }
  123. if (count < required) {
  124. dev_dbg(dev, "Not enough desc space");
  125. dev_dbg(dev, "%s %d required=%u, avail=%u\n",
  126. __func__, __LINE__, required, count);
  127. return -ENOMEM;
  128. } else {
  129. return count;
  130. }
  131. }
  132. /* Program memcpy descriptors into the descriptor ring and update s/w head ptr*/
  133. static int mic_dma_prog_memcpy_desc(struct mic_dma_chan *ch, dma_addr_t src,
  134. dma_addr_t dst, size_t len)
  135. {
  136. size_t current_transfer_len;
  137. size_t max_xfer_size = to_mic_dma_dev(ch)->max_xfer_size;
  138. /* 3 is added to make sure we have enough space for status desc */
  139. int num_desc = len / max_xfer_size + 3;
  140. int ret;
  141. if (len % max_xfer_size)
  142. num_desc++;
  143. ret = mic_dma_avail_desc_ring_space(ch, num_desc);
  144. if (ret < 0)
  145. return ret;
  146. do {
  147. current_transfer_len = min(len, max_xfer_size);
  148. mic_dma_memcpy_desc(&ch->desc_ring[ch->head],
  149. src, dst, current_transfer_len);
  150. mic_dma_hw_ring_inc_head(ch);
  151. len -= current_transfer_len;
  152. dst = dst + current_transfer_len;
  153. src = src + current_transfer_len;
  154. } while (len > 0);
  155. return 0;
  156. }
  157. /* It's a h/w quirk and h/w needs 2 status descriptors for every status desc */
  158. static void mic_dma_prog_intr(struct mic_dma_chan *ch)
  159. {
  160. mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,
  161. ch->status_dest_micpa, false);
  162. mic_dma_hw_ring_inc_head(ch);
  163. mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,
  164. ch->status_dest_micpa, true);
  165. mic_dma_hw_ring_inc_head(ch);
  166. }
  167. /* Wrapper function to program memcpy descriptors/status descriptors */
  168. static int mic_dma_do_dma(struct mic_dma_chan *ch, int flags, dma_addr_t src,
  169. dma_addr_t dst, size_t len)
  170. {
  171. if (len && -ENOMEM == mic_dma_prog_memcpy_desc(ch, src, dst, len)) {
  172. return -ENOMEM;
  173. } else {
  174. /* 3 is the maximum number of status descriptors */
  175. int ret = mic_dma_avail_desc_ring_space(ch, 3);
  176. if (ret < 0)
  177. return ret;
  178. }
  179. /* Above mic_dma_prog_memcpy_desc() makes sure we have enough space */
  180. if (flags & DMA_PREP_FENCE) {
  181. mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,
  182. ch->status_dest_micpa, false);
  183. mic_dma_hw_ring_inc_head(ch);
  184. }
  185. if (flags & DMA_PREP_INTERRUPT)
  186. mic_dma_prog_intr(ch);
  187. return 0;
  188. }
  189. static inline void mic_dma_issue_pending(struct dma_chan *ch)
  190. {
  191. struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
  192. spin_lock(&mic_ch->issue_lock);
  193. /*
  194. * Write to head triggers h/w to act on the descriptors.
  195. * On MIC, writing the same head value twice causes
  196. * a h/w error. On second write, h/w assumes we filled
  197. * the entire ring & overwrote some of the descriptors.
  198. */
  199. if (mic_ch->issued == mic_ch->submitted)
  200. goto out;
  201. mic_ch->issued = mic_ch->submitted;
  202. /*
  203. * make descriptor updates visible before advancing head,
  204. * this is purposefully not smp_wmb() since we are also
  205. * publishing the descriptor updates to a dma device
  206. */
  207. wmb();
  208. mic_dma_write_reg(mic_ch, MIC_DMA_REG_DHPR, mic_ch->issued);
  209. out:
  210. spin_unlock(&mic_ch->issue_lock);
  211. }
  212. static inline void mic_dma_update_pending(struct mic_dma_chan *ch)
  213. {
  214. if (mic_dma_ring_count(ch->issued, ch->submitted)
  215. > mic_dma_pending_level)
  216. mic_dma_issue_pending(&ch->api_ch);
  217. }
  218. static dma_cookie_t mic_dma_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
  219. {
  220. struct mic_dma_chan *mic_ch = to_mic_dma_chan(tx->chan);
  221. dma_cookie_t cookie;
  222. dma_cookie_assign(tx);
  223. cookie = tx->cookie;
  224. /*
  225. * We need an smp write barrier here because another CPU might see
  226. * an update to submitted and update h/w head even before we
  227. * assigned a cookie to this tx.
  228. */
  229. smp_wmb();
  230. mic_ch->submitted = mic_ch->head;
  231. spin_unlock(&mic_ch->prep_lock);
  232. mic_dma_update_pending(mic_ch);
  233. return cookie;
  234. }
  235. static inline struct dma_async_tx_descriptor *
  236. allocate_tx(struct mic_dma_chan *ch)
  237. {
  238. u32 idx = mic_dma_hw_ring_dec(ch->head);
  239. struct dma_async_tx_descriptor *tx = &ch->tx_array[idx];
  240. dma_async_tx_descriptor_init(tx, &ch->api_ch);
  241. tx->tx_submit = mic_dma_tx_submit_unlock;
  242. return tx;
  243. }
  244. /* Program a status descriptor with dst as address and value to be written */
  245. static struct dma_async_tx_descriptor *
  246. mic_dma_prep_status_lock(struct dma_chan *ch, dma_addr_t dst, u64 src_val,
  247. unsigned long flags)
  248. {
  249. struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
  250. int result;
  251. spin_lock(&mic_ch->prep_lock);
  252. result = mic_dma_avail_desc_ring_space(mic_ch, 4);
  253. if (result < 0)
  254. goto error;
  255. mic_dma_prep_status_desc(&mic_ch->desc_ring[mic_ch->head], src_val, dst,
  256. false);
  257. mic_dma_hw_ring_inc_head(mic_ch);
  258. result = mic_dma_do_dma(mic_ch, flags, 0, 0, 0);
  259. if (result < 0)
  260. goto error;
  261. return allocate_tx(mic_ch);
  262. error:
  263. dev_err(mic_dma_ch_to_device(mic_ch),
  264. "Error enqueueing dma status descriptor, error=%d\n", result);
  265. spin_unlock(&mic_ch->prep_lock);
  266. return NULL;
  267. }
  268. /*
  269. * Prepare a memcpy descriptor to be added to the ring.
  270. * Note that the temporary descriptor adds an extra overhead of copying the
  271. * descriptor to ring. So, we copy directly to the descriptor ring
  272. */
  273. static struct dma_async_tx_descriptor *
  274. mic_dma_prep_memcpy_lock(struct dma_chan *ch, dma_addr_t dma_dest,
  275. dma_addr_t dma_src, size_t len, unsigned long flags)
  276. {
  277. struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
  278. struct device *dev = mic_dma_ch_to_device(mic_ch);
  279. int result;
  280. if (!len && !flags)
  281. return NULL;
  282. spin_lock(&mic_ch->prep_lock);
  283. result = mic_dma_do_dma(mic_ch, flags, dma_src, dma_dest, len);
  284. if (result >= 0)
  285. return allocate_tx(mic_ch);
  286. dev_err(dev, "Error enqueueing dma, error=%d\n", result);
  287. spin_unlock(&mic_ch->prep_lock);
  288. return NULL;
  289. }
  290. static struct dma_async_tx_descriptor *
  291. mic_dma_prep_interrupt_lock(struct dma_chan *ch, unsigned long flags)
  292. {
  293. struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
  294. int ret;
  295. spin_lock(&mic_ch->prep_lock);
  296. ret = mic_dma_do_dma(mic_ch, flags, 0, 0, 0);
  297. if (!ret)
  298. return allocate_tx(mic_ch);
  299. spin_unlock(&mic_ch->prep_lock);
  300. return NULL;
  301. }
  302. /* Return the status of the transaction */
  303. static enum dma_status
  304. mic_dma_tx_status(struct dma_chan *ch, dma_cookie_t cookie,
  305. struct dma_tx_state *txstate)
  306. {
  307. struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
  308. if (DMA_COMPLETE != dma_cookie_status(ch, cookie, txstate))
  309. mic_dma_cleanup(mic_ch);
  310. return dma_cookie_status(ch, cookie, txstate);
  311. }
  312. static irqreturn_t mic_dma_thread_fn(int irq, void *data)
  313. {
  314. mic_dma_cleanup((struct mic_dma_chan *)data);
  315. return IRQ_HANDLED;
  316. }
  317. static irqreturn_t mic_dma_intr_handler(int irq, void *data)
  318. {
  319. struct mic_dma_chan *ch = ((struct mic_dma_chan *)data);
  320. mic_dma_ack_interrupt(ch);
  321. return IRQ_WAKE_THREAD;
  322. }
  323. static int mic_dma_alloc_desc_ring(struct mic_dma_chan *ch)
  324. {
  325. u64 desc_ring_size = MIC_DMA_DESC_RX_SIZE * sizeof(*ch->desc_ring);
  326. struct device *dev = &to_mbus_device(ch)->dev;
  327. desc_ring_size = ALIGN(desc_ring_size, MIC_DMA_ALIGN_BYTES);
  328. ch->desc_ring = kzalloc(desc_ring_size, GFP_KERNEL);
  329. if (!ch->desc_ring)
  330. return -ENOMEM;
  331. ch->desc_ring_micpa = dma_map_single(dev, ch->desc_ring,
  332. desc_ring_size, DMA_BIDIRECTIONAL);
  333. if (dma_mapping_error(dev, ch->desc_ring_micpa))
  334. goto map_error;
  335. ch->tx_array = vzalloc(MIC_DMA_DESC_RX_SIZE * sizeof(*ch->tx_array));
  336. if (!ch->tx_array)
  337. goto tx_error;
  338. return 0;
  339. tx_error:
  340. dma_unmap_single(dev, ch->desc_ring_micpa, desc_ring_size,
  341. DMA_BIDIRECTIONAL);
  342. map_error:
  343. kfree(ch->desc_ring);
  344. return -ENOMEM;
  345. }
  346. static void mic_dma_free_desc_ring(struct mic_dma_chan *ch)
  347. {
  348. u64 desc_ring_size = MIC_DMA_DESC_RX_SIZE * sizeof(*ch->desc_ring);
  349. vfree(ch->tx_array);
  350. desc_ring_size = ALIGN(desc_ring_size, MIC_DMA_ALIGN_BYTES);
  351. dma_unmap_single(&to_mbus_device(ch)->dev, ch->desc_ring_micpa,
  352. desc_ring_size, DMA_BIDIRECTIONAL);
  353. kfree(ch->desc_ring);
  354. ch->desc_ring = NULL;
  355. }
  356. static void mic_dma_free_status_dest(struct mic_dma_chan *ch)
  357. {
  358. dma_unmap_single(&to_mbus_device(ch)->dev, ch->status_dest_micpa,
  359. L1_CACHE_BYTES, DMA_BIDIRECTIONAL);
  360. kfree(ch->status_dest);
  361. }
  362. static int mic_dma_alloc_status_dest(struct mic_dma_chan *ch)
  363. {
  364. struct device *dev = &to_mbus_device(ch)->dev;
  365. ch->status_dest = kzalloc(L1_CACHE_BYTES, GFP_KERNEL);
  366. if (!ch->status_dest)
  367. return -ENOMEM;
  368. ch->status_dest_micpa = dma_map_single(dev, ch->status_dest,
  369. L1_CACHE_BYTES, DMA_BIDIRECTIONAL);
  370. if (dma_mapping_error(dev, ch->status_dest_micpa)) {
  371. kfree(ch->status_dest);
  372. ch->status_dest = NULL;
  373. return -ENOMEM;
  374. }
  375. return 0;
  376. }
  377. static int mic_dma_check_chan(struct mic_dma_chan *ch)
  378. {
  379. if (mic_dma_read_reg(ch, MIC_DMA_REG_DCHERR) ||
  380. mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT) & MIC_DMA_CHAN_QUIESCE) {
  381. mic_dma_disable_chan(ch);
  382. mic_dma_chan_mask_intr(ch);
  383. dev_err(mic_dma_ch_to_device(ch),
  384. "%s %d error setting up mic dma chan %d\n",
  385. __func__, __LINE__, ch->ch_num);
  386. return -EBUSY;
  387. }
  388. return 0;
  389. }
  390. static int mic_dma_chan_setup(struct mic_dma_chan *ch)
  391. {
  392. if (MIC_DMA_CHAN_MIC == ch->owner)
  393. mic_dma_chan_set_owner(ch);
  394. mic_dma_disable_chan(ch);
  395. mic_dma_chan_mask_intr(ch);
  396. mic_dma_write_reg(ch, MIC_DMA_REG_DCHERRMSK, 0);
  397. mic_dma_chan_set_desc_ring(ch);
  398. ch->last_tail = mic_dma_read_reg(ch, MIC_DMA_REG_DTPR);
  399. ch->head = ch->last_tail;
  400. ch->issued = 0;
  401. mic_dma_chan_unmask_intr(ch);
  402. mic_dma_enable_chan(ch);
  403. return mic_dma_check_chan(ch);
  404. }
  405. static void mic_dma_chan_destroy(struct mic_dma_chan *ch)
  406. {
  407. mic_dma_disable_chan(ch);
  408. mic_dma_chan_mask_intr(ch);
  409. }
  410. static void mic_dma_unregister_dma_device(struct mic_dma_device *mic_dma_dev)
  411. {
  412. dma_async_device_unregister(&mic_dma_dev->dma_dev);
  413. }
  414. static int mic_dma_setup_irq(struct mic_dma_chan *ch)
  415. {
  416. ch->cookie =
  417. to_mbus_hw_ops(ch)->request_threaded_irq(to_mbus_device(ch),
  418. mic_dma_intr_handler, mic_dma_thread_fn,
  419. "mic dma_channel", ch, ch->ch_num);
  420. if (IS_ERR(ch->cookie))
  421. return PTR_ERR(ch->cookie);
  422. return 0;
  423. }
  424. static inline void mic_dma_free_irq(struct mic_dma_chan *ch)
  425. {
  426. to_mbus_hw_ops(ch)->free_irq(to_mbus_device(ch), ch->cookie, ch);
  427. }
  428. static int mic_dma_chan_init(struct mic_dma_chan *ch)
  429. {
  430. int ret = mic_dma_alloc_desc_ring(ch);
  431. if (ret)
  432. goto ring_error;
  433. ret = mic_dma_alloc_status_dest(ch);
  434. if (ret)
  435. goto status_error;
  436. ret = mic_dma_chan_setup(ch);
  437. if (ret)
  438. goto chan_error;
  439. return ret;
  440. chan_error:
  441. mic_dma_free_status_dest(ch);
  442. status_error:
  443. mic_dma_free_desc_ring(ch);
  444. ring_error:
  445. return ret;
  446. }
  447. static int mic_dma_drain_chan(struct mic_dma_chan *ch)
  448. {
  449. struct dma_async_tx_descriptor *tx;
  450. int err = 0;
  451. dma_cookie_t cookie;
  452. tx = mic_dma_prep_memcpy_lock(&ch->api_ch, 0, 0, 0, DMA_PREP_FENCE);
  453. if (!tx) {
  454. err = -ENOMEM;
  455. goto error;
  456. }
  457. cookie = tx->tx_submit(tx);
  458. if (dma_submit_error(cookie))
  459. err = -ENOMEM;
  460. else
  461. err = dma_sync_wait(&ch->api_ch, cookie);
  462. if (err) {
  463. dev_err(mic_dma_ch_to_device(ch), "%s %d TO chan 0x%x\n",
  464. __func__, __LINE__, ch->ch_num);
  465. err = -EIO;
  466. }
  467. error:
  468. mic_dma_cleanup(ch);
  469. return err;
  470. }
  471. static inline void mic_dma_chan_uninit(struct mic_dma_chan *ch)
  472. {
  473. mic_dma_chan_destroy(ch);
  474. mic_dma_cleanup(ch);
  475. mic_dma_free_status_dest(ch);
  476. mic_dma_free_desc_ring(ch);
  477. }
  478. static int mic_dma_init(struct mic_dma_device *mic_dma_dev,
  479. enum mic_dma_chan_owner owner)
  480. {
  481. int i, first_chan = mic_dma_dev->start_ch;
  482. struct mic_dma_chan *ch;
  483. int ret;
  484. for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
  485. unsigned long data;
  486. ch = &mic_dma_dev->mic_ch[i];
  487. data = (unsigned long)ch;
  488. ch->ch_num = i;
  489. ch->owner = owner;
  490. spin_lock_init(&ch->cleanup_lock);
  491. spin_lock_init(&ch->prep_lock);
  492. spin_lock_init(&ch->issue_lock);
  493. ret = mic_dma_setup_irq(ch);
  494. if (ret)
  495. goto error;
  496. }
  497. return 0;
  498. error:
  499. for (i = i - 1; i >= first_chan; i--)
  500. mic_dma_free_irq(ch);
  501. return ret;
  502. }
  503. static void mic_dma_uninit(struct mic_dma_device *mic_dma_dev)
  504. {
  505. int i, first_chan = mic_dma_dev->start_ch;
  506. struct mic_dma_chan *ch;
  507. for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
  508. ch = &mic_dma_dev->mic_ch[i];
  509. mic_dma_free_irq(ch);
  510. }
  511. }
  512. static int mic_dma_alloc_chan_resources(struct dma_chan *ch)
  513. {
  514. int ret = mic_dma_chan_init(to_mic_dma_chan(ch));
  515. if (ret)
  516. return ret;
  517. return MIC_DMA_DESC_RX_SIZE;
  518. }
  519. static void mic_dma_free_chan_resources(struct dma_chan *ch)
  520. {
  521. struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
  522. mic_dma_drain_chan(mic_ch);
  523. mic_dma_chan_uninit(mic_ch);
  524. }
  525. /* Set the fn. handlers and register the dma device with dma api */
  526. static int mic_dma_register_dma_device(struct mic_dma_device *mic_dma_dev,
  527. enum mic_dma_chan_owner owner)
  528. {
  529. int i, first_chan = mic_dma_dev->start_ch;
  530. dma_cap_zero(mic_dma_dev->dma_dev.cap_mask);
  531. /*
  532. * This dma engine is not capable of host memory to host memory
  533. * transfers
  534. */
  535. dma_cap_set(DMA_MEMCPY, mic_dma_dev->dma_dev.cap_mask);
  536. if (MIC_DMA_CHAN_HOST == owner)
  537. dma_cap_set(DMA_PRIVATE, mic_dma_dev->dma_dev.cap_mask);
  538. mic_dma_dev->dma_dev.device_alloc_chan_resources =
  539. mic_dma_alloc_chan_resources;
  540. mic_dma_dev->dma_dev.device_free_chan_resources =
  541. mic_dma_free_chan_resources;
  542. mic_dma_dev->dma_dev.device_tx_status = mic_dma_tx_status;
  543. mic_dma_dev->dma_dev.device_prep_dma_memcpy = mic_dma_prep_memcpy_lock;
  544. mic_dma_dev->dma_dev.device_prep_dma_imm_data =
  545. mic_dma_prep_status_lock;
  546. mic_dma_dev->dma_dev.device_prep_dma_interrupt =
  547. mic_dma_prep_interrupt_lock;
  548. mic_dma_dev->dma_dev.device_issue_pending = mic_dma_issue_pending;
  549. mic_dma_dev->dma_dev.copy_align = MIC_DMA_ALIGN_SHIFT;
  550. INIT_LIST_HEAD(&mic_dma_dev->dma_dev.channels);
  551. for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
  552. mic_dma_dev->mic_ch[i].api_ch.device = &mic_dma_dev->dma_dev;
  553. dma_cookie_init(&mic_dma_dev->mic_ch[i].api_ch);
  554. list_add_tail(&mic_dma_dev->mic_ch[i].api_ch.device_node,
  555. &mic_dma_dev->dma_dev.channels);
  556. }
  557. return dma_async_device_register(&mic_dma_dev->dma_dev);
  558. }
  559. /*
  560. * Initializes dma channels and registers the dma device with the
  561. * dma engine api.
  562. */
  563. static struct mic_dma_device *mic_dma_dev_reg(struct mbus_device *mbdev,
  564. enum mic_dma_chan_owner owner)
  565. {
  566. struct mic_dma_device *mic_dma_dev;
  567. int ret;
  568. struct device *dev = &mbdev->dev;
  569. mic_dma_dev = kzalloc(sizeof(*mic_dma_dev), GFP_KERNEL);
  570. if (!mic_dma_dev) {
  571. ret = -ENOMEM;
  572. goto alloc_error;
  573. }
  574. mic_dma_dev->mbdev = mbdev;
  575. mic_dma_dev->dma_dev.dev = dev;
  576. mic_dma_dev->mmio = mbdev->mmio_va;
  577. if (MIC_DMA_CHAN_HOST == owner) {
  578. mic_dma_dev->start_ch = 0;
  579. mic_dma_dev->max_xfer_size = MIC_DMA_MAX_XFER_SIZE_HOST;
  580. } else {
  581. mic_dma_dev->start_ch = 4;
  582. mic_dma_dev->max_xfer_size = MIC_DMA_MAX_XFER_SIZE_CARD;
  583. }
  584. ret = mic_dma_init(mic_dma_dev, owner);
  585. if (ret)
  586. goto init_error;
  587. ret = mic_dma_register_dma_device(mic_dma_dev, owner);
  588. if (ret)
  589. goto reg_error;
  590. return mic_dma_dev;
  591. reg_error:
  592. mic_dma_uninit(mic_dma_dev);
  593. init_error:
  594. kfree(mic_dma_dev);
  595. mic_dma_dev = NULL;
  596. alloc_error:
  597. dev_err(dev, "Error at %s %d ret=%d\n", __func__, __LINE__, ret);
  598. return mic_dma_dev;
  599. }
  600. static void mic_dma_dev_unreg(struct mic_dma_device *mic_dma_dev)
  601. {
  602. mic_dma_unregister_dma_device(mic_dma_dev);
  603. mic_dma_uninit(mic_dma_dev);
  604. kfree(mic_dma_dev);
  605. }
  606. /* DEBUGFS CODE */
  607. static int mic_dma_reg_seq_show(struct seq_file *s, void *pos)
  608. {
  609. struct mic_dma_device *mic_dma_dev = s->private;
  610. int i, chan_num, first_chan = mic_dma_dev->start_ch;
  611. struct mic_dma_chan *ch;
  612. seq_printf(s, "SBOX_DCR: %#x\n",
  613. mic_dma_mmio_read(&mic_dma_dev->mic_ch[first_chan],
  614. MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR));
  615. seq_puts(s, "DMA Channel Registers\n");
  616. seq_printf(s, "%-10s| %-10s %-10s %-10s %-10s %-10s",
  617. "Channel", "DCAR", "DTPR", "DHPR", "DRAR_HI", "DRAR_LO");
  618. seq_printf(s, " %-11s %-14s %-10s\n", "DCHERR", "DCHERRMSK", "DSTAT");
  619. for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
  620. ch = &mic_dma_dev->mic_ch[i];
  621. chan_num = ch->ch_num;
  622. seq_printf(s, "%-10i| %-#10x %-#10x %-#10x %-#10x",
  623. chan_num,
  624. mic_dma_read_reg(ch, MIC_DMA_REG_DCAR),
  625. mic_dma_read_reg(ch, MIC_DMA_REG_DTPR),
  626. mic_dma_read_reg(ch, MIC_DMA_REG_DHPR),
  627. mic_dma_read_reg(ch, MIC_DMA_REG_DRAR_HI));
  628. seq_printf(s, " %-#10x %-#10x %-#14x %-#10x\n",
  629. mic_dma_read_reg(ch, MIC_DMA_REG_DRAR_LO),
  630. mic_dma_read_reg(ch, MIC_DMA_REG_DCHERR),
  631. mic_dma_read_reg(ch, MIC_DMA_REG_DCHERRMSK),
  632. mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT));
  633. }
  634. return 0;
  635. }
  636. static int mic_dma_reg_debug_open(struct inode *inode, struct file *file)
  637. {
  638. return single_open(file, mic_dma_reg_seq_show, inode->i_private);
  639. }
  640. static int mic_dma_reg_debug_release(struct inode *inode, struct file *file)
  641. {
  642. return single_release(inode, file);
  643. }
  644. static const struct file_operations mic_dma_reg_ops = {
  645. .owner = THIS_MODULE,
  646. .open = mic_dma_reg_debug_open,
  647. .read = seq_read,
  648. .llseek = seq_lseek,
  649. .release = mic_dma_reg_debug_release
  650. };
  651. /* Debugfs parent dir */
  652. static struct dentry *mic_dma_dbg;
  653. static int mic_dma_driver_probe(struct mbus_device *mbdev)
  654. {
  655. struct mic_dma_device *mic_dma_dev;
  656. enum mic_dma_chan_owner owner;
  657. if (MBUS_DEV_DMA_MIC == mbdev->id.device)
  658. owner = MIC_DMA_CHAN_MIC;
  659. else
  660. owner = MIC_DMA_CHAN_HOST;
  661. mic_dma_dev = mic_dma_dev_reg(mbdev, owner);
  662. dev_set_drvdata(&mbdev->dev, mic_dma_dev);
  663. if (mic_dma_dbg) {
  664. mic_dma_dev->dbg_dir = debugfs_create_dir(dev_name(&mbdev->dev),
  665. mic_dma_dbg);
  666. if (mic_dma_dev->dbg_dir)
  667. debugfs_create_file("mic_dma_reg", 0444,
  668. mic_dma_dev->dbg_dir, mic_dma_dev,
  669. &mic_dma_reg_ops);
  670. }
  671. return 0;
  672. }
  673. static void mic_dma_driver_remove(struct mbus_device *mbdev)
  674. {
  675. struct mic_dma_device *mic_dma_dev;
  676. mic_dma_dev = dev_get_drvdata(&mbdev->dev);
  677. debugfs_remove_recursive(mic_dma_dev->dbg_dir);
  678. mic_dma_dev_unreg(mic_dma_dev);
  679. }
  680. static struct mbus_device_id id_table[] = {
  681. {MBUS_DEV_DMA_MIC, MBUS_DEV_ANY_ID},
  682. {MBUS_DEV_DMA_HOST, MBUS_DEV_ANY_ID},
  683. {0},
  684. };
  685. static struct mbus_driver mic_dma_driver = {
  686. .driver.name = KBUILD_MODNAME,
  687. .driver.owner = THIS_MODULE,
  688. .id_table = id_table,
  689. .probe = mic_dma_driver_probe,
  690. .remove = mic_dma_driver_remove,
  691. };
  692. static int __init mic_x100_dma_init(void)
  693. {
  694. int rc = mbus_register_driver(&mic_dma_driver);
  695. if (rc)
  696. return rc;
  697. mic_dma_dbg = debugfs_create_dir(KBUILD_MODNAME, NULL);
  698. return 0;
  699. }
  700. static void __exit mic_x100_dma_exit(void)
  701. {
  702. debugfs_remove_recursive(mic_dma_dbg);
  703. mbus_unregister_driver(&mic_dma_driver);
  704. }
  705. module_init(mic_x100_dma_init);
  706. module_exit(mic_x100_dma_exit);
  707. MODULE_DEVICE_TABLE(mbus, id_table);
  708. MODULE_AUTHOR("Intel Corporation");
  709. MODULE_DESCRIPTION("Intel(R) MIC X100 DMA Driver");
  710. MODULE_LICENSE("GPL v2");