dma.c 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299
  1. /*
  2. * Copyright (C) 2006 Benjamin Herrenschmidt, IBM Corporation
  3. *
  4. * Provide default implementations of the DMA mapping callbacks for
  5. * directly mapped busses.
  6. */
  7. #include <linux/device.h>
  8. #include <linux/dma-mapping.h>
  9. #include <linux/dma-debug.h>
  10. #include <linux/gfp.h>
  11. #include <linux/memblock.h>
  12. #include <linux/export.h>
  13. #include <linux/pci.h>
  14. #include <asm/vio.h>
  15. #include <asm/bug.h>
  16. #include <asm/machdep.h>
  17. #include <asm/swiotlb.h>
  18. /*
  19. * Generic direct DMA implementation
  20. *
  21. * This implementation supports a per-device offset that can be applied if
  22. * the address at which memory is visible to devices is not 0. Platform code
  23. * can set archdata.dma_data to an unsigned long holding the offset. By
  24. * default the offset is PCI_DRAM_OFFSET.
  25. */
  26. static u64 __maybe_unused get_pfn_limit(struct device *dev)
  27. {
  28. u64 pfn = (dev->coherent_dma_mask >> PAGE_SHIFT) + 1;
  29. struct dev_archdata __maybe_unused *sd = &dev->archdata;
  30. #ifdef CONFIG_SWIOTLB
  31. if (sd->max_direct_dma_addr && sd->dma_ops == &swiotlb_dma_ops)
  32. pfn = min_t(u64, pfn, sd->max_direct_dma_addr >> PAGE_SHIFT);
  33. #endif
  34. return pfn;
  35. }
  36. void *dma_direct_alloc_coherent(struct device *dev, size_t size,
  37. dma_addr_t *dma_handle, gfp_t flag,
  38. struct dma_attrs *attrs)
  39. {
  40. void *ret;
  41. #ifdef CONFIG_NOT_COHERENT_CACHE
  42. ret = __dma_alloc_coherent(dev, size, dma_handle, flag);
  43. if (ret == NULL)
  44. return NULL;
  45. *dma_handle += get_dma_offset(dev);
  46. return ret;
  47. #else
  48. struct page *page;
  49. int node = dev_to_node(dev);
  50. #ifdef CONFIG_FSL_SOC
  51. u64 pfn = get_pfn_limit(dev);
  52. int zone;
  53. /*
  54. * This code should be OK on other platforms, but we have drivers that
  55. * don't set coherent_dma_mask. As a workaround we just ifdef it. This
  56. * whole routine needs some serious cleanup.
  57. */
  58. zone = dma_pfn_limit_to_zone(pfn);
  59. if (zone < 0) {
  60. dev_err(dev, "%s: No suitable zone for pfn %#llx\n",
  61. __func__, pfn);
  62. return NULL;
  63. }
  64. switch (zone) {
  65. case ZONE_DMA:
  66. flag |= GFP_DMA;
  67. break;
  68. #ifdef CONFIG_ZONE_DMA32
  69. case ZONE_DMA32:
  70. flag |= GFP_DMA32;
  71. break;
  72. #endif
  73. };
  74. #endif /* CONFIG_FSL_SOC */
  75. /* ignore region specifiers */
  76. flag &= ~(__GFP_HIGHMEM);
  77. page = alloc_pages_node(node, flag, get_order(size));
  78. if (page == NULL)
  79. return NULL;
  80. ret = page_address(page);
  81. memset(ret, 0, size);
  82. *dma_handle = __pa(ret) + get_dma_offset(dev);
  83. return ret;
  84. #endif
  85. }
  86. void dma_direct_free_coherent(struct device *dev, size_t size,
  87. void *vaddr, dma_addr_t dma_handle,
  88. struct dma_attrs *attrs)
  89. {
  90. #ifdef CONFIG_NOT_COHERENT_CACHE
  91. __dma_free_coherent(size, vaddr);
  92. #else
  93. free_pages((unsigned long)vaddr, get_order(size));
  94. #endif
  95. }
  96. int dma_direct_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
  97. void *cpu_addr, dma_addr_t handle, size_t size,
  98. struct dma_attrs *attrs)
  99. {
  100. unsigned long pfn;
  101. #ifdef CONFIG_NOT_COHERENT_CACHE
  102. vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
  103. pfn = __dma_get_coherent_pfn((unsigned long)cpu_addr);
  104. #else
  105. pfn = page_to_pfn(virt_to_page(cpu_addr));
  106. #endif
  107. return remap_pfn_range(vma, vma->vm_start,
  108. pfn + vma->vm_pgoff,
  109. vma->vm_end - vma->vm_start,
  110. vma->vm_page_prot);
  111. }
  112. static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl,
  113. int nents, enum dma_data_direction direction,
  114. struct dma_attrs *attrs)
  115. {
  116. struct scatterlist *sg;
  117. int i;
  118. for_each_sg(sgl, sg, nents, i) {
  119. sg->dma_address = sg_phys(sg) + get_dma_offset(dev);
  120. sg->dma_length = sg->length;
  121. __dma_sync_page(sg_page(sg), sg->offset, sg->length, direction);
  122. }
  123. return nents;
  124. }
  125. static void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sg,
  126. int nents, enum dma_data_direction direction,
  127. struct dma_attrs *attrs)
  128. {
  129. }
  130. static int dma_direct_dma_supported(struct device *dev, u64 mask)
  131. {
  132. #ifdef CONFIG_PPC64
  133. /* Could be improved so platforms can set the limit in case
  134. * they have limited DMA windows
  135. */
  136. return mask >= get_dma_offset(dev) + (memblock_end_of_DRAM() - 1);
  137. #else
  138. return 1;
  139. #endif
  140. }
  141. static u64 dma_direct_get_required_mask(struct device *dev)
  142. {
  143. u64 end, mask;
  144. end = memblock_end_of_DRAM() + get_dma_offset(dev);
  145. mask = 1ULL << (fls64(end) - 1);
  146. mask += mask - 1;
  147. return mask;
  148. }
  149. static inline dma_addr_t dma_direct_map_page(struct device *dev,
  150. struct page *page,
  151. unsigned long offset,
  152. size_t size,
  153. enum dma_data_direction dir,
  154. struct dma_attrs *attrs)
  155. {
  156. BUG_ON(dir == DMA_NONE);
  157. __dma_sync_page(page, offset, size, dir);
  158. return page_to_phys(page) + offset + get_dma_offset(dev);
  159. }
  160. static inline void dma_direct_unmap_page(struct device *dev,
  161. dma_addr_t dma_address,
  162. size_t size,
  163. enum dma_data_direction direction,
  164. struct dma_attrs *attrs)
  165. {
  166. }
  167. #ifdef CONFIG_NOT_COHERENT_CACHE
  168. static inline void dma_direct_sync_sg(struct device *dev,
  169. struct scatterlist *sgl, int nents,
  170. enum dma_data_direction direction)
  171. {
  172. struct scatterlist *sg;
  173. int i;
  174. for_each_sg(sgl, sg, nents, i)
  175. __dma_sync_page(sg_page(sg), sg->offset, sg->length, direction);
  176. }
  177. static inline void dma_direct_sync_single(struct device *dev,
  178. dma_addr_t dma_handle, size_t size,
  179. enum dma_data_direction direction)
  180. {
  181. __dma_sync(bus_to_virt(dma_handle), size, direction);
  182. }
  183. #endif
  184. struct dma_map_ops dma_direct_ops = {
  185. .alloc = dma_direct_alloc_coherent,
  186. .free = dma_direct_free_coherent,
  187. .mmap = dma_direct_mmap_coherent,
  188. .map_sg = dma_direct_map_sg,
  189. .unmap_sg = dma_direct_unmap_sg,
  190. .dma_supported = dma_direct_dma_supported,
  191. .map_page = dma_direct_map_page,
  192. .unmap_page = dma_direct_unmap_page,
  193. .get_required_mask = dma_direct_get_required_mask,
  194. #ifdef CONFIG_NOT_COHERENT_CACHE
  195. .sync_single_for_cpu = dma_direct_sync_single,
  196. .sync_single_for_device = dma_direct_sync_single,
  197. .sync_sg_for_cpu = dma_direct_sync_sg,
  198. .sync_sg_for_device = dma_direct_sync_sg,
  199. #endif
  200. };
  201. EXPORT_SYMBOL(dma_direct_ops);
  202. #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
  203. int __dma_set_mask(struct device *dev, u64 dma_mask)
  204. {
  205. struct dma_map_ops *dma_ops = get_dma_ops(dev);
  206. if ((dma_ops != NULL) && (dma_ops->set_dma_mask != NULL))
  207. return dma_ops->set_dma_mask(dev, dma_mask);
  208. if (!dev->dma_mask || !dma_supported(dev, dma_mask))
  209. return -EIO;
  210. *dev->dma_mask = dma_mask;
  211. return 0;
  212. }
  213. int dma_set_mask(struct device *dev, u64 dma_mask)
  214. {
  215. if (ppc_md.dma_set_mask)
  216. return ppc_md.dma_set_mask(dev, dma_mask);
  217. if (dev_is_pci(dev)) {
  218. struct pci_dev *pdev = to_pci_dev(dev);
  219. struct pci_controller *phb = pci_bus_to_host(pdev->bus);
  220. if (phb->controller_ops.dma_set_mask)
  221. return phb->controller_ops.dma_set_mask(pdev, dma_mask);
  222. }
  223. return __dma_set_mask(dev, dma_mask);
  224. }
  225. EXPORT_SYMBOL(dma_set_mask);
  226. u64 __dma_get_required_mask(struct device *dev)
  227. {
  228. struct dma_map_ops *dma_ops = get_dma_ops(dev);
  229. if (unlikely(dma_ops == NULL))
  230. return 0;
  231. if (dma_ops->get_required_mask)
  232. return dma_ops->get_required_mask(dev);
  233. return DMA_BIT_MASK(8 * sizeof(dma_addr_t));
  234. }
  235. u64 dma_get_required_mask(struct device *dev)
  236. {
  237. if (ppc_md.dma_get_required_mask)
  238. return ppc_md.dma_get_required_mask(dev);
  239. return __dma_get_required_mask(dev);
  240. }
  241. EXPORT_SYMBOL_GPL(dma_get_required_mask);
  242. static int __init dma_init(void)
  243. {
  244. dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
  245. #ifdef CONFIG_PCI
  246. dma_debug_add_bus(&pci_bus_type);
  247. #endif
  248. #ifdef CONFIG_IBMVIO
  249. dma_debug_add_bus(&vio_bus_type);
  250. #endif
  251. return 0;
  252. }
  253. fs_initcall(dma_init);