picoxcell_crypto.c 50 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright (c) 2010-2011 Picochip Ltd., Jamie Iles
  4. */
  5. #include <crypto/internal/aead.h>
  6. #include <crypto/aes.h>
  7. #include <crypto/algapi.h>
  8. #include <crypto/authenc.h>
  9. #include <crypto/internal/des.h>
  10. #include <crypto/md5.h>
  11. #include <crypto/sha.h>
  12. #include <crypto/internal/skcipher.h>
  13. #include <linux/clk.h>
  14. #include <linux/crypto.h>
  15. #include <linux/delay.h>
  16. #include <linux/dma-mapping.h>
  17. #include <linux/dmapool.h>
  18. #include <linux/err.h>
  19. #include <linux/init.h>
  20. #include <linux/interrupt.h>
  21. #include <linux/io.h>
  22. #include <linux/list.h>
  23. #include <linux/module.h>
  24. #include <linux/of.h>
  25. #include <linux/platform_device.h>
  26. #include <linux/pm.h>
  27. #include <linux/rtnetlink.h>
  28. #include <linux/scatterlist.h>
  29. #include <linux/sched.h>
  30. #include <linux/sizes.h>
  31. #include <linux/slab.h>
  32. #include <linux/timer.h>
  33. #include "picoxcell_crypto_regs.h"
  34. /*
  35. * The threshold for the number of entries in the CMD FIFO available before
  36. * the CMD0_CNT interrupt is raised. Increasing this value will reduce the
  37. * number of interrupts raised to the CPU.
  38. */
  39. #define CMD0_IRQ_THRESHOLD 1
  40. /*
  41. * The timeout period (in jiffies) for a PDU. When the the number of PDUs in
  42. * flight is greater than the STAT_IRQ_THRESHOLD or 0 the timer is disabled.
  43. * When there are packets in flight but lower than the threshold, we enable
  44. * the timer and at expiry, attempt to remove any processed packets from the
  45. * queue and if there are still packets left, schedule the timer again.
  46. */
  47. #define PACKET_TIMEOUT 1
  48. /* The priority to register each algorithm with. */
  49. #define SPACC_CRYPTO_ALG_PRIORITY 10000
  50. #define SPACC_CRYPTO_KASUMI_F8_KEY_LEN 16
  51. #define SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ 64
  52. #define SPACC_CRYPTO_IPSEC_HASH_PG_SZ 64
  53. #define SPACC_CRYPTO_IPSEC_MAX_CTXS 32
  54. #define SPACC_CRYPTO_IPSEC_FIFO_SZ 32
  55. #define SPACC_CRYPTO_L2_CIPHER_PG_SZ 64
  56. #define SPACC_CRYPTO_L2_HASH_PG_SZ 64
  57. #define SPACC_CRYPTO_L2_MAX_CTXS 128
  58. #define SPACC_CRYPTO_L2_FIFO_SZ 128
  59. #define MAX_DDT_LEN 16
  60. /* DDT format. This must match the hardware DDT format exactly. */
  61. struct spacc_ddt {
  62. dma_addr_t p;
  63. u32 len;
  64. };
  65. /*
  66. * Asynchronous crypto request structure.
  67. *
  68. * This structure defines a request that is either queued for processing or
  69. * being processed.
  70. */
  71. struct spacc_req {
  72. struct list_head list;
  73. struct spacc_engine *engine;
  74. struct crypto_async_request *req;
  75. int result;
  76. bool is_encrypt;
  77. unsigned ctx_id;
  78. dma_addr_t src_addr, dst_addr;
  79. struct spacc_ddt *src_ddt, *dst_ddt;
  80. void (*complete)(struct spacc_req *req);
  81. };
  82. struct spacc_aead {
  83. unsigned long ctrl_default;
  84. unsigned long type;
  85. struct aead_alg alg;
  86. struct spacc_engine *engine;
  87. struct list_head entry;
  88. int key_offs;
  89. int iv_offs;
  90. };
  91. struct spacc_engine {
  92. void __iomem *regs;
  93. struct list_head pending;
  94. int next_ctx;
  95. spinlock_t hw_lock;
  96. int in_flight;
  97. struct list_head completed;
  98. struct list_head in_progress;
  99. struct tasklet_struct complete;
  100. unsigned long fifo_sz;
  101. void __iomem *cipher_ctx_base;
  102. void __iomem *hash_key_base;
  103. struct spacc_alg *algs;
  104. unsigned num_algs;
  105. struct list_head registered_algs;
  106. struct spacc_aead *aeads;
  107. unsigned num_aeads;
  108. struct list_head registered_aeads;
  109. size_t cipher_pg_sz;
  110. size_t hash_pg_sz;
  111. const char *name;
  112. struct clk *clk;
  113. struct device *dev;
  114. unsigned max_ctxs;
  115. struct timer_list packet_timeout;
  116. unsigned stat_irq_thresh;
  117. struct dma_pool *req_pool;
  118. };
  119. /* Algorithm type mask. */
  120. #define SPACC_CRYPTO_ALG_MASK 0x7
  121. /* SPACC definition of a crypto algorithm. */
  122. struct spacc_alg {
  123. unsigned long ctrl_default;
  124. unsigned long type;
  125. struct crypto_alg alg;
  126. struct spacc_engine *engine;
  127. struct list_head entry;
  128. int key_offs;
  129. int iv_offs;
  130. };
  131. /* Generic context structure for any algorithm type. */
  132. struct spacc_generic_ctx {
  133. struct spacc_engine *engine;
  134. int flags;
  135. int key_offs;
  136. int iv_offs;
  137. };
  138. /* Block cipher context. */
  139. struct spacc_ablk_ctx {
  140. struct spacc_generic_ctx generic;
  141. u8 key[AES_MAX_KEY_SIZE];
  142. u8 key_len;
  143. /*
  144. * The fallback cipher. If the operation can't be done in hardware,
  145. * fallback to a software version.
  146. */
  147. struct crypto_sync_skcipher *sw_cipher;
  148. };
  149. /* AEAD cipher context. */
  150. struct spacc_aead_ctx {
  151. struct spacc_generic_ctx generic;
  152. u8 cipher_key[AES_MAX_KEY_SIZE];
  153. u8 hash_ctx[SPACC_CRYPTO_IPSEC_HASH_PG_SZ];
  154. u8 cipher_key_len;
  155. u8 hash_key_len;
  156. struct crypto_aead *sw_cipher;
  157. };
  158. static int spacc_ablk_submit(struct spacc_req *req);
  159. static inline struct spacc_alg *to_spacc_alg(struct crypto_alg *alg)
  160. {
  161. return alg ? container_of(alg, struct spacc_alg, alg) : NULL;
  162. }
  163. static inline struct spacc_aead *to_spacc_aead(struct aead_alg *alg)
  164. {
  165. return container_of(alg, struct spacc_aead, alg);
  166. }
  167. static inline int spacc_fifo_cmd_full(struct spacc_engine *engine)
  168. {
  169. u32 fifo_stat = readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET);
  170. return fifo_stat & SPA_FIFO_CMD_FULL;
  171. }
  172. /*
  173. * Given a cipher context, and a context number, get the base address of the
  174. * context page.
  175. *
  176. * Returns the address of the context page where the key/context may
  177. * be written.
  178. */
  179. static inline void __iomem *spacc_ctx_page_addr(struct spacc_generic_ctx *ctx,
  180. unsigned indx,
  181. bool is_cipher_ctx)
  182. {
  183. return is_cipher_ctx ? ctx->engine->cipher_ctx_base +
  184. (indx * ctx->engine->cipher_pg_sz) :
  185. ctx->engine->hash_key_base + (indx * ctx->engine->hash_pg_sz);
  186. }
  187. /* The context pages can only be written with 32-bit accesses. */
  188. static inline void memcpy_toio32(u32 __iomem *dst, const void *src,
  189. unsigned count)
  190. {
  191. const u32 *src32 = (const u32 *) src;
  192. while (count--)
  193. writel(*src32++, dst++);
  194. }
  195. static void spacc_cipher_write_ctx(struct spacc_generic_ctx *ctx,
  196. void __iomem *page_addr, const u8 *key,
  197. size_t key_len, const u8 *iv, size_t iv_len)
  198. {
  199. void __iomem *key_ptr = page_addr + ctx->key_offs;
  200. void __iomem *iv_ptr = page_addr + ctx->iv_offs;
  201. memcpy_toio32(key_ptr, key, key_len / 4);
  202. memcpy_toio32(iv_ptr, iv, iv_len / 4);
  203. }
  204. /*
  205. * Load a context into the engines context memory.
  206. *
  207. * Returns the index of the context page where the context was loaded.
  208. */
  209. static unsigned spacc_load_ctx(struct spacc_generic_ctx *ctx,
  210. const u8 *ciph_key, size_t ciph_len,
  211. const u8 *iv, size_t ivlen, const u8 *hash_key,
  212. size_t hash_len)
  213. {
  214. unsigned indx = ctx->engine->next_ctx++;
  215. void __iomem *ciph_page_addr, *hash_page_addr;
  216. ciph_page_addr = spacc_ctx_page_addr(ctx, indx, 1);
  217. hash_page_addr = spacc_ctx_page_addr(ctx, indx, 0);
  218. ctx->engine->next_ctx &= ctx->engine->fifo_sz - 1;
  219. spacc_cipher_write_ctx(ctx, ciph_page_addr, ciph_key, ciph_len, iv,
  220. ivlen);
  221. writel(ciph_len | (indx << SPA_KEY_SZ_CTX_INDEX_OFFSET) |
  222. (1 << SPA_KEY_SZ_CIPHER_OFFSET),
  223. ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET);
  224. if (hash_key) {
  225. memcpy_toio32(hash_page_addr, hash_key, hash_len / 4);
  226. writel(hash_len | (indx << SPA_KEY_SZ_CTX_INDEX_OFFSET),
  227. ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET);
  228. }
  229. return indx;
  230. }
  231. static inline void ddt_set(struct spacc_ddt *ddt, dma_addr_t phys, size_t len)
  232. {
  233. ddt->p = phys;
  234. ddt->len = len;
  235. }
  236. /*
  237. * Take a crypto request and scatterlists for the data and turn them into DDTs
  238. * for passing to the crypto engines. This also DMA maps the data so that the
  239. * crypto engines can DMA to/from them.
  240. */
  241. static struct spacc_ddt *spacc_sg_to_ddt(struct spacc_engine *engine,
  242. struct scatterlist *payload,
  243. unsigned nbytes,
  244. enum dma_data_direction dir,
  245. dma_addr_t *ddt_phys)
  246. {
  247. unsigned mapped_ents;
  248. struct scatterlist *cur;
  249. struct spacc_ddt *ddt;
  250. int i;
  251. int nents;
  252. nents = sg_nents_for_len(payload, nbytes);
  253. if (nents < 0) {
  254. dev_err(engine->dev, "Invalid numbers of SG.\n");
  255. return NULL;
  256. }
  257. mapped_ents = dma_map_sg(engine->dev, payload, nents, dir);
  258. if (mapped_ents + 1 > MAX_DDT_LEN)
  259. goto out;
  260. ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, ddt_phys);
  261. if (!ddt)
  262. goto out;
  263. for_each_sg(payload, cur, mapped_ents, i)
  264. ddt_set(&ddt[i], sg_dma_address(cur), sg_dma_len(cur));
  265. ddt_set(&ddt[mapped_ents], 0, 0);
  266. return ddt;
  267. out:
  268. dma_unmap_sg(engine->dev, payload, nents, dir);
  269. return NULL;
  270. }
  271. static int spacc_aead_make_ddts(struct aead_request *areq)
  272. {
  273. struct crypto_aead *aead = crypto_aead_reqtfm(areq);
  274. struct spacc_req *req = aead_request_ctx(areq);
  275. struct spacc_engine *engine = req->engine;
  276. struct spacc_ddt *src_ddt, *dst_ddt;
  277. unsigned total;
  278. int src_nents, dst_nents;
  279. struct scatterlist *cur;
  280. int i, dst_ents, src_ents;
  281. total = areq->assoclen + areq->cryptlen;
  282. if (req->is_encrypt)
  283. total += crypto_aead_authsize(aead);
  284. src_nents = sg_nents_for_len(areq->src, total);
  285. if (src_nents < 0) {
  286. dev_err(engine->dev, "Invalid numbers of src SG.\n");
  287. return src_nents;
  288. }
  289. if (src_nents + 1 > MAX_DDT_LEN)
  290. return -E2BIG;
  291. dst_nents = 0;
  292. if (areq->src != areq->dst) {
  293. dst_nents = sg_nents_for_len(areq->dst, total);
  294. if (dst_nents < 0) {
  295. dev_err(engine->dev, "Invalid numbers of dst SG.\n");
  296. return dst_nents;
  297. }
  298. if (src_nents + 1 > MAX_DDT_LEN)
  299. return -E2BIG;
  300. }
  301. src_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->src_addr);
  302. if (!src_ddt)
  303. goto err;
  304. dst_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->dst_addr);
  305. if (!dst_ddt)
  306. goto err_free_src;
  307. req->src_ddt = src_ddt;
  308. req->dst_ddt = dst_ddt;
  309. if (dst_nents) {
  310. src_ents = dma_map_sg(engine->dev, areq->src, src_nents,
  311. DMA_TO_DEVICE);
  312. if (!src_ents)
  313. goto err_free_dst;
  314. dst_ents = dma_map_sg(engine->dev, areq->dst, dst_nents,
  315. DMA_FROM_DEVICE);
  316. if (!dst_ents) {
  317. dma_unmap_sg(engine->dev, areq->src, src_nents,
  318. DMA_TO_DEVICE);
  319. goto err_free_dst;
  320. }
  321. } else {
  322. src_ents = dma_map_sg(engine->dev, areq->src, src_nents,
  323. DMA_BIDIRECTIONAL);
  324. if (!src_ents)
  325. goto err_free_dst;
  326. dst_ents = src_ents;
  327. }
  328. /*
  329. * Now map in the payload for the source and destination and terminate
  330. * with the NULL pointers.
  331. */
  332. for_each_sg(areq->src, cur, src_ents, i)
  333. ddt_set(src_ddt++, sg_dma_address(cur), sg_dma_len(cur));
  334. /* For decryption we need to skip the associated data. */
  335. total = req->is_encrypt ? 0 : areq->assoclen;
  336. for_each_sg(areq->dst, cur, dst_ents, i) {
  337. unsigned len = sg_dma_len(cur);
  338. if (len <= total) {
  339. total -= len;
  340. continue;
  341. }
  342. ddt_set(dst_ddt++, sg_dma_address(cur) + total, len - total);
  343. }
  344. ddt_set(src_ddt, 0, 0);
  345. ddt_set(dst_ddt, 0, 0);
  346. return 0;
  347. err_free_dst:
  348. dma_pool_free(engine->req_pool, dst_ddt, req->dst_addr);
  349. err_free_src:
  350. dma_pool_free(engine->req_pool, src_ddt, req->src_addr);
  351. err:
  352. return -ENOMEM;
  353. }
  354. static void spacc_aead_free_ddts(struct spacc_req *req)
  355. {
  356. struct aead_request *areq = container_of(req->req, struct aead_request,
  357. base);
  358. struct crypto_aead *aead = crypto_aead_reqtfm(areq);
  359. unsigned total = areq->assoclen + areq->cryptlen +
  360. (req->is_encrypt ? crypto_aead_authsize(aead) : 0);
  361. struct spacc_aead_ctx *aead_ctx = crypto_aead_ctx(aead);
  362. struct spacc_engine *engine = aead_ctx->generic.engine;
  363. int nents = sg_nents_for_len(areq->src, total);
  364. /* sg_nents_for_len should not fail since it works when mapping sg */
  365. if (unlikely(nents < 0)) {
  366. dev_err(engine->dev, "Invalid numbers of src SG.\n");
  367. return;
  368. }
  369. if (areq->src != areq->dst) {
  370. dma_unmap_sg(engine->dev, areq->src, nents, DMA_TO_DEVICE);
  371. nents = sg_nents_for_len(areq->dst, total);
  372. if (unlikely(nents < 0)) {
  373. dev_err(engine->dev, "Invalid numbers of dst SG.\n");
  374. return;
  375. }
  376. dma_unmap_sg(engine->dev, areq->dst, nents, DMA_FROM_DEVICE);
  377. } else
  378. dma_unmap_sg(engine->dev, areq->src, nents, DMA_BIDIRECTIONAL);
  379. dma_pool_free(engine->req_pool, req->src_ddt, req->src_addr);
  380. dma_pool_free(engine->req_pool, req->dst_ddt, req->dst_addr);
  381. }
  382. static void spacc_free_ddt(struct spacc_req *req, struct spacc_ddt *ddt,
  383. dma_addr_t ddt_addr, struct scatterlist *payload,
  384. unsigned nbytes, enum dma_data_direction dir)
  385. {
  386. int nents = sg_nents_for_len(payload, nbytes);
  387. if (nents < 0) {
  388. dev_err(req->engine->dev, "Invalid numbers of SG.\n");
  389. return;
  390. }
  391. dma_unmap_sg(req->engine->dev, payload, nents, dir);
  392. dma_pool_free(req->engine->req_pool, ddt, ddt_addr);
  393. }
  394. static int spacc_aead_setkey(struct crypto_aead *tfm, const u8 *key,
  395. unsigned int keylen)
  396. {
  397. struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm);
  398. struct crypto_authenc_keys keys;
  399. int err;
  400. crypto_aead_clear_flags(ctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
  401. crypto_aead_set_flags(ctx->sw_cipher, crypto_aead_get_flags(tfm) &
  402. CRYPTO_TFM_REQ_MASK);
  403. err = crypto_aead_setkey(ctx->sw_cipher, key, keylen);
  404. crypto_aead_clear_flags(tfm, CRYPTO_TFM_RES_MASK);
  405. crypto_aead_set_flags(tfm, crypto_aead_get_flags(ctx->sw_cipher) &
  406. CRYPTO_TFM_RES_MASK);
  407. if (err)
  408. return err;
  409. if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
  410. goto badkey;
  411. if (keys.enckeylen > AES_MAX_KEY_SIZE)
  412. goto badkey;
  413. if (keys.authkeylen > sizeof(ctx->hash_ctx))
  414. goto badkey;
  415. memcpy(ctx->cipher_key, keys.enckey, keys.enckeylen);
  416. ctx->cipher_key_len = keys.enckeylen;
  417. memcpy(ctx->hash_ctx, keys.authkey, keys.authkeylen);
  418. ctx->hash_key_len = keys.authkeylen;
  419. memzero_explicit(&keys, sizeof(keys));
  420. return 0;
  421. badkey:
  422. crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
  423. memzero_explicit(&keys, sizeof(keys));
  424. return -EINVAL;
  425. }
  426. static int spacc_aead_setauthsize(struct crypto_aead *tfm,
  427. unsigned int authsize)
  428. {
  429. struct spacc_aead_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm));
  430. return crypto_aead_setauthsize(ctx->sw_cipher, authsize);
  431. }
  432. /*
  433. * Check if an AEAD request requires a fallback operation. Some requests can't
  434. * be completed in hardware because the hardware may not support certain key
  435. * sizes. In these cases we need to complete the request in software.
  436. */
  437. static int spacc_aead_need_fallback(struct aead_request *aead_req)
  438. {
  439. struct crypto_aead *aead = crypto_aead_reqtfm(aead_req);
  440. struct aead_alg *alg = crypto_aead_alg(aead);
  441. struct spacc_aead *spacc_alg = to_spacc_aead(alg);
  442. struct spacc_aead_ctx *ctx = crypto_aead_ctx(aead);
  443. /*
  444. * If we have a non-supported key-length, then we need to do a
  445. * software fallback.
  446. */
  447. if ((spacc_alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) ==
  448. SPA_CTRL_CIPH_ALG_AES &&
  449. ctx->cipher_key_len != AES_KEYSIZE_128 &&
  450. ctx->cipher_key_len != AES_KEYSIZE_256)
  451. return 1;
  452. return 0;
  453. }
  454. static int spacc_aead_do_fallback(struct aead_request *req, unsigned alg_type,
  455. bool is_encrypt)
  456. {
  457. struct crypto_tfm *old_tfm = crypto_aead_tfm(crypto_aead_reqtfm(req));
  458. struct spacc_aead_ctx *ctx = crypto_tfm_ctx(old_tfm);
  459. struct aead_request *subreq = aead_request_ctx(req);
  460. aead_request_set_tfm(subreq, ctx->sw_cipher);
  461. aead_request_set_callback(subreq, req->base.flags,
  462. req->base.complete, req->base.data);
  463. aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
  464. req->iv);
  465. aead_request_set_ad(subreq, req->assoclen);
  466. return is_encrypt ? crypto_aead_encrypt(subreq) :
  467. crypto_aead_decrypt(subreq);
  468. }
  469. static void spacc_aead_complete(struct spacc_req *req)
  470. {
  471. spacc_aead_free_ddts(req);
  472. req->req->complete(req->req, req->result);
  473. }
  474. static int spacc_aead_submit(struct spacc_req *req)
  475. {
  476. struct aead_request *aead_req =
  477. container_of(req->req, struct aead_request, base);
  478. struct crypto_aead *aead = crypto_aead_reqtfm(aead_req);
  479. unsigned int authsize = crypto_aead_authsize(aead);
  480. struct spacc_aead_ctx *ctx = crypto_aead_ctx(aead);
  481. struct aead_alg *alg = crypto_aead_alg(aead);
  482. struct spacc_aead *spacc_alg = to_spacc_aead(alg);
  483. struct spacc_engine *engine = ctx->generic.engine;
  484. u32 ctrl, proc_len, assoc_len;
  485. req->result = -EINPROGRESS;
  486. req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->cipher_key,
  487. ctx->cipher_key_len, aead_req->iv, crypto_aead_ivsize(aead),
  488. ctx->hash_ctx, ctx->hash_key_len);
  489. /* Set the source and destination DDT pointers. */
  490. writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET);
  491. writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET);
  492. writel(0, engine->regs + SPA_OFFSET_REG_OFFSET);
  493. assoc_len = aead_req->assoclen;
  494. proc_len = aead_req->cryptlen + assoc_len;
  495. /*
  496. * If we are decrypting, we need to take the length of the ICV out of
  497. * the processing length.
  498. */
  499. if (!req->is_encrypt)
  500. proc_len -= authsize;
  501. writel(proc_len, engine->regs + SPA_PROC_LEN_REG_OFFSET);
  502. writel(assoc_len, engine->regs + SPA_AAD_LEN_REG_OFFSET);
  503. writel(authsize, engine->regs + SPA_ICV_LEN_REG_OFFSET);
  504. writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET);
  505. writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET);
  506. ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) |
  507. (1 << SPA_CTRL_ICV_APPEND);
  508. if (req->is_encrypt)
  509. ctrl |= (1 << SPA_CTRL_ENCRYPT_IDX) | (1 << SPA_CTRL_AAD_COPY);
  510. else
  511. ctrl |= (1 << SPA_CTRL_KEY_EXP);
  512. mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);
  513. writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET);
  514. return -EINPROGRESS;
  515. }
  516. static int spacc_req_submit(struct spacc_req *req);
  517. static void spacc_push(struct spacc_engine *engine)
  518. {
  519. struct spacc_req *req;
  520. while (!list_empty(&engine->pending) &&
  521. engine->in_flight + 1 <= engine->fifo_sz) {
  522. ++engine->in_flight;
  523. req = list_first_entry(&engine->pending, struct spacc_req,
  524. list);
  525. list_move_tail(&req->list, &engine->in_progress);
  526. req->result = spacc_req_submit(req);
  527. }
  528. }
  529. /*
  530. * Setup an AEAD request for processing. This will configure the engine, load
  531. * the context and then start the packet processing.
  532. */
  533. static int spacc_aead_setup(struct aead_request *req,
  534. unsigned alg_type, bool is_encrypt)
  535. {
  536. struct crypto_aead *aead = crypto_aead_reqtfm(req);
  537. struct aead_alg *alg = crypto_aead_alg(aead);
  538. struct spacc_engine *engine = to_spacc_aead(alg)->engine;
  539. struct spacc_req *dev_req = aead_request_ctx(req);
  540. int err;
  541. unsigned long flags;
  542. dev_req->req = &req->base;
  543. dev_req->is_encrypt = is_encrypt;
  544. dev_req->result = -EBUSY;
  545. dev_req->engine = engine;
  546. dev_req->complete = spacc_aead_complete;
  547. if (unlikely(spacc_aead_need_fallback(req) ||
  548. ((err = spacc_aead_make_ddts(req)) == -E2BIG)))
  549. return spacc_aead_do_fallback(req, alg_type, is_encrypt);
  550. if (err)
  551. goto out;
  552. err = -EINPROGRESS;
  553. spin_lock_irqsave(&engine->hw_lock, flags);
  554. if (unlikely(spacc_fifo_cmd_full(engine)) ||
  555. engine->in_flight + 1 > engine->fifo_sz) {
  556. if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
  557. err = -EBUSY;
  558. spin_unlock_irqrestore(&engine->hw_lock, flags);
  559. goto out_free_ddts;
  560. }
  561. list_add_tail(&dev_req->list, &engine->pending);
  562. } else {
  563. list_add_tail(&dev_req->list, &engine->pending);
  564. spacc_push(engine);
  565. }
  566. spin_unlock_irqrestore(&engine->hw_lock, flags);
  567. goto out;
  568. out_free_ddts:
  569. spacc_aead_free_ddts(dev_req);
  570. out:
  571. return err;
  572. }
  573. static int spacc_aead_encrypt(struct aead_request *req)
  574. {
  575. struct crypto_aead *aead = crypto_aead_reqtfm(req);
  576. struct spacc_aead *alg = to_spacc_aead(crypto_aead_alg(aead));
  577. return spacc_aead_setup(req, alg->type, 1);
  578. }
  579. static int spacc_aead_decrypt(struct aead_request *req)
  580. {
  581. struct crypto_aead *aead = crypto_aead_reqtfm(req);
  582. struct spacc_aead *alg = to_spacc_aead(crypto_aead_alg(aead));
  583. return spacc_aead_setup(req, alg->type, 0);
  584. }
  585. /*
  586. * Initialise a new AEAD context. This is responsible for allocating the
  587. * fallback cipher and initialising the context.
  588. */
  589. static int spacc_aead_cra_init(struct crypto_aead *tfm)
  590. {
  591. struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm);
  592. struct aead_alg *alg = crypto_aead_alg(tfm);
  593. struct spacc_aead *spacc_alg = to_spacc_aead(alg);
  594. struct spacc_engine *engine = spacc_alg->engine;
  595. ctx->generic.flags = spacc_alg->type;
  596. ctx->generic.engine = engine;
  597. ctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0,
  598. CRYPTO_ALG_NEED_FALLBACK);
  599. if (IS_ERR(ctx->sw_cipher))
  600. return PTR_ERR(ctx->sw_cipher);
  601. ctx->generic.key_offs = spacc_alg->key_offs;
  602. ctx->generic.iv_offs = spacc_alg->iv_offs;
  603. crypto_aead_set_reqsize(
  604. tfm,
  605. max(sizeof(struct spacc_req),
  606. sizeof(struct aead_request) +
  607. crypto_aead_reqsize(ctx->sw_cipher)));
  608. return 0;
  609. }
  610. /*
  611. * Destructor for an AEAD context. This is called when the transform is freed
  612. * and must free the fallback cipher.
  613. */
  614. static void spacc_aead_cra_exit(struct crypto_aead *tfm)
  615. {
  616. struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm);
  617. crypto_free_aead(ctx->sw_cipher);
  618. }
  619. /*
  620. * Set the DES key for a block cipher transform. This also performs weak key
  621. * checking if the transform has requested it.
  622. */
  623. static int spacc_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
  624. unsigned int len)
  625. {
  626. struct spacc_ablk_ctx *ctx = crypto_ablkcipher_ctx(cipher);
  627. int err;
  628. err = verify_ablkcipher_des_key(cipher, key);
  629. if (err)
  630. return err;
  631. memcpy(ctx->key, key, len);
  632. ctx->key_len = len;
  633. return 0;
  634. }
  635. /*
  636. * Set the 3DES key for a block cipher transform. This also performs weak key
  637. * checking if the transform has requested it.
  638. */
  639. static int spacc_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
  640. unsigned int len)
  641. {
  642. struct spacc_ablk_ctx *ctx = crypto_ablkcipher_ctx(cipher);
  643. int err;
  644. err = verify_ablkcipher_des3_key(cipher, key);
  645. if (err)
  646. return err;
  647. memcpy(ctx->key, key, len);
  648. ctx->key_len = len;
  649. return 0;
  650. }
  651. /*
  652. * Set the key for an AES block cipher. Some key lengths are not supported in
  653. * hardware so this must also check whether a fallback is needed.
  654. */
  655. static int spacc_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
  656. unsigned int len)
  657. {
  658. struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
  659. struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
  660. int err = 0;
  661. if (len > AES_MAX_KEY_SIZE) {
  662. crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
  663. return -EINVAL;
  664. }
  665. /*
  666. * IPSec engine only supports 128 and 256 bit AES keys. If we get a
  667. * request for any other size (192 bits) then we need to do a software
  668. * fallback.
  669. */
  670. if (len != AES_KEYSIZE_128 && len != AES_KEYSIZE_256) {
  671. if (!ctx->sw_cipher)
  672. return -EINVAL;
  673. /*
  674. * Set the fallback transform to use the same request flags as
  675. * the hardware transform.
  676. */
  677. crypto_sync_skcipher_clear_flags(ctx->sw_cipher,
  678. CRYPTO_TFM_REQ_MASK);
  679. crypto_sync_skcipher_set_flags(ctx->sw_cipher,
  680. cipher->base.crt_flags &
  681. CRYPTO_TFM_REQ_MASK);
  682. err = crypto_sync_skcipher_setkey(ctx->sw_cipher, key, len);
  683. tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
  684. tfm->crt_flags |=
  685. crypto_sync_skcipher_get_flags(ctx->sw_cipher) &
  686. CRYPTO_TFM_RES_MASK;
  687. if (err)
  688. goto sw_setkey_failed;
  689. }
  690. memcpy(ctx->key, key, len);
  691. ctx->key_len = len;
  692. sw_setkey_failed:
  693. return err;
  694. }
  695. static int spacc_kasumi_f8_setkey(struct crypto_ablkcipher *cipher,
  696. const u8 *key, unsigned int len)
  697. {
  698. struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
  699. struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
  700. int err = 0;
  701. if (len > AES_MAX_KEY_SIZE) {
  702. crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
  703. err = -EINVAL;
  704. goto out;
  705. }
  706. memcpy(ctx->key, key, len);
  707. ctx->key_len = len;
  708. out:
  709. return err;
  710. }
  711. static int spacc_ablk_need_fallback(struct spacc_req *req)
  712. {
  713. struct spacc_ablk_ctx *ctx;
  714. struct crypto_tfm *tfm = req->req->tfm;
  715. struct crypto_alg *alg = req->req->tfm->__crt_alg;
  716. struct spacc_alg *spacc_alg = to_spacc_alg(alg);
  717. ctx = crypto_tfm_ctx(tfm);
  718. return (spacc_alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) ==
  719. SPA_CTRL_CIPH_ALG_AES &&
  720. ctx->key_len != AES_KEYSIZE_128 &&
  721. ctx->key_len != AES_KEYSIZE_256;
  722. }
  723. static void spacc_ablk_complete(struct spacc_req *req)
  724. {
  725. struct ablkcipher_request *ablk_req = ablkcipher_request_cast(req->req);
  726. if (ablk_req->src != ablk_req->dst) {
  727. spacc_free_ddt(req, req->src_ddt, req->src_addr, ablk_req->src,
  728. ablk_req->nbytes, DMA_TO_DEVICE);
  729. spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst,
  730. ablk_req->nbytes, DMA_FROM_DEVICE);
  731. } else
  732. spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst,
  733. ablk_req->nbytes, DMA_BIDIRECTIONAL);
  734. req->req->complete(req->req, req->result);
  735. }
  736. static int spacc_ablk_submit(struct spacc_req *req)
  737. {
  738. struct crypto_tfm *tfm = req->req->tfm;
  739. struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
  740. struct ablkcipher_request *ablk_req = ablkcipher_request_cast(req->req);
  741. struct crypto_alg *alg = req->req->tfm->__crt_alg;
  742. struct spacc_alg *spacc_alg = to_spacc_alg(alg);
  743. struct spacc_engine *engine = ctx->generic.engine;
  744. u32 ctrl;
  745. req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->key,
  746. ctx->key_len, ablk_req->info, alg->cra_ablkcipher.ivsize,
  747. NULL, 0);
  748. writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET);
  749. writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET);
  750. writel(0, engine->regs + SPA_OFFSET_REG_OFFSET);
  751. writel(ablk_req->nbytes, engine->regs + SPA_PROC_LEN_REG_OFFSET);
  752. writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET);
  753. writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET);
  754. writel(0, engine->regs + SPA_AAD_LEN_REG_OFFSET);
  755. ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) |
  756. (req->is_encrypt ? (1 << SPA_CTRL_ENCRYPT_IDX) :
  757. (1 << SPA_CTRL_KEY_EXP));
  758. mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);
  759. writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET);
  760. return -EINPROGRESS;
  761. }
  762. static int spacc_ablk_do_fallback(struct ablkcipher_request *req,
  763. unsigned alg_type, bool is_encrypt)
  764. {
  765. struct crypto_tfm *old_tfm =
  766. crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
  767. struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(old_tfm);
  768. SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->sw_cipher);
  769. int err;
  770. /*
  771. * Change the request to use the software fallback transform, and once
  772. * the ciphering has completed, put the old transform back into the
  773. * request.
  774. */
  775. skcipher_request_set_sync_tfm(subreq, ctx->sw_cipher);
  776. skcipher_request_set_callback(subreq, req->base.flags, NULL, NULL);
  777. skcipher_request_set_crypt(subreq, req->src, req->dst,
  778. req->nbytes, req->info);
  779. err = is_encrypt ? crypto_skcipher_encrypt(subreq) :
  780. crypto_skcipher_decrypt(subreq);
  781. skcipher_request_zero(subreq);
  782. return err;
  783. }
  784. static int spacc_ablk_setup(struct ablkcipher_request *req, unsigned alg_type,
  785. bool is_encrypt)
  786. {
  787. struct crypto_alg *alg = req->base.tfm->__crt_alg;
  788. struct spacc_engine *engine = to_spacc_alg(alg)->engine;
  789. struct spacc_req *dev_req = ablkcipher_request_ctx(req);
  790. unsigned long flags;
  791. int err = -ENOMEM;
  792. dev_req->req = &req->base;
  793. dev_req->is_encrypt = is_encrypt;
  794. dev_req->engine = engine;
  795. dev_req->complete = spacc_ablk_complete;
  796. dev_req->result = -EINPROGRESS;
  797. if (unlikely(spacc_ablk_need_fallback(dev_req)))
  798. return spacc_ablk_do_fallback(req, alg_type, is_encrypt);
  799. /*
  800. * Create the DDT's for the engine. If we share the same source and
  801. * destination then we can optimize by reusing the DDT's.
  802. */
  803. if (req->src != req->dst) {
  804. dev_req->src_ddt = spacc_sg_to_ddt(engine, req->src,
  805. req->nbytes, DMA_TO_DEVICE, &dev_req->src_addr);
  806. if (!dev_req->src_ddt)
  807. goto out;
  808. dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst,
  809. req->nbytes, DMA_FROM_DEVICE, &dev_req->dst_addr);
  810. if (!dev_req->dst_ddt)
  811. goto out_free_src;
  812. } else {
  813. dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst,
  814. req->nbytes, DMA_BIDIRECTIONAL, &dev_req->dst_addr);
  815. if (!dev_req->dst_ddt)
  816. goto out;
  817. dev_req->src_ddt = NULL;
  818. dev_req->src_addr = dev_req->dst_addr;
  819. }
  820. err = -EINPROGRESS;
  821. spin_lock_irqsave(&engine->hw_lock, flags);
  822. /*
  823. * Check if the engine will accept the operation now. If it won't then
  824. * we either stick it on the end of a pending list if we can backlog,
  825. * or bailout with an error if not.
  826. */
  827. if (unlikely(spacc_fifo_cmd_full(engine)) ||
  828. engine->in_flight + 1 > engine->fifo_sz) {
  829. if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
  830. err = -EBUSY;
  831. spin_unlock_irqrestore(&engine->hw_lock, flags);
  832. goto out_free_ddts;
  833. }
  834. list_add_tail(&dev_req->list, &engine->pending);
  835. } else {
  836. list_add_tail(&dev_req->list, &engine->pending);
  837. spacc_push(engine);
  838. }
  839. spin_unlock_irqrestore(&engine->hw_lock, flags);
  840. goto out;
  841. out_free_ddts:
  842. spacc_free_ddt(dev_req, dev_req->dst_ddt, dev_req->dst_addr, req->dst,
  843. req->nbytes, req->src == req->dst ?
  844. DMA_BIDIRECTIONAL : DMA_FROM_DEVICE);
  845. out_free_src:
  846. if (req->src != req->dst)
  847. spacc_free_ddt(dev_req, dev_req->src_ddt, dev_req->src_addr,
  848. req->src, req->nbytes, DMA_TO_DEVICE);
  849. out:
  850. return err;
  851. }
  852. static int spacc_ablk_cra_init(struct crypto_tfm *tfm)
  853. {
  854. struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
  855. struct crypto_alg *alg = tfm->__crt_alg;
  856. struct spacc_alg *spacc_alg = to_spacc_alg(alg);
  857. struct spacc_engine *engine = spacc_alg->engine;
  858. ctx->generic.flags = spacc_alg->type;
  859. ctx->generic.engine = engine;
  860. if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
  861. ctx->sw_cipher = crypto_alloc_sync_skcipher(
  862. alg->cra_name, 0, CRYPTO_ALG_NEED_FALLBACK);
  863. if (IS_ERR(ctx->sw_cipher)) {
  864. dev_warn(engine->dev, "failed to allocate fallback for %s\n",
  865. alg->cra_name);
  866. return PTR_ERR(ctx->sw_cipher);
  867. }
  868. }
  869. ctx->generic.key_offs = spacc_alg->key_offs;
  870. ctx->generic.iv_offs = spacc_alg->iv_offs;
  871. tfm->crt_ablkcipher.reqsize = sizeof(struct spacc_req);
  872. return 0;
  873. }
  874. static void spacc_ablk_cra_exit(struct crypto_tfm *tfm)
  875. {
  876. struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
  877. crypto_free_sync_skcipher(ctx->sw_cipher);
  878. }
  879. static int spacc_ablk_encrypt(struct ablkcipher_request *req)
  880. {
  881. struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req);
  882. struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
  883. struct spacc_alg *alg = to_spacc_alg(tfm->__crt_alg);
  884. return spacc_ablk_setup(req, alg->type, 1);
  885. }
  886. static int spacc_ablk_decrypt(struct ablkcipher_request *req)
  887. {
  888. struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req);
  889. struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
  890. struct spacc_alg *alg = to_spacc_alg(tfm->__crt_alg);
  891. return spacc_ablk_setup(req, alg->type, 0);
  892. }
  893. static inline int spacc_fifo_stat_empty(struct spacc_engine *engine)
  894. {
  895. return readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET) &
  896. SPA_FIFO_STAT_EMPTY;
  897. }
  898. static void spacc_process_done(struct spacc_engine *engine)
  899. {
  900. struct spacc_req *req;
  901. unsigned long flags;
  902. spin_lock_irqsave(&engine->hw_lock, flags);
  903. while (!spacc_fifo_stat_empty(engine)) {
  904. req = list_first_entry(&engine->in_progress, struct spacc_req,
  905. list);
  906. list_move_tail(&req->list, &engine->completed);
  907. --engine->in_flight;
  908. /* POP the status register. */
  909. writel(~0, engine->regs + SPA_STAT_POP_REG_OFFSET);
  910. req->result = (readl(engine->regs + SPA_STATUS_REG_OFFSET) &
  911. SPA_STATUS_RES_CODE_MASK) >> SPA_STATUS_RES_CODE_OFFSET;
  912. /*
  913. * Convert the SPAcc error status into the standard POSIX error
  914. * codes.
  915. */
  916. if (unlikely(req->result)) {
  917. switch (req->result) {
  918. case SPA_STATUS_ICV_FAIL:
  919. req->result = -EBADMSG;
  920. break;
  921. case SPA_STATUS_MEMORY_ERROR:
  922. dev_warn(engine->dev,
  923. "memory error triggered\n");
  924. req->result = -EFAULT;
  925. break;
  926. case SPA_STATUS_BLOCK_ERROR:
  927. dev_warn(engine->dev,
  928. "block error triggered\n");
  929. req->result = -EIO;
  930. break;
  931. }
  932. }
  933. }
  934. tasklet_schedule(&engine->complete);
  935. spin_unlock_irqrestore(&engine->hw_lock, flags);
  936. }
  937. static irqreturn_t spacc_spacc_irq(int irq, void *dev)
  938. {
  939. struct spacc_engine *engine = (struct spacc_engine *)dev;
  940. u32 spacc_irq_stat = readl(engine->regs + SPA_IRQ_STAT_REG_OFFSET);
  941. writel(spacc_irq_stat, engine->regs + SPA_IRQ_STAT_REG_OFFSET);
  942. spacc_process_done(engine);
  943. return IRQ_HANDLED;
  944. }
  945. static void spacc_packet_timeout(struct timer_list *t)
  946. {
  947. struct spacc_engine *engine = from_timer(engine, t, packet_timeout);
  948. spacc_process_done(engine);
  949. }
  950. static int spacc_req_submit(struct spacc_req *req)
  951. {
  952. struct crypto_alg *alg = req->req->tfm->__crt_alg;
  953. if (CRYPTO_ALG_TYPE_AEAD == (CRYPTO_ALG_TYPE_MASK & alg->cra_flags))
  954. return spacc_aead_submit(req);
  955. else
  956. return spacc_ablk_submit(req);
  957. }
  958. static void spacc_spacc_complete(unsigned long data)
  959. {
  960. struct spacc_engine *engine = (struct spacc_engine *)data;
  961. struct spacc_req *req, *tmp;
  962. unsigned long flags;
  963. LIST_HEAD(completed);
  964. spin_lock_irqsave(&engine->hw_lock, flags);
  965. list_splice_init(&engine->completed, &completed);
  966. spacc_push(engine);
  967. if (engine->in_flight)
  968. mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);
  969. spin_unlock_irqrestore(&engine->hw_lock, flags);
  970. list_for_each_entry_safe(req, tmp, &completed, list) {
  971. list_del(&req->list);
  972. req->complete(req);
  973. }
  974. }
  975. #ifdef CONFIG_PM
  976. static int spacc_suspend(struct device *dev)
  977. {
  978. struct spacc_engine *engine = dev_get_drvdata(dev);
  979. /*
  980. * We only support standby mode. All we have to do is gate the clock to
  981. * the spacc. The hardware will preserve state until we turn it back
  982. * on again.
  983. */
  984. clk_disable(engine->clk);
  985. return 0;
  986. }
  987. static int spacc_resume(struct device *dev)
  988. {
  989. struct spacc_engine *engine = dev_get_drvdata(dev);
  990. return clk_enable(engine->clk);
  991. }
  992. static const struct dev_pm_ops spacc_pm_ops = {
  993. .suspend = spacc_suspend,
  994. .resume = spacc_resume,
  995. };
  996. #endif /* CONFIG_PM */
  997. static inline struct spacc_engine *spacc_dev_to_engine(struct device *dev)
  998. {
  999. return dev ? dev_get_drvdata(dev) : NULL;
  1000. }
  1001. static ssize_t spacc_stat_irq_thresh_show(struct device *dev,
  1002. struct device_attribute *attr,
  1003. char *buf)
  1004. {
  1005. struct spacc_engine *engine = spacc_dev_to_engine(dev);
  1006. return snprintf(buf, PAGE_SIZE, "%u\n", engine->stat_irq_thresh);
  1007. }
  1008. static ssize_t spacc_stat_irq_thresh_store(struct device *dev,
  1009. struct device_attribute *attr,
  1010. const char *buf, size_t len)
  1011. {
  1012. struct spacc_engine *engine = spacc_dev_to_engine(dev);
  1013. unsigned long thresh;
  1014. if (kstrtoul(buf, 0, &thresh))
  1015. return -EINVAL;
  1016. thresh = clamp(thresh, 1UL, engine->fifo_sz - 1);
  1017. engine->stat_irq_thresh = thresh;
  1018. writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET,
  1019. engine->regs + SPA_IRQ_CTRL_REG_OFFSET);
  1020. return len;
  1021. }
  1022. static DEVICE_ATTR(stat_irq_thresh, 0644, spacc_stat_irq_thresh_show,
  1023. spacc_stat_irq_thresh_store);
  1024. static struct spacc_alg ipsec_engine_algs[] = {
  1025. {
  1026. .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC,
  1027. .key_offs = 0,
  1028. .iv_offs = AES_MAX_KEY_SIZE,
  1029. .alg = {
  1030. .cra_name = "cbc(aes)",
  1031. .cra_driver_name = "cbc-aes-picoxcell",
  1032. .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
  1033. .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
  1034. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1035. CRYPTO_ALG_ASYNC |
  1036. CRYPTO_ALG_NEED_FALLBACK,
  1037. .cra_blocksize = AES_BLOCK_SIZE,
  1038. .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
  1039. .cra_type = &crypto_ablkcipher_type,
  1040. .cra_module = THIS_MODULE,
  1041. .cra_ablkcipher = {
  1042. .setkey = spacc_aes_setkey,
  1043. .encrypt = spacc_ablk_encrypt,
  1044. .decrypt = spacc_ablk_decrypt,
  1045. .min_keysize = AES_MIN_KEY_SIZE,
  1046. .max_keysize = AES_MAX_KEY_SIZE,
  1047. .ivsize = AES_BLOCK_SIZE,
  1048. },
  1049. .cra_init = spacc_ablk_cra_init,
  1050. .cra_exit = spacc_ablk_cra_exit,
  1051. },
  1052. },
  1053. {
  1054. .key_offs = 0,
  1055. .iv_offs = AES_MAX_KEY_SIZE,
  1056. .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_ECB,
  1057. .alg = {
  1058. .cra_name = "ecb(aes)",
  1059. .cra_driver_name = "ecb-aes-picoxcell",
  1060. .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
  1061. .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
  1062. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1063. CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
  1064. .cra_blocksize = AES_BLOCK_SIZE,
  1065. .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
  1066. .cra_type = &crypto_ablkcipher_type,
  1067. .cra_module = THIS_MODULE,
  1068. .cra_ablkcipher = {
  1069. .setkey = spacc_aes_setkey,
  1070. .encrypt = spacc_ablk_encrypt,
  1071. .decrypt = spacc_ablk_decrypt,
  1072. .min_keysize = AES_MIN_KEY_SIZE,
  1073. .max_keysize = AES_MAX_KEY_SIZE,
  1074. },
  1075. .cra_init = spacc_ablk_cra_init,
  1076. .cra_exit = spacc_ablk_cra_exit,
  1077. },
  1078. },
  1079. {
  1080. .key_offs = DES_BLOCK_SIZE,
  1081. .iv_offs = 0,
  1082. .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC,
  1083. .alg = {
  1084. .cra_name = "cbc(des)",
  1085. .cra_driver_name = "cbc-des-picoxcell",
  1086. .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
  1087. .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
  1088. CRYPTO_ALG_ASYNC |
  1089. CRYPTO_ALG_KERN_DRIVER_ONLY,
  1090. .cra_blocksize = DES_BLOCK_SIZE,
  1091. .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
  1092. .cra_type = &crypto_ablkcipher_type,
  1093. .cra_module = THIS_MODULE,
  1094. .cra_ablkcipher = {
  1095. .setkey = spacc_des_setkey,
  1096. .encrypt = spacc_ablk_encrypt,
  1097. .decrypt = spacc_ablk_decrypt,
  1098. .min_keysize = DES_KEY_SIZE,
  1099. .max_keysize = DES_KEY_SIZE,
  1100. .ivsize = DES_BLOCK_SIZE,
  1101. },
  1102. .cra_init = spacc_ablk_cra_init,
  1103. .cra_exit = spacc_ablk_cra_exit,
  1104. },
  1105. },
  1106. {
  1107. .key_offs = DES_BLOCK_SIZE,
  1108. .iv_offs = 0,
  1109. .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_ECB,
  1110. .alg = {
  1111. .cra_name = "ecb(des)",
  1112. .cra_driver_name = "ecb-des-picoxcell",
  1113. .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
  1114. .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
  1115. CRYPTO_ALG_ASYNC |
  1116. CRYPTO_ALG_KERN_DRIVER_ONLY,
  1117. .cra_blocksize = DES_BLOCK_SIZE,
  1118. .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
  1119. .cra_type = &crypto_ablkcipher_type,
  1120. .cra_module = THIS_MODULE,
  1121. .cra_ablkcipher = {
  1122. .setkey = spacc_des_setkey,
  1123. .encrypt = spacc_ablk_encrypt,
  1124. .decrypt = spacc_ablk_decrypt,
  1125. .min_keysize = DES_KEY_SIZE,
  1126. .max_keysize = DES_KEY_SIZE,
  1127. },
  1128. .cra_init = spacc_ablk_cra_init,
  1129. .cra_exit = spacc_ablk_cra_exit,
  1130. },
  1131. },
  1132. {
  1133. .key_offs = DES_BLOCK_SIZE,
  1134. .iv_offs = 0,
  1135. .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC,
  1136. .alg = {
  1137. .cra_name = "cbc(des3_ede)",
  1138. .cra_driver_name = "cbc-des3-ede-picoxcell",
  1139. .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
  1140. .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
  1141. CRYPTO_ALG_ASYNC |
  1142. CRYPTO_ALG_KERN_DRIVER_ONLY,
  1143. .cra_blocksize = DES3_EDE_BLOCK_SIZE,
  1144. .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
  1145. .cra_type = &crypto_ablkcipher_type,
  1146. .cra_module = THIS_MODULE,
  1147. .cra_ablkcipher = {
  1148. .setkey = spacc_des3_setkey,
  1149. .encrypt = spacc_ablk_encrypt,
  1150. .decrypt = spacc_ablk_decrypt,
  1151. .min_keysize = DES3_EDE_KEY_SIZE,
  1152. .max_keysize = DES3_EDE_KEY_SIZE,
  1153. .ivsize = DES3_EDE_BLOCK_SIZE,
  1154. },
  1155. .cra_init = spacc_ablk_cra_init,
  1156. .cra_exit = spacc_ablk_cra_exit,
  1157. },
  1158. },
  1159. {
  1160. .key_offs = DES_BLOCK_SIZE,
  1161. .iv_offs = 0,
  1162. .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_ECB,
  1163. .alg = {
  1164. .cra_name = "ecb(des3_ede)",
  1165. .cra_driver_name = "ecb-des3-ede-picoxcell",
  1166. .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
  1167. .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
  1168. CRYPTO_ALG_ASYNC |
  1169. CRYPTO_ALG_KERN_DRIVER_ONLY,
  1170. .cra_blocksize = DES3_EDE_BLOCK_SIZE,
  1171. .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
  1172. .cra_type = &crypto_ablkcipher_type,
  1173. .cra_module = THIS_MODULE,
  1174. .cra_ablkcipher = {
  1175. .setkey = spacc_des3_setkey,
  1176. .encrypt = spacc_ablk_encrypt,
  1177. .decrypt = spacc_ablk_decrypt,
  1178. .min_keysize = DES3_EDE_KEY_SIZE,
  1179. .max_keysize = DES3_EDE_KEY_SIZE,
  1180. },
  1181. .cra_init = spacc_ablk_cra_init,
  1182. .cra_exit = spacc_ablk_cra_exit,
  1183. },
  1184. },
  1185. };
  1186. static struct spacc_aead ipsec_engine_aeads[] = {
  1187. {
  1188. .ctrl_default = SPA_CTRL_CIPH_ALG_AES |
  1189. SPA_CTRL_CIPH_MODE_CBC |
  1190. SPA_CTRL_HASH_ALG_SHA |
  1191. SPA_CTRL_HASH_MODE_HMAC,
  1192. .key_offs = 0,
  1193. .iv_offs = AES_MAX_KEY_SIZE,
  1194. .alg = {
  1195. .base = {
  1196. .cra_name = "authenc(hmac(sha1),cbc(aes))",
  1197. .cra_driver_name = "authenc-hmac-sha1-"
  1198. "cbc-aes-picoxcell",
  1199. .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
  1200. .cra_flags = CRYPTO_ALG_ASYNC |
  1201. CRYPTO_ALG_NEED_FALLBACK |
  1202. CRYPTO_ALG_KERN_DRIVER_ONLY,
  1203. .cra_blocksize = AES_BLOCK_SIZE,
  1204. .cra_ctxsize = sizeof(struct spacc_aead_ctx),
  1205. .cra_module = THIS_MODULE,
  1206. },
  1207. .setkey = spacc_aead_setkey,
  1208. .setauthsize = spacc_aead_setauthsize,
  1209. .encrypt = spacc_aead_encrypt,
  1210. .decrypt = spacc_aead_decrypt,
  1211. .ivsize = AES_BLOCK_SIZE,
  1212. .maxauthsize = SHA1_DIGEST_SIZE,
  1213. .init = spacc_aead_cra_init,
  1214. .exit = spacc_aead_cra_exit,
  1215. },
  1216. },
  1217. {
  1218. .ctrl_default = SPA_CTRL_CIPH_ALG_AES |
  1219. SPA_CTRL_CIPH_MODE_CBC |
  1220. SPA_CTRL_HASH_ALG_SHA256 |
  1221. SPA_CTRL_HASH_MODE_HMAC,
  1222. .key_offs = 0,
  1223. .iv_offs = AES_MAX_KEY_SIZE,
  1224. .alg = {
  1225. .base = {
  1226. .cra_name = "authenc(hmac(sha256),cbc(aes))",
  1227. .cra_driver_name = "authenc-hmac-sha256-"
  1228. "cbc-aes-picoxcell",
  1229. .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
  1230. .cra_flags = CRYPTO_ALG_ASYNC |
  1231. CRYPTO_ALG_NEED_FALLBACK |
  1232. CRYPTO_ALG_KERN_DRIVER_ONLY,
  1233. .cra_blocksize = AES_BLOCK_SIZE,
  1234. .cra_ctxsize = sizeof(struct spacc_aead_ctx),
  1235. .cra_module = THIS_MODULE,
  1236. },
  1237. .setkey = spacc_aead_setkey,
  1238. .setauthsize = spacc_aead_setauthsize,
  1239. .encrypt = spacc_aead_encrypt,
  1240. .decrypt = spacc_aead_decrypt,
  1241. .ivsize = AES_BLOCK_SIZE,
  1242. .maxauthsize = SHA256_DIGEST_SIZE,
  1243. .init = spacc_aead_cra_init,
  1244. .exit = spacc_aead_cra_exit,
  1245. },
  1246. },
  1247. {
  1248. .key_offs = 0,
  1249. .iv_offs = AES_MAX_KEY_SIZE,
  1250. .ctrl_default = SPA_CTRL_CIPH_ALG_AES |
  1251. SPA_CTRL_CIPH_MODE_CBC |
  1252. SPA_CTRL_HASH_ALG_MD5 |
  1253. SPA_CTRL_HASH_MODE_HMAC,
  1254. .alg = {
  1255. .base = {
  1256. .cra_name = "authenc(hmac(md5),cbc(aes))",
  1257. .cra_driver_name = "authenc-hmac-md5-"
  1258. "cbc-aes-picoxcell",
  1259. .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
  1260. .cra_flags = CRYPTO_ALG_ASYNC |
  1261. CRYPTO_ALG_NEED_FALLBACK |
  1262. CRYPTO_ALG_KERN_DRIVER_ONLY,
  1263. .cra_blocksize = AES_BLOCK_SIZE,
  1264. .cra_ctxsize = sizeof(struct spacc_aead_ctx),
  1265. .cra_module = THIS_MODULE,
  1266. },
  1267. .setkey = spacc_aead_setkey,
  1268. .setauthsize = spacc_aead_setauthsize,
  1269. .encrypt = spacc_aead_encrypt,
  1270. .decrypt = spacc_aead_decrypt,
  1271. .ivsize = AES_BLOCK_SIZE,
  1272. .maxauthsize = MD5_DIGEST_SIZE,
  1273. .init = spacc_aead_cra_init,
  1274. .exit = spacc_aead_cra_exit,
  1275. },
  1276. },
  1277. {
  1278. .key_offs = DES_BLOCK_SIZE,
  1279. .iv_offs = 0,
  1280. .ctrl_default = SPA_CTRL_CIPH_ALG_DES |
  1281. SPA_CTRL_CIPH_MODE_CBC |
  1282. SPA_CTRL_HASH_ALG_SHA |
  1283. SPA_CTRL_HASH_MODE_HMAC,
  1284. .alg = {
  1285. .base = {
  1286. .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
  1287. .cra_driver_name = "authenc-hmac-sha1-"
  1288. "cbc-3des-picoxcell",
  1289. .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
  1290. .cra_flags = CRYPTO_ALG_ASYNC |
  1291. CRYPTO_ALG_NEED_FALLBACK |
  1292. CRYPTO_ALG_KERN_DRIVER_ONLY,
  1293. .cra_blocksize = DES3_EDE_BLOCK_SIZE,
  1294. .cra_ctxsize = sizeof(struct spacc_aead_ctx),
  1295. .cra_module = THIS_MODULE,
  1296. },
  1297. .setkey = spacc_aead_setkey,
  1298. .setauthsize = spacc_aead_setauthsize,
  1299. .encrypt = spacc_aead_encrypt,
  1300. .decrypt = spacc_aead_decrypt,
  1301. .ivsize = DES3_EDE_BLOCK_SIZE,
  1302. .maxauthsize = SHA1_DIGEST_SIZE,
  1303. .init = spacc_aead_cra_init,
  1304. .exit = spacc_aead_cra_exit,
  1305. },
  1306. },
  1307. {
  1308. .key_offs = DES_BLOCK_SIZE,
  1309. .iv_offs = 0,
  1310. .ctrl_default = SPA_CTRL_CIPH_ALG_AES |
  1311. SPA_CTRL_CIPH_MODE_CBC |
  1312. SPA_CTRL_HASH_ALG_SHA256 |
  1313. SPA_CTRL_HASH_MODE_HMAC,
  1314. .alg = {
  1315. .base = {
  1316. .cra_name = "authenc(hmac(sha256),"
  1317. "cbc(des3_ede))",
  1318. .cra_driver_name = "authenc-hmac-sha256-"
  1319. "cbc-3des-picoxcell",
  1320. .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
  1321. .cra_flags = CRYPTO_ALG_ASYNC |
  1322. CRYPTO_ALG_NEED_FALLBACK |
  1323. CRYPTO_ALG_KERN_DRIVER_ONLY,
  1324. .cra_blocksize = DES3_EDE_BLOCK_SIZE,
  1325. .cra_ctxsize = sizeof(struct spacc_aead_ctx),
  1326. .cra_module = THIS_MODULE,
  1327. },
  1328. .setkey = spacc_aead_setkey,
  1329. .setauthsize = spacc_aead_setauthsize,
  1330. .encrypt = spacc_aead_encrypt,
  1331. .decrypt = spacc_aead_decrypt,
  1332. .ivsize = DES3_EDE_BLOCK_SIZE,
  1333. .maxauthsize = SHA256_DIGEST_SIZE,
  1334. .init = spacc_aead_cra_init,
  1335. .exit = spacc_aead_cra_exit,
  1336. },
  1337. },
  1338. {
  1339. .key_offs = DES_BLOCK_SIZE,
  1340. .iv_offs = 0,
  1341. .ctrl_default = SPA_CTRL_CIPH_ALG_DES |
  1342. SPA_CTRL_CIPH_MODE_CBC |
  1343. SPA_CTRL_HASH_ALG_MD5 |
  1344. SPA_CTRL_HASH_MODE_HMAC,
  1345. .alg = {
  1346. .base = {
  1347. .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
  1348. .cra_driver_name = "authenc-hmac-md5-"
  1349. "cbc-3des-picoxcell",
  1350. .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
  1351. .cra_flags = CRYPTO_ALG_ASYNC |
  1352. CRYPTO_ALG_NEED_FALLBACK |
  1353. CRYPTO_ALG_KERN_DRIVER_ONLY,
  1354. .cra_blocksize = DES3_EDE_BLOCK_SIZE,
  1355. .cra_ctxsize = sizeof(struct spacc_aead_ctx),
  1356. .cra_module = THIS_MODULE,
  1357. },
  1358. .setkey = spacc_aead_setkey,
  1359. .setauthsize = spacc_aead_setauthsize,
  1360. .encrypt = spacc_aead_encrypt,
  1361. .decrypt = spacc_aead_decrypt,
  1362. .ivsize = DES3_EDE_BLOCK_SIZE,
  1363. .maxauthsize = MD5_DIGEST_SIZE,
  1364. .init = spacc_aead_cra_init,
  1365. .exit = spacc_aead_cra_exit,
  1366. },
  1367. },
  1368. };
  1369. static struct spacc_alg l2_engine_algs[] = {
  1370. {
  1371. .key_offs = 0,
  1372. .iv_offs = SPACC_CRYPTO_KASUMI_F8_KEY_LEN,
  1373. .ctrl_default = SPA_CTRL_CIPH_ALG_KASUMI |
  1374. SPA_CTRL_CIPH_MODE_F8,
  1375. .alg = {
  1376. .cra_name = "f8(kasumi)",
  1377. .cra_driver_name = "f8-kasumi-picoxcell",
  1378. .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
  1379. .cra_flags = CRYPTO_ALG_ASYNC |
  1380. CRYPTO_ALG_KERN_DRIVER_ONLY,
  1381. .cra_blocksize = 8,
  1382. .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
  1383. .cra_type = &crypto_ablkcipher_type,
  1384. .cra_module = THIS_MODULE,
  1385. .cra_ablkcipher = {
  1386. .setkey = spacc_kasumi_f8_setkey,
  1387. .encrypt = spacc_ablk_encrypt,
  1388. .decrypt = spacc_ablk_decrypt,
  1389. .min_keysize = 16,
  1390. .max_keysize = 16,
  1391. .ivsize = 8,
  1392. },
  1393. .cra_init = spacc_ablk_cra_init,
  1394. .cra_exit = spacc_ablk_cra_exit,
  1395. },
  1396. },
  1397. };
  1398. #ifdef CONFIG_OF
  1399. static const struct of_device_id spacc_of_id_table[] = {
  1400. { .compatible = "picochip,spacc-ipsec" },
  1401. { .compatible = "picochip,spacc-l2" },
  1402. {}
  1403. };
  1404. MODULE_DEVICE_TABLE(of, spacc_of_id_table);
  1405. #endif /* CONFIG_OF */
  1406. static void spacc_tasklet_kill(void *data)
  1407. {
  1408. tasklet_kill(data);
  1409. }
  1410. static int spacc_probe(struct platform_device *pdev)
  1411. {
  1412. int i, err, ret;
  1413. struct resource *irq;
  1414. struct device_node *np = pdev->dev.of_node;
  1415. struct spacc_engine *engine = devm_kzalloc(&pdev->dev, sizeof(*engine),
  1416. GFP_KERNEL);
  1417. if (!engine)
  1418. return -ENOMEM;
  1419. if (of_device_is_compatible(np, "picochip,spacc-ipsec")) {
  1420. engine->max_ctxs = SPACC_CRYPTO_IPSEC_MAX_CTXS;
  1421. engine->cipher_pg_sz = SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ;
  1422. engine->hash_pg_sz = SPACC_CRYPTO_IPSEC_HASH_PG_SZ;
  1423. engine->fifo_sz = SPACC_CRYPTO_IPSEC_FIFO_SZ;
  1424. engine->algs = ipsec_engine_algs;
  1425. engine->num_algs = ARRAY_SIZE(ipsec_engine_algs);
  1426. engine->aeads = ipsec_engine_aeads;
  1427. engine->num_aeads = ARRAY_SIZE(ipsec_engine_aeads);
  1428. } else if (of_device_is_compatible(np, "picochip,spacc-l2")) {
  1429. engine->max_ctxs = SPACC_CRYPTO_L2_MAX_CTXS;
  1430. engine->cipher_pg_sz = SPACC_CRYPTO_L2_CIPHER_PG_SZ;
  1431. engine->hash_pg_sz = SPACC_CRYPTO_L2_HASH_PG_SZ;
  1432. engine->fifo_sz = SPACC_CRYPTO_L2_FIFO_SZ;
  1433. engine->algs = l2_engine_algs;
  1434. engine->num_algs = ARRAY_SIZE(l2_engine_algs);
  1435. } else {
  1436. return -EINVAL;
  1437. }
  1438. engine->name = dev_name(&pdev->dev);
  1439. engine->regs = devm_platform_ioremap_resource(pdev, 0);
  1440. if (IS_ERR(engine->regs))
  1441. return PTR_ERR(engine->regs);
  1442. irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
  1443. if (!irq) {
  1444. dev_err(&pdev->dev, "no memory/irq resource for engine\n");
  1445. return -ENXIO;
  1446. }
  1447. tasklet_init(&engine->complete, spacc_spacc_complete,
  1448. (unsigned long)engine);
  1449. ret = devm_add_action(&pdev->dev, spacc_tasklet_kill,
  1450. &engine->complete);
  1451. if (ret)
  1452. return ret;
  1453. if (devm_request_irq(&pdev->dev, irq->start, spacc_spacc_irq, 0,
  1454. engine->name, engine)) {
  1455. dev_err(engine->dev, "failed to request IRQ\n");
  1456. return -EBUSY;
  1457. }
  1458. engine->dev = &pdev->dev;
  1459. engine->cipher_ctx_base = engine->regs + SPA_CIPH_KEY_BASE_REG_OFFSET;
  1460. engine->hash_key_base = engine->regs + SPA_HASH_KEY_BASE_REG_OFFSET;
  1461. engine->req_pool = dmam_pool_create(engine->name, engine->dev,
  1462. MAX_DDT_LEN * sizeof(struct spacc_ddt), 8, SZ_64K);
  1463. if (!engine->req_pool)
  1464. return -ENOMEM;
  1465. spin_lock_init(&engine->hw_lock);
  1466. engine->clk = clk_get(&pdev->dev, "ref");
  1467. if (IS_ERR(engine->clk)) {
  1468. dev_info(&pdev->dev, "clk unavailable\n");
  1469. return PTR_ERR(engine->clk);
  1470. }
  1471. if (clk_prepare_enable(engine->clk)) {
  1472. dev_info(&pdev->dev, "unable to prepare/enable clk\n");
  1473. ret = -EIO;
  1474. goto err_clk_put;
  1475. }
  1476. /*
  1477. * Use an IRQ threshold of 50% as a default. This seems to be a
  1478. * reasonable trade off of latency against throughput but can be
  1479. * changed at runtime.
  1480. */
  1481. engine->stat_irq_thresh = (engine->fifo_sz / 2);
  1482. ret = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh);
  1483. if (ret)
  1484. goto err_clk_disable;
  1485. /*
  1486. * Configure the interrupts. We only use the STAT_CNT interrupt as we
  1487. * only submit a new packet for processing when we complete another in
  1488. * the queue. This minimizes time spent in the interrupt handler.
  1489. */
  1490. writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET,
  1491. engine->regs + SPA_IRQ_CTRL_REG_OFFSET);
  1492. writel(SPA_IRQ_EN_STAT_EN | SPA_IRQ_EN_GLBL_EN,
  1493. engine->regs + SPA_IRQ_EN_REG_OFFSET);
  1494. timer_setup(&engine->packet_timeout, spacc_packet_timeout, 0);
  1495. INIT_LIST_HEAD(&engine->pending);
  1496. INIT_LIST_HEAD(&engine->completed);
  1497. INIT_LIST_HEAD(&engine->in_progress);
  1498. engine->in_flight = 0;
  1499. platform_set_drvdata(pdev, engine);
  1500. ret = -EINVAL;
  1501. INIT_LIST_HEAD(&engine->registered_algs);
  1502. for (i = 0; i < engine->num_algs; ++i) {
  1503. engine->algs[i].engine = engine;
  1504. err = crypto_register_alg(&engine->algs[i].alg);
  1505. if (!err) {
  1506. list_add_tail(&engine->algs[i].entry,
  1507. &engine->registered_algs);
  1508. ret = 0;
  1509. }
  1510. if (err)
  1511. dev_err(engine->dev, "failed to register alg \"%s\"\n",
  1512. engine->algs[i].alg.cra_name);
  1513. else
  1514. dev_dbg(engine->dev, "registered alg \"%s\"\n",
  1515. engine->algs[i].alg.cra_name);
  1516. }
  1517. INIT_LIST_HEAD(&engine->registered_aeads);
  1518. for (i = 0; i < engine->num_aeads; ++i) {
  1519. engine->aeads[i].engine = engine;
  1520. err = crypto_register_aead(&engine->aeads[i].alg);
  1521. if (!err) {
  1522. list_add_tail(&engine->aeads[i].entry,
  1523. &engine->registered_aeads);
  1524. ret = 0;
  1525. }
  1526. if (err)
  1527. dev_err(engine->dev, "failed to register alg \"%s\"\n",
  1528. engine->aeads[i].alg.base.cra_name);
  1529. else
  1530. dev_dbg(engine->dev, "registered alg \"%s\"\n",
  1531. engine->aeads[i].alg.base.cra_name);
  1532. }
  1533. if (!ret)
  1534. return 0;
  1535. del_timer_sync(&engine->packet_timeout);
  1536. device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh);
  1537. err_clk_disable:
  1538. clk_disable_unprepare(engine->clk);
  1539. err_clk_put:
  1540. clk_put(engine->clk);
  1541. return ret;
  1542. }
  1543. static int spacc_remove(struct platform_device *pdev)
  1544. {
  1545. struct spacc_aead *aead, *an;
  1546. struct spacc_alg *alg, *next;
  1547. struct spacc_engine *engine = platform_get_drvdata(pdev);
  1548. del_timer_sync(&engine->packet_timeout);
  1549. device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh);
  1550. list_for_each_entry_safe(aead, an, &engine->registered_aeads, entry) {
  1551. list_del(&aead->entry);
  1552. crypto_unregister_aead(&aead->alg);
  1553. }
  1554. list_for_each_entry_safe(alg, next, &engine->registered_algs, entry) {
  1555. list_del(&alg->entry);
  1556. crypto_unregister_alg(&alg->alg);
  1557. }
  1558. clk_disable_unprepare(engine->clk);
  1559. clk_put(engine->clk);
  1560. return 0;
  1561. }
  1562. static struct platform_driver spacc_driver = {
  1563. .probe = spacc_probe,
  1564. .remove = spacc_remove,
  1565. .driver = {
  1566. .name = "picochip,spacc",
  1567. #ifdef CONFIG_PM
  1568. .pm = &spacc_pm_ops,
  1569. #endif /* CONFIG_PM */
  1570. .of_match_table = of_match_ptr(spacc_of_id_table),
  1571. },
  1572. };
  1573. module_platform_driver(spacc_driver);
  1574. MODULE_LICENSE("GPL");
  1575. MODULE_AUTHOR("Jamie Iles");