esp4.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074
  1. #define pr_fmt(fmt) "IPsec: " fmt
  2. #include <crypto/aead.h>
  3. #include <crypto/authenc.h>
  4. #include <linux/err.h>
  5. #include <linux/module.h>
  6. #include <net/ip.h>
  7. #include <net/xfrm.h>
  8. #include <net/esp.h>
  9. #include <linux/scatterlist.h>
  10. #include <linux/kernel.h>
  11. #include <linux/pfkeyv2.h>
  12. #include <linux/rtnetlink.h>
  13. #include <linux/slab.h>
  14. #include <linux/spinlock.h>
  15. #include <linux/in6.h>
  16. #include <net/icmp.h>
  17. #include <net/protocol.h>
  18. #include <net/udp.h>
  19. #include <linux/highmem.h>
  20. struct esp_skb_cb {
  21. struct xfrm_skb_cb xfrm;
  22. void *tmp;
  23. };
  24. struct esp_output_extra {
  25. __be32 seqhi;
  26. u32 esphoff;
  27. };
  28. #define ESP_SKB_CB(__skb) ((struct esp_skb_cb *)&((__skb)->cb[0]))
  29. static u32 esp4_get_mtu(struct xfrm_state *x, int mtu);
  30. /*
  31. * Allocate an AEAD request structure with extra space for SG and IV.
  32. *
  33. * For alignment considerations the IV is placed at the front, followed
  34. * by the request and finally the SG list.
  35. *
  36. * TODO: Use spare space in skb for this where possible.
  37. */
  38. static void *esp_alloc_tmp(struct crypto_aead *aead, int nfrags, int extralen)
  39. {
  40. unsigned int len;
  41. len = extralen;
  42. len += crypto_aead_ivsize(aead);
  43. if (len) {
  44. len += crypto_aead_alignmask(aead) &
  45. ~(crypto_tfm_ctx_alignment() - 1);
  46. len = ALIGN(len, crypto_tfm_ctx_alignment());
  47. }
  48. len += sizeof(struct aead_request) + crypto_aead_reqsize(aead);
  49. len = ALIGN(len, __alignof__(struct scatterlist));
  50. len += sizeof(struct scatterlist) * nfrags;
  51. return kmalloc(len, GFP_ATOMIC);
  52. }
  53. static inline void *esp_tmp_extra(void *tmp)
  54. {
  55. return PTR_ALIGN(tmp, __alignof__(struct esp_output_extra));
  56. }
  57. static inline u8 *esp_tmp_iv(struct crypto_aead *aead, void *tmp, int extralen)
  58. {
  59. return crypto_aead_ivsize(aead) ?
  60. PTR_ALIGN((u8 *)tmp + extralen,
  61. crypto_aead_alignmask(aead) + 1) : tmp + extralen;
  62. }
  63. static inline struct aead_request *esp_tmp_req(struct crypto_aead *aead, u8 *iv)
  64. {
  65. struct aead_request *req;
  66. req = (void *)PTR_ALIGN(iv + crypto_aead_ivsize(aead),
  67. crypto_tfm_ctx_alignment());
  68. aead_request_set_tfm(req, aead);
  69. return req;
  70. }
  71. static inline struct scatterlist *esp_req_sg(struct crypto_aead *aead,
  72. struct aead_request *req)
  73. {
  74. return (void *)ALIGN((unsigned long)(req + 1) +
  75. crypto_aead_reqsize(aead),
  76. __alignof__(struct scatterlist));
  77. }
  78. static void esp_ssg_unref(struct xfrm_state *x, void *tmp)
  79. {
  80. struct esp_output_extra *extra = esp_tmp_extra(tmp);
  81. struct crypto_aead *aead = x->data;
  82. int extralen = 0;
  83. u8 *iv;
  84. struct aead_request *req;
  85. struct scatterlist *sg;
  86. if (x->props.flags & XFRM_STATE_ESN)
  87. extralen += sizeof(*extra);
  88. extra = esp_tmp_extra(tmp);
  89. iv = esp_tmp_iv(aead, tmp, extralen);
  90. req = esp_tmp_req(aead, iv);
  91. /* Unref skb_frag_pages in the src scatterlist if necessary.
  92. * Skip the first sg which comes from skb->data.
  93. */
  94. if (req->src != req->dst)
  95. for (sg = sg_next(req->src); sg; sg = sg_next(sg))
  96. put_page(sg_page(sg));
  97. }
  98. static void esp_output_done(struct crypto_async_request *base, int err)
  99. {
  100. struct sk_buff *skb = base->data;
  101. struct xfrm_offload *xo = xfrm_offload(skb);
  102. void *tmp;
  103. struct xfrm_state *x;
  104. if (xo && (xo->flags & XFRM_DEV_RESUME))
  105. x = skb->sp->xvec[skb->sp->len - 1];
  106. else
  107. x = skb_dst(skb)->xfrm;
  108. tmp = ESP_SKB_CB(skb)->tmp;
  109. esp_ssg_unref(x, tmp);
  110. kfree(tmp);
  111. if (xo && (xo->flags & XFRM_DEV_RESUME)) {
  112. if (err) {
  113. XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
  114. kfree_skb(skb);
  115. return;
  116. }
  117. skb_push(skb, skb->data - skb_mac_header(skb));
  118. secpath_reset(skb);
  119. xfrm_dev_resume(skb);
  120. } else {
  121. xfrm_output_resume(skb, err);
  122. }
  123. }
  124. /* Move ESP header back into place. */
  125. static void esp_restore_header(struct sk_buff *skb, unsigned int offset)
  126. {
  127. struct ip_esp_hdr *esph = (void *)(skb->data + offset);
  128. void *tmp = ESP_SKB_CB(skb)->tmp;
  129. __be32 *seqhi = esp_tmp_extra(tmp);
  130. esph->seq_no = esph->spi;
  131. esph->spi = *seqhi;
  132. }
  133. static void esp_output_restore_header(struct sk_buff *skb)
  134. {
  135. void *tmp = ESP_SKB_CB(skb)->tmp;
  136. struct esp_output_extra *extra = esp_tmp_extra(tmp);
  137. esp_restore_header(skb, skb_transport_offset(skb) + extra->esphoff -
  138. sizeof(__be32));
  139. }
  140. static struct ip_esp_hdr *esp_output_set_extra(struct sk_buff *skb,
  141. struct xfrm_state *x,
  142. struct ip_esp_hdr *esph,
  143. struct esp_output_extra *extra)
  144. {
  145. /* For ESN we move the header forward by 4 bytes to
  146. * accomodate the high bits. We will move it back after
  147. * encryption.
  148. */
  149. if ((x->props.flags & XFRM_STATE_ESN)) {
  150. __u32 seqhi;
  151. struct xfrm_offload *xo = xfrm_offload(skb);
  152. if (xo)
  153. seqhi = xo->seq.hi;
  154. else
  155. seqhi = XFRM_SKB_CB(skb)->seq.output.hi;
  156. extra->esphoff = (unsigned char *)esph -
  157. skb_transport_header(skb);
  158. esph = (struct ip_esp_hdr *)((unsigned char *)esph - 4);
  159. extra->seqhi = esph->spi;
  160. esph->seq_no = htonl(seqhi);
  161. }
  162. esph->spi = x->id.spi;
  163. return esph;
  164. }
  165. static void esp_output_done_esn(struct crypto_async_request *base, int err)
  166. {
  167. struct sk_buff *skb = base->data;
  168. esp_output_restore_header(skb);
  169. esp_output_done(base, err);
  170. }
  171. static void esp_output_fill_trailer(u8 *tail, int tfclen, int plen, __u8 proto)
  172. {
  173. /* Fill padding... */
  174. if (tfclen) {
  175. memset(tail, 0, tfclen);
  176. tail += tfclen;
  177. }
  178. do {
  179. int i;
  180. for (i = 0; i < plen - 2; i++)
  181. tail[i] = i + 1;
  182. } while (0);
  183. tail[plen - 2] = plen - 2;
  184. tail[plen - 1] = proto;
  185. }
  186. static int esp_output_udp_encap(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp)
  187. {
  188. int encap_type;
  189. struct udphdr *uh;
  190. __be32 *udpdata32;
  191. __be16 sport, dport;
  192. struct xfrm_encap_tmpl *encap = x->encap;
  193. struct ip_esp_hdr *esph = esp->esph;
  194. unsigned int len;
  195. spin_lock_bh(&x->lock);
  196. sport = encap->encap_sport;
  197. dport = encap->encap_dport;
  198. encap_type = encap->encap_type;
  199. spin_unlock_bh(&x->lock);
  200. len = skb->len + esp->tailen - skb_transport_offset(skb);
  201. if (len + sizeof(struct iphdr) >= IP_MAX_MTU)
  202. return -EMSGSIZE;
  203. uh = (struct udphdr *)esph;
  204. uh->source = sport;
  205. uh->dest = dport;
  206. uh->len = htons(len);
  207. uh->check = 0;
  208. switch (encap_type) {
  209. default:
  210. case UDP_ENCAP_ESPINUDP:
  211. esph = (struct ip_esp_hdr *)(uh + 1);
  212. break;
  213. case UDP_ENCAP_ESPINUDP_NON_IKE:
  214. udpdata32 = (__be32 *)(uh + 1);
  215. udpdata32[0] = udpdata32[1] = 0;
  216. esph = (struct ip_esp_hdr *)(udpdata32 + 2);
  217. break;
  218. }
  219. *skb_mac_header(skb) = IPPROTO_UDP;
  220. esp->esph = esph;
  221. return 0;
  222. }
  223. int esp_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp)
  224. {
  225. u8 *tail;
  226. u8 *vaddr;
  227. int nfrags;
  228. int esph_offset;
  229. struct page *page;
  230. struct sk_buff *trailer;
  231. int tailen = esp->tailen;
  232. /* this is non-NULL only with UDP Encapsulation */
  233. if (x->encap) {
  234. int err = esp_output_udp_encap(x, skb, esp);
  235. if (err < 0)
  236. return err;
  237. }
  238. if (!skb_cloned(skb)) {
  239. if (tailen <= skb_tailroom(skb)) {
  240. nfrags = 1;
  241. trailer = skb;
  242. tail = skb_tail_pointer(trailer);
  243. goto skip_cow;
  244. } else if ((skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS)
  245. && !skb_has_frag_list(skb)) {
  246. int allocsize;
  247. struct sock *sk = skb->sk;
  248. struct page_frag *pfrag = &x->xfrag;
  249. esp->inplace = false;
  250. allocsize = ALIGN(tailen, L1_CACHE_BYTES);
  251. spin_lock_bh(&x->lock);
  252. if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) {
  253. spin_unlock_bh(&x->lock);
  254. goto cow;
  255. }
  256. page = pfrag->page;
  257. get_page(page);
  258. vaddr = kmap_atomic(page);
  259. tail = vaddr + pfrag->offset;
  260. esp_output_fill_trailer(tail, esp->tfclen, esp->plen, esp->proto);
  261. kunmap_atomic(vaddr);
  262. nfrags = skb_shinfo(skb)->nr_frags;
  263. __skb_fill_page_desc(skb, nfrags, page, pfrag->offset,
  264. tailen);
  265. skb_shinfo(skb)->nr_frags = ++nfrags;
  266. pfrag->offset = pfrag->offset + allocsize;
  267. spin_unlock_bh(&x->lock);
  268. nfrags++;
  269. skb->len += tailen;
  270. skb->data_len += tailen;
  271. skb->truesize += tailen;
  272. if (sk && sk_fullsock(sk))
  273. refcount_add(tailen, &sk->sk_wmem_alloc);
  274. goto out;
  275. }
  276. }
  277. cow:
  278. esph_offset = (unsigned char *)esp->esph - skb_transport_header(skb);
  279. nfrags = skb_cow_data(skb, tailen, &trailer);
  280. if (nfrags < 0)
  281. goto out;
  282. tail = skb_tail_pointer(trailer);
  283. esp->esph = (struct ip_esp_hdr *)(skb_transport_header(skb) + esph_offset);
  284. skip_cow:
  285. esp_output_fill_trailer(tail, esp->tfclen, esp->plen, esp->proto);
  286. pskb_put(skb, trailer, tailen);
  287. out:
  288. return nfrags;
  289. }
  290. EXPORT_SYMBOL_GPL(esp_output_head);
  291. int esp_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp)
  292. {
  293. u8 *iv;
  294. int alen;
  295. void *tmp;
  296. int ivlen;
  297. int assoclen;
  298. int extralen;
  299. struct page *page;
  300. struct ip_esp_hdr *esph;
  301. struct crypto_aead *aead;
  302. struct aead_request *req;
  303. struct scatterlist *sg, *dsg;
  304. struct esp_output_extra *extra;
  305. int err = -ENOMEM;
  306. assoclen = sizeof(struct ip_esp_hdr);
  307. extralen = 0;
  308. if (x->props.flags & XFRM_STATE_ESN) {
  309. extralen += sizeof(*extra);
  310. assoclen += sizeof(__be32);
  311. }
  312. aead = x->data;
  313. alen = crypto_aead_authsize(aead);
  314. ivlen = crypto_aead_ivsize(aead);
  315. tmp = esp_alloc_tmp(aead, esp->nfrags + 2, extralen);
  316. if (!tmp)
  317. goto error;
  318. extra = esp_tmp_extra(tmp);
  319. iv = esp_tmp_iv(aead, tmp, extralen);
  320. req = esp_tmp_req(aead, iv);
  321. sg = esp_req_sg(aead, req);
  322. if (esp->inplace)
  323. dsg = sg;
  324. else
  325. dsg = &sg[esp->nfrags];
  326. esph = esp_output_set_extra(skb, x, esp->esph, extra);
  327. esp->esph = esph;
  328. sg_init_table(sg, esp->nfrags);
  329. err = skb_to_sgvec(skb, sg,
  330. (unsigned char *)esph - skb->data,
  331. assoclen + ivlen + esp->clen + alen);
  332. if (unlikely(err < 0))
  333. goto error_free;
  334. if (!esp->inplace) {
  335. int allocsize;
  336. struct page_frag *pfrag = &x->xfrag;
  337. allocsize = ALIGN(skb->data_len, L1_CACHE_BYTES);
  338. spin_lock_bh(&x->lock);
  339. if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) {
  340. spin_unlock_bh(&x->lock);
  341. goto error_free;
  342. }
  343. skb_shinfo(skb)->nr_frags = 1;
  344. page = pfrag->page;
  345. get_page(page);
  346. /* replace page frags in skb with new page */
  347. __skb_fill_page_desc(skb, 0, page, pfrag->offset, skb->data_len);
  348. pfrag->offset = pfrag->offset + allocsize;
  349. spin_unlock_bh(&x->lock);
  350. sg_init_table(dsg, skb_shinfo(skb)->nr_frags + 1);
  351. err = skb_to_sgvec(skb, dsg,
  352. (unsigned char *)esph - skb->data,
  353. assoclen + ivlen + esp->clen + alen);
  354. if (unlikely(err < 0))
  355. goto error_free;
  356. }
  357. if ((x->props.flags & XFRM_STATE_ESN))
  358. aead_request_set_callback(req, 0, esp_output_done_esn, skb);
  359. else
  360. aead_request_set_callback(req, 0, esp_output_done, skb);
  361. aead_request_set_crypt(req, sg, dsg, ivlen + esp->clen, iv);
  362. aead_request_set_ad(req, assoclen);
  363. memset(iv, 0, ivlen);
  364. memcpy(iv + ivlen - min(ivlen, 8), (u8 *)&esp->seqno + 8 - min(ivlen, 8),
  365. min(ivlen, 8));
  366. ESP_SKB_CB(skb)->tmp = tmp;
  367. err = crypto_aead_encrypt(req);
  368. switch (err) {
  369. case -EINPROGRESS:
  370. goto error;
  371. case -ENOSPC:
  372. err = NET_XMIT_DROP;
  373. break;
  374. case 0:
  375. if ((x->props.flags & XFRM_STATE_ESN))
  376. esp_output_restore_header(skb);
  377. }
  378. if (sg != dsg)
  379. esp_ssg_unref(x, tmp);
  380. error_free:
  381. kfree(tmp);
  382. error:
  383. return err;
  384. }
  385. EXPORT_SYMBOL_GPL(esp_output_tail);
  386. static int esp_output(struct xfrm_state *x, struct sk_buff *skb)
  387. {
  388. int alen;
  389. int blksize;
  390. struct ip_esp_hdr *esph;
  391. struct crypto_aead *aead;
  392. struct esp_info esp;
  393. esp.inplace = true;
  394. esp.proto = *skb_mac_header(skb);
  395. *skb_mac_header(skb) = IPPROTO_ESP;
  396. /* skb is pure payload to encrypt */
  397. aead = x->data;
  398. alen = crypto_aead_authsize(aead);
  399. esp.tfclen = 0;
  400. if (x->tfcpad) {
  401. struct xfrm_dst *dst = (struct xfrm_dst *)skb_dst(skb);
  402. u32 padto;
  403. padto = min(x->tfcpad, esp4_get_mtu(x, dst->child_mtu_cached));
  404. if (skb->len < padto)
  405. esp.tfclen = padto - skb->len;
  406. }
  407. blksize = ALIGN(crypto_aead_blocksize(aead), 4);
  408. esp.clen = ALIGN(skb->len + 2 + esp.tfclen, blksize);
  409. esp.plen = esp.clen - skb->len - esp.tfclen;
  410. esp.tailen = esp.tfclen + esp.plen + alen;
  411. esp.esph = ip_esp_hdr(skb);
  412. esp.nfrags = esp_output_head(x, skb, &esp);
  413. if (esp.nfrags < 0)
  414. return esp.nfrags;
  415. esph = esp.esph;
  416. esph->spi = x->id.spi;
  417. esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.low);
  418. esp.seqno = cpu_to_be64(XFRM_SKB_CB(skb)->seq.output.low +
  419. ((u64)XFRM_SKB_CB(skb)->seq.output.hi << 32));
  420. skb_push(skb, -skb_network_offset(skb));
  421. return esp_output_tail(x, skb, &esp);
  422. }
  423. static inline int esp_remove_trailer(struct sk_buff *skb)
  424. {
  425. struct xfrm_state *x = xfrm_input_state(skb);
  426. struct xfrm_offload *xo = xfrm_offload(skb);
  427. struct crypto_aead *aead = x->data;
  428. int alen, hlen, elen;
  429. int padlen, trimlen;
  430. __wsum csumdiff;
  431. u8 nexthdr[2];
  432. int ret;
  433. alen = crypto_aead_authsize(aead);
  434. hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);
  435. elen = skb->len - hlen;
  436. if (xo && (xo->flags & XFRM_ESP_NO_TRAILER)) {
  437. ret = xo->proto;
  438. goto out;
  439. }
  440. if (skb_copy_bits(skb, skb->len - alen - 2, nexthdr, 2))
  441. BUG();
  442. ret = -EINVAL;
  443. padlen = nexthdr[0];
  444. if (padlen + 2 + alen >= elen) {
  445. net_dbg_ratelimited("ipsec esp packet is garbage padlen=%d, elen=%d\n",
  446. padlen + 2, elen - alen);
  447. goto out;
  448. }
  449. trimlen = alen + padlen + 2;
  450. if (skb->ip_summed == CHECKSUM_COMPLETE) {
  451. csumdiff = skb_checksum(skb, skb->len - trimlen, trimlen, 0);
  452. skb->csum = csum_block_sub(skb->csum, csumdiff,
  453. skb->len - trimlen);
  454. }
  455. pskb_trim(skb, skb->len - trimlen);
  456. ret = nexthdr[1];
  457. out:
  458. return ret;
  459. }
  460. int esp_input_done2(struct sk_buff *skb, int err)
  461. {
  462. const struct iphdr *iph;
  463. struct xfrm_state *x = xfrm_input_state(skb);
  464. struct xfrm_offload *xo = xfrm_offload(skb);
  465. struct crypto_aead *aead = x->data;
  466. int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);
  467. int ihl;
  468. if (!xo || (xo && !(xo->flags & CRYPTO_DONE)))
  469. kfree(ESP_SKB_CB(skb)->tmp);
  470. if (unlikely(err))
  471. goto out;
  472. err = esp_remove_trailer(skb);
  473. if (unlikely(err < 0))
  474. goto out;
  475. iph = ip_hdr(skb);
  476. ihl = iph->ihl * 4;
  477. if (x->encap) {
  478. struct xfrm_encap_tmpl *encap = x->encap;
  479. struct udphdr *uh = (void *)(skb_network_header(skb) + ihl);
  480. /*
  481. * 1) if the NAT-T peer's IP or port changed then
  482. * advertize the change to the keying daemon.
  483. * This is an inbound SA, so just compare
  484. * SRC ports.
  485. */
  486. if (iph->saddr != x->props.saddr.a4 ||
  487. uh->source != encap->encap_sport) {
  488. xfrm_address_t ipaddr;
  489. ipaddr.a4 = iph->saddr;
  490. km_new_mapping(x, &ipaddr, uh->source);
  491. /* XXX: perhaps add an extra
  492. * policy check here, to see
  493. * if we should allow or
  494. * reject a packet from a
  495. * different source
  496. * address/port.
  497. */
  498. }
  499. /*
  500. * 2) ignore UDP/TCP checksums in case
  501. * of NAT-T in Transport Mode, or
  502. * perform other post-processing fixes
  503. * as per draft-ietf-ipsec-udp-encaps-06,
  504. * section 3.1.2
  505. */
  506. if (x->props.mode == XFRM_MODE_TRANSPORT)
  507. skb->ip_summed = CHECKSUM_UNNECESSARY;
  508. }
  509. skb_pull_rcsum(skb, hlen);
  510. if (x->props.mode == XFRM_MODE_TUNNEL)
  511. skb_reset_transport_header(skb);
  512. else
  513. skb_set_transport_header(skb, -ihl);
  514. /* RFC4303: Drop dummy packets without any error */
  515. if (err == IPPROTO_NONE)
  516. err = -EINVAL;
  517. out:
  518. return err;
  519. }
  520. EXPORT_SYMBOL_GPL(esp_input_done2);
  521. static void esp_input_done(struct crypto_async_request *base, int err)
  522. {
  523. struct sk_buff *skb = base->data;
  524. xfrm_input_resume(skb, esp_input_done2(skb, err));
  525. }
  526. static void esp_input_restore_header(struct sk_buff *skb)
  527. {
  528. esp_restore_header(skb, 0);
  529. __skb_pull(skb, 4);
  530. }
  531. static void esp_input_set_header(struct sk_buff *skb, __be32 *seqhi)
  532. {
  533. struct xfrm_state *x = xfrm_input_state(skb);
  534. struct ip_esp_hdr *esph;
  535. /* For ESN we move the header forward by 4 bytes to
  536. * accomodate the high bits. We will move it back after
  537. * decryption.
  538. */
  539. if ((x->props.flags & XFRM_STATE_ESN)) {
  540. esph = skb_push(skb, 4);
  541. *seqhi = esph->spi;
  542. esph->spi = esph->seq_no;
  543. esph->seq_no = XFRM_SKB_CB(skb)->seq.input.hi;
  544. }
  545. }
  546. static void esp_input_done_esn(struct crypto_async_request *base, int err)
  547. {
  548. struct sk_buff *skb = base->data;
  549. esp_input_restore_header(skb);
  550. esp_input_done(base, err);
  551. }
  552. /*
  553. * Note: detecting truncated vs. non-truncated authentication data is very
  554. * expensive, so we only support truncated data, which is the recommended
  555. * and common case.
  556. */
  557. static int esp_input(struct xfrm_state *x, struct sk_buff *skb)
  558. {
  559. struct ip_esp_hdr *esph;
  560. struct crypto_aead *aead = x->data;
  561. struct aead_request *req;
  562. struct sk_buff *trailer;
  563. int ivlen = crypto_aead_ivsize(aead);
  564. int elen = skb->len - sizeof(*esph) - ivlen;
  565. int nfrags;
  566. int assoclen;
  567. int seqhilen;
  568. __be32 *seqhi;
  569. void *tmp;
  570. u8 *iv;
  571. struct scatterlist *sg;
  572. int err = -EINVAL;
  573. if (!pskb_may_pull(skb, sizeof(*esph) + ivlen))
  574. goto out;
  575. if (elen <= 0)
  576. goto out;
  577. assoclen = sizeof(*esph);
  578. seqhilen = 0;
  579. if (x->props.flags & XFRM_STATE_ESN) {
  580. seqhilen += sizeof(__be32);
  581. assoclen += seqhilen;
  582. }
  583. if (!skb_cloned(skb)) {
  584. if (!skb_is_nonlinear(skb)) {
  585. nfrags = 1;
  586. goto skip_cow;
  587. } else if (!skb_has_frag_list(skb)) {
  588. nfrags = skb_shinfo(skb)->nr_frags;
  589. nfrags++;
  590. goto skip_cow;
  591. }
  592. }
  593. err = skb_cow_data(skb, 0, &trailer);
  594. if (err < 0)
  595. goto out;
  596. nfrags = err;
  597. skip_cow:
  598. err = -ENOMEM;
  599. tmp = esp_alloc_tmp(aead, nfrags, seqhilen);
  600. if (!tmp)
  601. goto out;
  602. ESP_SKB_CB(skb)->tmp = tmp;
  603. seqhi = esp_tmp_extra(tmp);
  604. iv = esp_tmp_iv(aead, tmp, seqhilen);
  605. req = esp_tmp_req(aead, iv);
  606. sg = esp_req_sg(aead, req);
  607. esp_input_set_header(skb, seqhi);
  608. sg_init_table(sg, nfrags);
  609. err = skb_to_sgvec(skb, sg, 0, skb->len);
  610. if (unlikely(err < 0)) {
  611. kfree(tmp);
  612. goto out;
  613. }
  614. skb->ip_summed = CHECKSUM_NONE;
  615. if ((x->props.flags & XFRM_STATE_ESN))
  616. aead_request_set_callback(req, 0, esp_input_done_esn, skb);
  617. else
  618. aead_request_set_callback(req, 0, esp_input_done, skb);
  619. aead_request_set_crypt(req, sg, sg, elen + ivlen, iv);
  620. aead_request_set_ad(req, assoclen);
  621. err = crypto_aead_decrypt(req);
  622. if (err == -EINPROGRESS)
  623. goto out;
  624. if ((x->props.flags & XFRM_STATE_ESN))
  625. esp_input_restore_header(skb);
  626. err = esp_input_done2(skb, err);
  627. out:
  628. return err;
  629. }
  630. static u32 esp4_get_mtu(struct xfrm_state *x, int mtu)
  631. {
  632. struct crypto_aead *aead = x->data;
  633. u32 blksize = ALIGN(crypto_aead_blocksize(aead), 4);
  634. unsigned int net_adj;
  635. switch (x->props.mode) {
  636. case XFRM_MODE_TRANSPORT:
  637. case XFRM_MODE_BEET:
  638. net_adj = sizeof(struct iphdr);
  639. break;
  640. case XFRM_MODE_TUNNEL:
  641. net_adj = 0;
  642. break;
  643. default:
  644. BUG();
  645. }
  646. return ((mtu - x->props.header_len - crypto_aead_authsize(aead) -
  647. net_adj) & ~(blksize - 1)) + net_adj - 2;
  648. }
  649. static int esp4_err(struct sk_buff *skb, u32 info)
  650. {
  651. struct net *net = dev_net(skb->dev);
  652. const struct iphdr *iph = (const struct iphdr *)skb->data;
  653. struct ip_esp_hdr *esph = (struct ip_esp_hdr *)(skb->data+(iph->ihl<<2));
  654. struct xfrm_state *x;
  655. switch (icmp_hdr(skb)->type) {
  656. case ICMP_DEST_UNREACH:
  657. if (icmp_hdr(skb)->code != ICMP_FRAG_NEEDED)
  658. return 0;
  659. case ICMP_REDIRECT:
  660. break;
  661. default:
  662. return 0;
  663. }
  664. x = xfrm_state_lookup(net, skb->mark, (const xfrm_address_t *)&iph->daddr,
  665. esph->spi, IPPROTO_ESP, AF_INET);
  666. if (!x)
  667. return 0;
  668. if (icmp_hdr(skb)->type == ICMP_DEST_UNREACH)
  669. ipv4_update_pmtu(skb, net, info, 0, 0, IPPROTO_ESP, 0);
  670. else
  671. ipv4_redirect(skb, net, 0, 0, IPPROTO_ESP, 0);
  672. xfrm_state_put(x);
  673. return 0;
  674. }
  675. static void esp_destroy(struct xfrm_state *x)
  676. {
  677. struct crypto_aead *aead = x->data;
  678. if (!aead)
  679. return;
  680. crypto_free_aead(aead);
  681. }
  682. static int esp_init_aead(struct xfrm_state *x)
  683. {
  684. char aead_name[CRYPTO_MAX_ALG_NAME];
  685. struct crypto_aead *aead;
  686. int err;
  687. err = -ENAMETOOLONG;
  688. if (snprintf(aead_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",
  689. x->geniv, x->aead->alg_name) >= CRYPTO_MAX_ALG_NAME)
  690. goto error;
  691. aead = crypto_alloc_aead(aead_name, 0, 0);
  692. err = PTR_ERR(aead);
  693. if (IS_ERR(aead))
  694. goto error;
  695. x->data = aead;
  696. err = crypto_aead_setkey(aead, x->aead->alg_key,
  697. (x->aead->alg_key_len + 7) / 8);
  698. if (err)
  699. goto error;
  700. err = crypto_aead_setauthsize(aead, x->aead->alg_icv_len / 8);
  701. if (err)
  702. goto error;
  703. error:
  704. return err;
  705. }
  706. static int esp_init_authenc(struct xfrm_state *x)
  707. {
  708. struct crypto_aead *aead;
  709. struct crypto_authenc_key_param *param;
  710. struct rtattr *rta;
  711. char *key;
  712. char *p;
  713. char authenc_name[CRYPTO_MAX_ALG_NAME];
  714. unsigned int keylen;
  715. int err;
  716. err = -EINVAL;
  717. if (!x->ealg)
  718. goto error;
  719. err = -ENAMETOOLONG;
  720. if ((x->props.flags & XFRM_STATE_ESN)) {
  721. if (snprintf(authenc_name, CRYPTO_MAX_ALG_NAME,
  722. "%s%sauthencesn(%s,%s)%s",
  723. x->geniv ?: "", x->geniv ? "(" : "",
  724. x->aalg ? x->aalg->alg_name : "digest_null",
  725. x->ealg->alg_name,
  726. x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME)
  727. goto error;
  728. } else {
  729. if (snprintf(authenc_name, CRYPTO_MAX_ALG_NAME,
  730. "%s%sauthenc(%s,%s)%s",
  731. x->geniv ?: "", x->geniv ? "(" : "",
  732. x->aalg ? x->aalg->alg_name : "digest_null",
  733. x->ealg->alg_name,
  734. x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME)
  735. goto error;
  736. }
  737. aead = crypto_alloc_aead(authenc_name, 0, 0);
  738. err = PTR_ERR(aead);
  739. if (IS_ERR(aead))
  740. goto error;
  741. x->data = aead;
  742. keylen = (x->aalg ? (x->aalg->alg_key_len + 7) / 8 : 0) +
  743. (x->ealg->alg_key_len + 7) / 8 + RTA_SPACE(sizeof(*param));
  744. err = -ENOMEM;
  745. key = kmalloc(keylen, GFP_KERNEL);
  746. if (!key)
  747. goto error;
  748. p = key;
  749. rta = (void *)p;
  750. rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM;
  751. rta->rta_len = RTA_LENGTH(sizeof(*param));
  752. param = RTA_DATA(rta);
  753. p += RTA_SPACE(sizeof(*param));
  754. if (x->aalg) {
  755. struct xfrm_algo_desc *aalg_desc;
  756. memcpy(p, x->aalg->alg_key, (x->aalg->alg_key_len + 7) / 8);
  757. p += (x->aalg->alg_key_len + 7) / 8;
  758. aalg_desc = xfrm_aalg_get_byname(x->aalg->alg_name, 0);
  759. BUG_ON(!aalg_desc);
  760. err = -EINVAL;
  761. if (aalg_desc->uinfo.auth.icv_fullbits / 8 !=
  762. crypto_aead_authsize(aead)) {
  763. pr_info("ESP: %s digestsize %u != %hu\n",
  764. x->aalg->alg_name,
  765. crypto_aead_authsize(aead),
  766. aalg_desc->uinfo.auth.icv_fullbits / 8);
  767. goto free_key;
  768. }
  769. err = crypto_aead_setauthsize(
  770. aead, x->aalg->alg_trunc_len / 8);
  771. if (err)
  772. goto free_key;
  773. }
  774. param->enckeylen = cpu_to_be32((x->ealg->alg_key_len + 7) / 8);
  775. memcpy(p, x->ealg->alg_key, (x->ealg->alg_key_len + 7) / 8);
  776. err = crypto_aead_setkey(aead, key, keylen);
  777. free_key:
  778. kfree(key);
  779. error:
  780. return err;
  781. }
  782. static int esp_init_state(struct xfrm_state *x)
  783. {
  784. struct crypto_aead *aead;
  785. u32 align;
  786. int err;
  787. x->data = NULL;
  788. if (x->aead)
  789. err = esp_init_aead(x);
  790. else
  791. err = esp_init_authenc(x);
  792. if (err)
  793. goto error;
  794. aead = x->data;
  795. x->props.header_len = sizeof(struct ip_esp_hdr) +
  796. crypto_aead_ivsize(aead);
  797. if (x->props.mode == XFRM_MODE_TUNNEL)
  798. x->props.header_len += sizeof(struct iphdr);
  799. else if (x->props.mode == XFRM_MODE_BEET && x->sel.family != AF_INET6)
  800. x->props.header_len += IPV4_BEET_PHMAXLEN;
  801. if (x->encap) {
  802. struct xfrm_encap_tmpl *encap = x->encap;
  803. switch (encap->encap_type) {
  804. default:
  805. err = -EINVAL;
  806. goto error;
  807. case UDP_ENCAP_ESPINUDP:
  808. x->props.header_len += sizeof(struct udphdr);
  809. break;
  810. case UDP_ENCAP_ESPINUDP_NON_IKE:
  811. x->props.header_len += sizeof(struct udphdr) + 2 * sizeof(u32);
  812. break;
  813. }
  814. }
  815. align = ALIGN(crypto_aead_blocksize(aead), 4);
  816. x->props.trailer_len = align + 1 + crypto_aead_authsize(aead);
  817. error:
  818. return err;
  819. }
  820. static int esp4_rcv_cb(struct sk_buff *skb, int err)
  821. {
  822. return 0;
  823. }
  824. static const struct xfrm_type esp_type =
  825. {
  826. .description = "ESP4",
  827. .owner = THIS_MODULE,
  828. .proto = IPPROTO_ESP,
  829. .flags = XFRM_TYPE_REPLAY_PROT,
  830. .init_state = esp_init_state,
  831. .destructor = esp_destroy,
  832. .get_mtu = esp4_get_mtu,
  833. .input = esp_input,
  834. .output = esp_output,
  835. };
  836. static struct xfrm4_protocol esp4_protocol = {
  837. .handler = xfrm4_rcv,
  838. .input_handler = xfrm_input,
  839. .cb_handler = esp4_rcv_cb,
  840. .err_handler = esp4_err,
  841. .priority = 0,
  842. };
  843. static int __init esp4_init(void)
  844. {
  845. if (xfrm_register_type(&esp_type, AF_INET) < 0) {
  846. pr_info("%s: can't add xfrm type\n", __func__);
  847. return -EAGAIN;
  848. }
  849. if (xfrm4_protocol_register(&esp4_protocol, IPPROTO_ESP) < 0) {
  850. pr_info("%s: can't add protocol\n", __func__);
  851. xfrm_unregister_type(&esp_type, AF_INET);
  852. return -EAGAIN;
  853. }
  854. return 0;
  855. }
  856. static void __exit esp4_fini(void)
  857. {
  858. if (xfrm4_protocol_deregister(&esp4_protocol, IPPROTO_ESP) < 0)
  859. pr_info("%s: can't remove protocol\n", __func__);
  860. if (xfrm_unregister_type(&esp_type, AF_INET) < 0)
  861. pr_info("%s: can't remove xfrm type\n", __func__);
  862. }
  863. module_init(esp4_init);
  864. module_exit(esp4_fini);
  865. MODULE_LICENSE("GPL");
  866. MODULE_ALIAS_XFRM_TYPE(AF_INET, XFRM_PROTO_ESP);