cipher-gcm-ppc.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549
  1. /* cipher-gcm-ppc.c - Power 8 vpmsum accelerated Galois Counter Mode
  2. * implementation
  3. * Copyright (C) 2019 Shawn Landden <shawn@git.icu>
  4. *
  5. * This file is part of Libgcrypt.
  6. *
  7. * Libgcrypt is free software; you can redistribute it and/or modify
  8. * it under the terms of the GNU Lesser General Public License as
  9. * published by the Free Software Foundation; either version 2.1 of
  10. * the License, or (at your option) any later version.
  11. *
  12. * Libgcrypt is distributed in the hope that it will be useful,
  13. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. * GNU Lesser General Public License for more details.
  16. *
  17. * You should have received a copy of the GNU Lesser General Public
  18. * License along with this program; if not, see <http://www.gnu.org/licenses/>.
  19. *
  20. * Based on GHASH implementation by Andy Polyakov from CRYPTOGAMS
  21. * distribution (ppc/ghashp8-ppc.pl). Specifically, it uses his register
  22. * allocation (which then defers to your compiler's register allocation),
  23. * instead of re-implementing Gerald Estrin's Scheme of parallelized
  24. * multiplication of polynomials, as I did not understand this algorithm at
  25. * the time.
  26. *
  27. * Original copyright license follows:
  28. *
  29. * Copyright (c) 2006, CRYPTOGAMS by <appro@openssl.org>
  30. * All rights reserved.
  31. *
  32. * Redistribution and use in source and binary forms, with or without
  33. * modification, are permitted provided that the following conditions
  34. * are met:
  35. *
  36. * * Redistributions of source code must retain copyright notices,
  37. * this list of conditions and the following disclaimer.
  38. *
  39. * * Redistributions in binary form must reproduce the above
  40. * copyright notice, this list of conditions and the following
  41. * disclaimer in the documentation and/or other materials
  42. * provided with the distribution.
  43. *
  44. * * Neither the name of the CRYPTOGAMS nor the names of its
  45. * copyright holder and contributors may be used to endorse or
  46. * promote products derived from this software without specific
  47. * prior written permission.
  48. *
  49. * ALTERNATIVELY, provided that this notice is retained in full, this
  50. * product may be distributed under the terms of the GNU General Public
  51. * License (GPL), in which case the provisions of the GPL apply INSTEAD OF
  52. * those given above.
  53. *
  54. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
  55. * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  56. * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  57. * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  58. * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  59. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  60. * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  61. * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  62. * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  63. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  64. * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  65. *
  66. * SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
  67. */
  68. #include <config.h>
  69. #include <stdio.h>
  70. #include <stdlib.h>
  71. #include <string.h>
  72. #include <errno.h>
  73. #include <stdint.h>
  74. #include "g10lib.h"
  75. #include "cipher.h"
  76. #include "bufhelp.h"
  77. #include "./cipher-internal.h"
  78. #ifdef GCM_USE_PPC_VPMSUM
  79. #include <altivec.h>
  80. #define ALWAYS_INLINE inline __attribute__((always_inline))
  81. #define NO_INSTRUMENT_FUNCTION __attribute__((no_instrument_function))
  82. #define ASM_FUNC_ATTR NO_INSTRUMENT_FUNCTION
  83. #define ASM_FUNC_ATTR_INLINE ASM_FUNC_ATTR ALWAYS_INLINE
  84. #define ALIGNED_16 __attribute__ ((aligned (16)))
  85. typedef vector unsigned char vector16x_u8;
  86. typedef vector signed char vector16x_s8;
  87. typedef vector unsigned long long vector2x_u64;
  88. typedef vector unsigned long long block;
  89. static ASM_FUNC_ATTR_INLINE block
  90. asm_xor(block a, block b)
  91. {
  92. block r;
  93. __asm__ volatile ("xxlxor %x0, %x1, %x2"
  94. : "=wa" (r)
  95. : "wa" (a), "wa" (b));
  96. return r;
  97. }
  98. static ASM_FUNC_ATTR_INLINE block
  99. asm_vpmsumd(block a, block b)
  100. {
  101. block r;
  102. __asm__ volatile ("vpmsumd %0, %1, %2"
  103. : "=v" (r)
  104. : "v" (a), "v" (b));
  105. return r;
  106. }
  107. static ASM_FUNC_ATTR_INLINE block
  108. asm_swap_u64(block a)
  109. {
  110. block r;
  111. __asm__ volatile ("xxswapd %x0, %x1"
  112. : "=wa" (r)
  113. : "wa" (a));
  114. return r;
  115. }
  116. static ASM_FUNC_ATTR_INLINE block
  117. asm_mergelo(block l, block r)
  118. {
  119. block ret;
  120. __asm__ volatile ("xxmrgld %x0, %x1, %x2\n\t"
  121. : "=wa" (ret)
  122. : "wa" (l), "wa" (r));
  123. return ret;
  124. }
  125. static ASM_FUNC_ATTR_INLINE block
  126. asm_mergehi(block l, block r)
  127. {
  128. block ret;
  129. __asm__ volatile ("xxmrghd %x0, %x1, %x2\n\t"
  130. : "=wa" (ret)
  131. : "wa" (l), "wa" (r));
  132. return ret;
  133. }
  134. static ASM_FUNC_ATTR_INLINE block
  135. asm_rot_block_left(block a)
  136. {
  137. block r;
  138. block zero = { 0, 0 };
  139. __asm__ volatile ("xxmrgld %x0, %x1, %x2"
  140. : "=wa" (r)
  141. : "wa" (a), "wa" (zero));
  142. return r;
  143. }
  144. static ASM_FUNC_ATTR_INLINE block
  145. asm_rot_block_right(block a)
  146. {
  147. block r;
  148. block zero = { 0, 0 };
  149. __asm__ volatile ("xxsldwi %x0, %x2, %x1, 2"
  150. : "=wa" (r)
  151. : "wa" (a), "wa" (zero));
  152. return r;
  153. }
  154. /* vsl is a slightly strange function in the way the shift is passed... */
  155. static ASM_FUNC_ATTR_INLINE block
  156. asm_ashl_128(block a, vector16x_u8 shift)
  157. {
  158. block r;
  159. __asm__ volatile ("vsl %0, %1, %2"
  160. : "=v" (r)
  161. : "v" (a), "v" (shift));
  162. return r;
  163. }
  164. #define STORE_TABLE(gcm_table, slot, vec) \
  165. vec_store_he (((block)vec), slot * 16, (unsigned char *)(gcm_table));
  166. static ASM_FUNC_ATTR_INLINE void
  167. vec_store_he(block vec, unsigned long offset, unsigned char *ptr)
  168. {
  169. /* GCC vec_vsx_ld is generating two instructions on little-endian. Use
  170. * lxvd2x directly instead. */
  171. #if __GNUC__ >= 4
  172. if (__builtin_constant_p (offset) && offset == 0)
  173. __asm__ volatile ("stxvd2x %x0, 0, %1\n\t"
  174. :
  175. : "wa" (vec), "r" ((uintptr_t)ptr)
  176. : "memory", "r0");
  177. else
  178. #endif
  179. __asm__ volatile ("stxvd2x %x0, %1, %2\n\t"
  180. :
  181. : "wa" (vec), "r" (offset), "r" ((uintptr_t)ptr)
  182. : "memory", "r0");
  183. }
  184. #define VEC_LOAD_BE(in_ptr, bswap_const) \
  185. vec_be_swap(vec_load_he (0, (const unsigned char *)(in_ptr)), bswap_const)
  186. static ASM_FUNC_ATTR_INLINE block
  187. vec_load_he(unsigned long offset, const unsigned char *ptr)
  188. {
  189. block vec;
  190. /* GCC vec_vsx_ld is generating two instructions on little-endian. Use
  191. * lxvd2x directly instead. */
  192. #if __GNUC__ >= 4
  193. if (__builtin_constant_p (offset) && offset == 0)
  194. __asm__ volatile ("lxvd2x %x0, 0, %1\n\t"
  195. : "=wa" (vec)
  196. : "r" ((uintptr_t)ptr)
  197. : "memory", "r0");
  198. else
  199. #endif
  200. __asm__ volatile ("lxvd2x %x0, %1, %2\n\t"
  201. : "=wa" (vec)
  202. : "r" (offset), "r" ((uintptr_t)ptr)
  203. : "memory", "r0");
  204. return vec;
  205. }
  206. static ASM_FUNC_ATTR_INLINE block
  207. vec_be_swap(block vec, vector16x_u8 be_bswap_const)
  208. {
  209. #ifndef WORDS_BIGENDIAN
  210. __asm__ volatile ("vperm %0, %1, %1, %2\n\t"
  211. : "=v" (vec)
  212. : "v" (vec), "v" (be_bswap_const));
  213. #else
  214. (void)be_bswap_const;
  215. #endif
  216. return vec;
  217. }
  218. static ASM_FUNC_ATTR_INLINE block
  219. vec_dup_byte_elem(block vec, int idx)
  220. {
  221. #ifndef WORDS_BIGENDIAN
  222. return (block)vec_splat((vector16x_s8)vec, idx);
  223. #else
  224. return (block)vec_splat((vector16x_s8)vec, (15 - idx) & 15);
  225. #endif
  226. }
  227. /* Power ghash based on papers:
  228. "The Galois/Counter Mode of Operation (GCM)"; David A. McGrew, John Viega
  229. "Intel® Carry-Less Multiplication Instruction and its Usage for Computing
  230. the GCM Mode - Rev 2.01"; Shay Gueron, Michael E. Kounavis.
  231. After saving the magic c2 constant and pre-formatted version of the key,
  232. we pre-process the key for parallel hashing. This takes advantage of the
  233. identity of addition over a galois field being identital to XOR, and thus
  234. can be parellized (S 2.2, page 3). We multiply and add (galois field
  235. versions) the key over multiple iterations and save the result. This can
  236. later be galois added (XORed) with parallel processed input (Estrin's
  237. Scheme).
  238. The ghash "key" is a salt. */
  239. void ASM_FUNC_ATTR
  240. _gcry_ghash_setup_ppc_vpmsum (void *gcm_table_arg, void *gcm_key)
  241. {
  242. static const vector16x_u8 bswap_const ALIGNED_16 =
  243. { ~7, ~6, ~5, ~4, ~3, ~2, ~1, ~0, ~15, ~14, ~13, ~12, ~11, ~10, ~9, ~8 };
  244. static const byte c2[16] ALIGNED_16 =
  245. { 0xc2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 };
  246. static const vector16x_u8 one ALIGNED_16 =
  247. { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
  248. uint64_t *gcm_table = gcm_table_arg;
  249. block T0, T1, T2;
  250. block C2, H, H1, H1l, H1h, H2, H2l, H2h;
  251. block H3l, H3, H3h, H4l, H4, H4h, T3, T4;
  252. vector16x_s8 most_sig_of_H, t7, carry;
  253. H = VEC_LOAD_BE(gcm_key, bswap_const);
  254. C2 = VEC_LOAD_BE(c2, bswap_const);
  255. most_sig_of_H = (vector16x_s8)vec_dup_byte_elem(H, 15);
  256. t7 = vec_splat_s8(7);
  257. carry = most_sig_of_H >> t7;
  258. carry &= (vector16x_s8)C2; /* only interested in certain carries. */
  259. H1 = asm_ashl_128(H, one);
  260. H1 ^= (block)carry; /* complete the <<< 1 */
  261. T1 = asm_swap_u64 (H1);
  262. H1l = asm_rot_block_right (T1);
  263. H1h = asm_rot_block_left (T1);
  264. C2 = asm_rot_block_right (C2);
  265. STORE_TABLE (gcm_table, 0, C2);
  266. STORE_TABLE (gcm_table, 1, H1l);
  267. STORE_TABLE (gcm_table, 2, T1);
  268. STORE_TABLE (gcm_table, 3, H1h);
  269. /* pre-process coefficients for Gerald Estrin's scheme for parallel
  270. * multiplication of polynomials
  271. */
  272. H2l = asm_vpmsumd (H1l, H1); /* do not need to mask in
  273. because 0 * anything -> 0 */
  274. H2 = asm_vpmsumd (T1, H1);
  275. H2h = asm_vpmsumd (H1h, H1);
  276. /* reduce 1 */
  277. T0 = asm_vpmsumd (H2l, C2);
  278. H2l ^= asm_rot_block_left (H2);
  279. H2h ^= asm_rot_block_right (H2);
  280. H2l = asm_swap_u64 (H2l);
  281. H2l ^= T0;
  282. /* reduce 2 */
  283. T0 = asm_swap_u64 (H2l);
  284. H2l = asm_vpmsumd (H2l, C2);
  285. H2 = H2l ^ H2h ^ T0;
  286. T2 = asm_swap_u64 (H2);
  287. H2l = asm_rot_block_right (T2);
  288. H2h = asm_rot_block_left (T2);
  289. STORE_TABLE (gcm_table, 4, H2l);
  290. STORE_TABLE (gcm_table, 5, T2);
  291. STORE_TABLE (gcm_table, 6, H2h);
  292. H3l = asm_vpmsumd (H2l, H1);
  293. H4l = asm_vpmsumd (H2l, H2);
  294. H3 = asm_vpmsumd (T2, H1);
  295. H4 = asm_vpmsumd (T2, H2);
  296. H3h = asm_vpmsumd (H2h, H1);
  297. H4h = asm_vpmsumd (H2h, H2);
  298. T3 = asm_vpmsumd (H3l, C2);
  299. T4 = asm_vpmsumd (H4l, C2);
  300. H3l ^= asm_rot_block_left (H3);
  301. H3h ^= asm_rot_block_right (H3);
  302. H4l ^= asm_rot_block_left (H4);
  303. H4h ^= asm_rot_block_right (H4);
  304. H3 = asm_swap_u64 (H3l);
  305. H4 = asm_swap_u64 (H4l);
  306. H3 ^= T3;
  307. H4 ^= T4;
  308. /* We could have also b64 switched reduce and reduce2, however as we are
  309. using the unrotated H and H2 above to vpmsum, this is marginally better. */
  310. T3 = asm_swap_u64 (H3);
  311. T4 = asm_swap_u64 (H4);
  312. H3 = asm_vpmsumd (H3, C2);
  313. H4 = asm_vpmsumd (H4, C2);
  314. T3 ^= H3h;
  315. T4 ^= H4h;
  316. H3 ^= T3;
  317. H4 ^= T4;
  318. H3 = asm_swap_u64 (H3);
  319. H4 = asm_swap_u64 (H4);
  320. H3l = asm_rot_block_right (H3);
  321. H3h = asm_rot_block_left (H3);
  322. H4l = asm_rot_block_right (H4);
  323. H4h = asm_rot_block_left (H4);
  324. STORE_TABLE (gcm_table, 7, H3l);
  325. STORE_TABLE (gcm_table, 8, H3);
  326. STORE_TABLE (gcm_table, 9, H3h);
  327. STORE_TABLE (gcm_table, 10, H4l);
  328. STORE_TABLE (gcm_table, 11, H4);
  329. STORE_TABLE (gcm_table, 12, H4h);
  330. }
  331. unsigned int ASM_FUNC_ATTR
  332. _gcry_ghash_ppc_vpmsum (byte *result, void *gcm_table,
  333. const byte *buf, const size_t nblocks)
  334. {
  335. static const vector16x_u8 bswap_const ALIGNED_16 =
  336. { ~7, ~6, ~5, ~4, ~3, ~2, ~1, ~0, ~15, ~14, ~13, ~12, ~11, ~10, ~9, ~8 };
  337. block c2, H0l, H0m, H0h, H4l, H4m, H4h, H2m, H3l, H3m, H3h, Hl;
  338. block Hm, Hh, in, in0, in1, in2, in3, Hm_right, Hl_rotate, cur;
  339. size_t blocks_remaining = nblocks;
  340. size_t not_multiple_of_four;
  341. block t0;
  342. cur = vec_be_swap (vec_load_he (0, result), bswap_const);
  343. c2 = vec_load_he (0, gcm_table);
  344. H0l = vec_load_he (16, gcm_table);
  345. H0m = vec_load_he (32, gcm_table);
  346. H0h = vec_load_he (48, gcm_table);
  347. for (not_multiple_of_four = nblocks % 4; not_multiple_of_four;
  348. not_multiple_of_four--)
  349. {
  350. in = vec_be_swap (vec_load_he (0, buf), bswap_const);
  351. buf += 16;
  352. blocks_remaining--;
  353. cur ^= in;
  354. Hl = asm_vpmsumd (cur, H0l);
  355. Hm = asm_vpmsumd (cur, H0m);
  356. Hh = asm_vpmsumd (cur, H0h);
  357. t0 = asm_vpmsumd (Hl, c2);
  358. Hl ^= asm_rot_block_left (Hm);
  359. Hm_right = asm_rot_block_right (Hm);
  360. Hh ^= Hm_right;
  361. Hl_rotate = asm_swap_u64 (Hl);
  362. Hl_rotate ^= t0;
  363. Hl = asm_swap_u64 (Hl_rotate);
  364. Hl_rotate = asm_vpmsumd (Hl_rotate, c2);
  365. Hl ^= Hh;
  366. Hl ^= Hl_rotate;
  367. cur = Hl;
  368. }
  369. if (blocks_remaining > 0)
  370. {
  371. block Xl, Xm, Xh, Xl1, Xm1, Xh1, Xm2, Xl3, Xm3, Xh3, Xl_rotate;
  372. block H21l, H21h, merge_l, merge_h;
  373. block t1, t2;
  374. H2m = vec_load_he (48 + 32, gcm_table);
  375. H3l = vec_load_he (48 * 2 + 16, gcm_table);
  376. H3m = vec_load_he (48 * 2 + 32, gcm_table);
  377. H3h = vec_load_he (48 * 2 + 48, gcm_table);
  378. H4l = vec_load_he (48 * 3 + 16, gcm_table);
  379. H4m = vec_load_he (48 * 3 + 32, gcm_table);
  380. H4h = vec_load_he (48 * 3 + 48, gcm_table);
  381. in0 = vec_load_he (0, buf);
  382. in1 = vec_load_he (16, buf);
  383. in2 = vec_load_he (32, buf);
  384. in3 = vec_load_he (48, buf);
  385. buf += 64;
  386. in0 = vec_be_swap(in0, bswap_const);
  387. in1 = vec_be_swap(in1, bswap_const);
  388. in2 = vec_be_swap(in2, bswap_const);
  389. in3 = vec_be_swap(in3, bswap_const);
  390. Xh = asm_xor (in0, cur);
  391. Xl1 = asm_vpmsumd (in1, H3l);
  392. Xm1 = asm_vpmsumd (in1, H3m);
  393. Xh1 = asm_vpmsumd (in1, H3h);
  394. H21l = asm_mergehi (H2m, H0m);
  395. H21h = asm_mergelo (H2m, H0m);
  396. merge_l = asm_mergelo (in2, in3);
  397. merge_h = asm_mergehi (in2, in3);
  398. Xm2 = asm_vpmsumd (in2, H2m);
  399. Xl3 = asm_vpmsumd (merge_l, H21l);
  400. Xm3 = asm_vpmsumd (in3, H0m);
  401. Xh3 = asm_vpmsumd (merge_h, H21h);
  402. Xm2 = asm_xor (Xm2, Xm1);
  403. Xl3 = asm_xor (Xl3, Xl1);
  404. Xm3 = asm_xor (Xm3, Xm2);
  405. Xh3 = asm_xor (Xh3, Xh1);
  406. /* Gerald Estrin's scheme for parallel multiplication of polynomials */
  407. for (; blocks_remaining > 4; blocks_remaining -= 4)
  408. {
  409. in0 = vec_load_he (0, buf);
  410. in1 = vec_load_he (16, buf);
  411. in2 = vec_load_he (32, buf);
  412. in3 = vec_load_he (48, buf);
  413. buf += 64;
  414. in1 = vec_be_swap(in1, bswap_const);
  415. in2 = vec_be_swap(in2, bswap_const);
  416. in3 = vec_be_swap(in3, bswap_const);
  417. in0 = vec_be_swap(in0, bswap_const);
  418. Xl = asm_vpmsumd (Xh, H4l);
  419. Xm = asm_vpmsumd (Xh, H4m);
  420. Xh = asm_vpmsumd (Xh, H4h);
  421. Xl1 = asm_vpmsumd (in1, H3l);
  422. Xm1 = asm_vpmsumd (in1, H3m);
  423. Xh1 = asm_vpmsumd (in1, H3h);
  424. Xl = asm_xor (Xl, Xl3);
  425. Xm = asm_xor (Xm, Xm3);
  426. Xh = asm_xor (Xh, Xh3);
  427. merge_l = asm_mergelo (in2, in3);
  428. merge_h = asm_mergehi (in2, in3);
  429. t0 = asm_vpmsumd (Xl, c2);
  430. Xl3 = asm_vpmsumd (merge_l, H21l);
  431. Xh3 = asm_vpmsumd (merge_h, H21h);
  432. t1 = asm_rot_block_left (Xm);
  433. t2 = asm_rot_block_right (Xm);
  434. Xl = asm_xor(Xl, t1);
  435. Xh = asm_xor(Xh, t2);
  436. Xl = asm_swap_u64 (Xl);
  437. Xl = asm_xor(Xl, t0);
  438. Xl_rotate = asm_swap_u64 (Xl);
  439. Xm2 = asm_vpmsumd (in2, H2m);
  440. Xm3 = asm_vpmsumd (in3, H0m);
  441. Xl = asm_vpmsumd (Xl, c2);
  442. Xl3 = asm_xor (Xl3, Xl1);
  443. Xh3 = asm_xor (Xh3, Xh1);
  444. Xh = asm_xor (Xh, in0);
  445. Xm2 = asm_xor (Xm2, Xm1);
  446. Xh = asm_xor (Xh, Xl_rotate);
  447. Xm3 = asm_xor (Xm3, Xm2);
  448. Xh = asm_xor (Xh, Xl);
  449. }
  450. Xl = asm_vpmsumd (Xh, H4l);
  451. Xm = asm_vpmsumd (Xh, H4m);
  452. Xh = asm_vpmsumd (Xh, H4h);
  453. Xl = asm_xor (Xl, Xl3);
  454. Xm = asm_xor (Xm, Xm3);
  455. t0 = asm_vpmsumd (Xl, c2);
  456. Xh = asm_xor (Xh, Xh3);
  457. t1 = asm_rot_block_left (Xm);
  458. t2 = asm_rot_block_right (Xm);
  459. Xl = asm_xor (Xl, t1);
  460. Xh = asm_xor (Xh, t2);
  461. Xl = asm_swap_u64 (Xl);
  462. Xl = asm_xor (Xl, t0);
  463. Xl_rotate = asm_swap_u64 (Xl);
  464. Xl = asm_vpmsumd (Xl, c2);
  465. Xh = asm_xor (Xh, Xl_rotate);
  466. cur = asm_xor (Xh, Xl);
  467. }
  468. vec_store_he (vec_be_swap (cur, bswap_const), 0, result);
  469. return 0;
  470. }
  471. #endif /* GCM_USE_PPC_VPMSUM */