lossless_enc_sse2.c 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712
  1. // Copyright 2015 Google Inc. All Rights Reserved.
  2. //
  3. // Use of this source code is governed by a BSD-style license
  4. // that can be found in the COPYING file in the root of the source
  5. // tree. An additional intellectual property rights grant can be found
  6. // in the file PATENTS. All contributing project authors may
  7. // be found in the AUTHORS file in the root of the source tree.
  8. // -----------------------------------------------------------------------------
  9. //
  10. // SSE2 variant of methods for lossless encoder
  11. //
  12. // Author: Skal (pascal.massimino@gmail.com)
  13. #include "./dsp.h"
  14. #if defined(WEBP_USE_SSE2)
  15. #include <assert.h>
  16. #include <emmintrin.h>
  17. #include "./lossless.h"
  18. #include "./common_sse2.h"
  19. #include "./lossless_common.h"
  20. // For sign-extended multiplying constants, pre-shifted by 5:
  21. #define CST_5b(X) (((int16_t)((uint16_t)X << 8)) >> 5)
  22. //------------------------------------------------------------------------------
  23. // Subtract-Green Transform
  24. static void SubtractGreenFromBlueAndRed(uint32_t* argb_data, int num_pixels) {
  25. int i;
  26. for (i = 0; i + 4 <= num_pixels; i += 4) {
  27. const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb
  28. const __m128i A = _mm_srli_epi16(in, 8); // 0 a 0 g
  29. const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0));
  30. const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // 0g0g
  31. const __m128i out = _mm_sub_epi8(in, C);
  32. _mm_storeu_si128((__m128i*)&argb_data[i], out);
  33. }
  34. // fallthrough and finish off with plain-C
  35. if (i != num_pixels) {
  36. VP8LSubtractGreenFromBlueAndRed_C(argb_data + i, num_pixels - i);
  37. }
  38. }
  39. //------------------------------------------------------------------------------
  40. // Color Transform
  41. static void TransformColor(const VP8LMultipliers* const m,
  42. uint32_t* argb_data, int num_pixels) {
  43. const __m128i mults_rb = _mm_set_epi16(
  44. CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_),
  45. CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_),
  46. CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_),
  47. CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_));
  48. const __m128i mults_b2 = _mm_set_epi16(
  49. CST_5b(m->red_to_blue_), 0, CST_5b(m->red_to_blue_), 0,
  50. CST_5b(m->red_to_blue_), 0, CST_5b(m->red_to_blue_), 0);
  51. const __m128i mask_ag = _mm_set1_epi32(0xff00ff00); // alpha-green masks
  52. const __m128i mask_rb = _mm_set1_epi32(0x00ff00ff); // red-blue masks
  53. int i;
  54. for (i = 0; i + 4 <= num_pixels; i += 4) {
  55. const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb
  56. const __m128i A = _mm_and_si128(in, mask_ag); // a 0 g 0
  57. const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0));
  58. const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // g0g0
  59. const __m128i D = _mm_mulhi_epi16(C, mults_rb); // x dr x db1
  60. const __m128i E = _mm_slli_epi16(in, 8); // r 0 b 0
  61. const __m128i F = _mm_mulhi_epi16(E, mults_b2); // x db2 0 0
  62. const __m128i G = _mm_srli_epi32(F, 16); // 0 0 x db2
  63. const __m128i H = _mm_add_epi8(G, D); // x dr x db
  64. const __m128i I = _mm_and_si128(H, mask_rb); // 0 dr 0 db
  65. const __m128i out = _mm_sub_epi8(in, I);
  66. _mm_storeu_si128((__m128i*)&argb_data[i], out);
  67. }
  68. // fallthrough and finish off with plain-C
  69. if (i != num_pixels) {
  70. VP8LTransformColor_C(m, argb_data + i, num_pixels - i);
  71. }
  72. }
  73. //------------------------------------------------------------------------------
  74. #define SPAN 8
  75. static void CollectColorBlueTransforms(const uint32_t* argb, int stride,
  76. int tile_width, int tile_height,
  77. int green_to_blue, int red_to_blue,
  78. int histo[]) {
  79. const __m128i mults_r = _mm_set_epi16(
  80. CST_5b(red_to_blue), 0, CST_5b(red_to_blue), 0,
  81. CST_5b(red_to_blue), 0, CST_5b(red_to_blue), 0);
  82. const __m128i mults_g = _mm_set_epi16(
  83. 0, CST_5b(green_to_blue), 0, CST_5b(green_to_blue),
  84. 0, CST_5b(green_to_blue), 0, CST_5b(green_to_blue));
  85. const __m128i mask_g = _mm_set1_epi32(0x00ff00); // green mask
  86. const __m128i mask_b = _mm_set1_epi32(0x0000ff); // blue mask
  87. int y;
  88. for (y = 0; y < tile_height; ++y) {
  89. const uint32_t* const src = argb + y * stride;
  90. int i, x;
  91. for (x = 0; x + SPAN <= tile_width; x += SPAN) {
  92. uint16_t values[SPAN];
  93. const __m128i in0 = _mm_loadu_si128((__m128i*)&src[x + 0]);
  94. const __m128i in1 = _mm_loadu_si128((__m128i*)&src[x + SPAN / 2]);
  95. const __m128i A0 = _mm_slli_epi16(in0, 8); // r 0 | b 0
  96. const __m128i A1 = _mm_slli_epi16(in1, 8);
  97. const __m128i B0 = _mm_and_si128(in0, mask_g); // 0 0 | g 0
  98. const __m128i B1 = _mm_and_si128(in1, mask_g);
  99. const __m128i C0 = _mm_mulhi_epi16(A0, mults_r); // x db | 0 0
  100. const __m128i C1 = _mm_mulhi_epi16(A1, mults_r);
  101. const __m128i D0 = _mm_mulhi_epi16(B0, mults_g); // 0 0 | x db
  102. const __m128i D1 = _mm_mulhi_epi16(B1, mults_g);
  103. const __m128i E0 = _mm_sub_epi8(in0, D0); // x x | x b'
  104. const __m128i E1 = _mm_sub_epi8(in1, D1);
  105. const __m128i F0 = _mm_srli_epi32(C0, 16); // 0 0 | x db
  106. const __m128i F1 = _mm_srli_epi32(C1, 16);
  107. const __m128i G0 = _mm_sub_epi8(E0, F0); // 0 0 | x b'
  108. const __m128i G1 = _mm_sub_epi8(E1, F1);
  109. const __m128i H0 = _mm_and_si128(G0, mask_b); // 0 0 | 0 b
  110. const __m128i H1 = _mm_and_si128(G1, mask_b);
  111. const __m128i I = _mm_packs_epi32(H0, H1); // 0 b' | 0 b'
  112. _mm_storeu_si128((__m128i*)values, I);
  113. for (i = 0; i < SPAN; ++i) ++histo[values[i]];
  114. }
  115. }
  116. {
  117. const int left_over = tile_width & (SPAN - 1);
  118. if (left_over > 0) {
  119. VP8LCollectColorBlueTransforms_C(argb + tile_width - left_over, stride,
  120. left_over, tile_height,
  121. green_to_blue, red_to_blue, histo);
  122. }
  123. }
  124. }
  125. static void CollectColorRedTransforms(const uint32_t* argb, int stride,
  126. int tile_width, int tile_height,
  127. int green_to_red, int histo[]) {
  128. const __m128i mults_g = _mm_set_epi16(
  129. 0, CST_5b(green_to_red), 0, CST_5b(green_to_red),
  130. 0, CST_5b(green_to_red), 0, CST_5b(green_to_red));
  131. const __m128i mask_g = _mm_set1_epi32(0x00ff00); // green mask
  132. const __m128i mask = _mm_set1_epi32(0xff);
  133. int y;
  134. for (y = 0; y < tile_height; ++y) {
  135. const uint32_t* const src = argb + y * stride;
  136. int i, x;
  137. for (x = 0; x + SPAN <= tile_width; x += SPAN) {
  138. uint16_t values[SPAN];
  139. const __m128i in0 = _mm_loadu_si128((__m128i*)&src[x + 0]);
  140. const __m128i in1 = _mm_loadu_si128((__m128i*)&src[x + SPAN / 2]);
  141. const __m128i A0 = _mm_and_si128(in0, mask_g); // 0 0 | g 0
  142. const __m128i A1 = _mm_and_si128(in1, mask_g);
  143. const __m128i B0 = _mm_srli_epi32(in0, 16); // 0 0 | x r
  144. const __m128i B1 = _mm_srli_epi32(in1, 16);
  145. const __m128i C0 = _mm_mulhi_epi16(A0, mults_g); // 0 0 | x dr
  146. const __m128i C1 = _mm_mulhi_epi16(A1, mults_g);
  147. const __m128i E0 = _mm_sub_epi8(B0, C0); // x x | x r'
  148. const __m128i E1 = _mm_sub_epi8(B1, C1);
  149. const __m128i F0 = _mm_and_si128(E0, mask); // 0 0 | 0 r'
  150. const __m128i F1 = _mm_and_si128(E1, mask);
  151. const __m128i I = _mm_packs_epi32(F0, F1);
  152. _mm_storeu_si128((__m128i*)values, I);
  153. for (i = 0; i < SPAN; ++i) ++histo[values[i]];
  154. }
  155. }
  156. {
  157. const int left_over = tile_width & (SPAN - 1);
  158. if (left_over > 0) {
  159. VP8LCollectColorRedTransforms_C(argb + tile_width - left_over, stride,
  160. left_over, tile_height,
  161. green_to_red, histo);
  162. }
  163. }
  164. }
  165. #undef SPAN
  166. //------------------------------------------------------------------------------
  167. #define LINE_SIZE 16 // 8 or 16
  168. static void AddVector(const uint32_t* a, const uint32_t* b, uint32_t* out,
  169. int size) {
  170. int i;
  171. assert(size % LINE_SIZE == 0);
  172. for (i = 0; i < size; i += LINE_SIZE) {
  173. const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i + 0]);
  174. const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]);
  175. #if (LINE_SIZE == 16)
  176. const __m128i a2 = _mm_loadu_si128((const __m128i*)&a[i + 8]);
  177. const __m128i a3 = _mm_loadu_si128((const __m128i*)&a[i + 12]);
  178. #endif
  179. const __m128i b0 = _mm_loadu_si128((const __m128i*)&b[i + 0]);
  180. const __m128i b1 = _mm_loadu_si128((const __m128i*)&b[i + 4]);
  181. #if (LINE_SIZE == 16)
  182. const __m128i b2 = _mm_loadu_si128((const __m128i*)&b[i + 8]);
  183. const __m128i b3 = _mm_loadu_si128((const __m128i*)&b[i + 12]);
  184. #endif
  185. _mm_storeu_si128((__m128i*)&out[i + 0], _mm_add_epi32(a0, b0));
  186. _mm_storeu_si128((__m128i*)&out[i + 4], _mm_add_epi32(a1, b1));
  187. #if (LINE_SIZE == 16)
  188. _mm_storeu_si128((__m128i*)&out[i + 8], _mm_add_epi32(a2, b2));
  189. _mm_storeu_si128((__m128i*)&out[i + 12], _mm_add_epi32(a3, b3));
  190. #endif
  191. }
  192. }
  193. static void AddVectorEq(const uint32_t* a, uint32_t* out, int size) {
  194. int i;
  195. assert(size % LINE_SIZE == 0);
  196. for (i = 0; i < size; i += LINE_SIZE) {
  197. const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i + 0]);
  198. const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]);
  199. #if (LINE_SIZE == 16)
  200. const __m128i a2 = _mm_loadu_si128((const __m128i*)&a[i + 8]);
  201. const __m128i a3 = _mm_loadu_si128((const __m128i*)&a[i + 12]);
  202. #endif
  203. const __m128i b0 = _mm_loadu_si128((const __m128i*)&out[i + 0]);
  204. const __m128i b1 = _mm_loadu_si128((const __m128i*)&out[i + 4]);
  205. #if (LINE_SIZE == 16)
  206. const __m128i b2 = _mm_loadu_si128((const __m128i*)&out[i + 8]);
  207. const __m128i b3 = _mm_loadu_si128((const __m128i*)&out[i + 12]);
  208. #endif
  209. _mm_storeu_si128((__m128i*)&out[i + 0], _mm_add_epi32(a0, b0));
  210. _mm_storeu_si128((__m128i*)&out[i + 4], _mm_add_epi32(a1, b1));
  211. #if (LINE_SIZE == 16)
  212. _mm_storeu_si128((__m128i*)&out[i + 8], _mm_add_epi32(a2, b2));
  213. _mm_storeu_si128((__m128i*)&out[i + 12], _mm_add_epi32(a3, b3));
  214. #endif
  215. }
  216. }
  217. #undef LINE_SIZE
  218. // Note we are adding uint32_t's as *signed* int32's (using _mm_add_epi32). But
  219. // that's ok since the histogram values are less than 1<<28 (max picture size).
  220. static void HistogramAdd(const VP8LHistogram* const a,
  221. const VP8LHistogram* const b,
  222. VP8LHistogram* const out) {
  223. int i;
  224. const int literal_size = VP8LHistogramNumCodes(a->palette_code_bits_);
  225. assert(a->palette_code_bits_ == b->palette_code_bits_);
  226. if (b != out) {
  227. AddVector(a->literal_, b->literal_, out->literal_, NUM_LITERAL_CODES);
  228. AddVector(a->red_, b->red_, out->red_, NUM_LITERAL_CODES);
  229. AddVector(a->blue_, b->blue_, out->blue_, NUM_LITERAL_CODES);
  230. AddVector(a->alpha_, b->alpha_, out->alpha_, NUM_LITERAL_CODES);
  231. } else {
  232. AddVectorEq(a->literal_, out->literal_, NUM_LITERAL_CODES);
  233. AddVectorEq(a->red_, out->red_, NUM_LITERAL_CODES);
  234. AddVectorEq(a->blue_, out->blue_, NUM_LITERAL_CODES);
  235. AddVectorEq(a->alpha_, out->alpha_, NUM_LITERAL_CODES);
  236. }
  237. for (i = NUM_LITERAL_CODES; i < literal_size; ++i) {
  238. out->literal_[i] = a->literal_[i] + b->literal_[i];
  239. }
  240. for (i = 0; i < NUM_DISTANCE_CODES; ++i) {
  241. out->distance_[i] = a->distance_[i] + b->distance_[i];
  242. }
  243. }
  244. //------------------------------------------------------------------------------
  245. // Entropy
  246. // Checks whether the X or Y contribution is worth computing and adding.
  247. // Used in loop unrolling.
  248. #define ANALYZE_X_OR_Y(x_or_y, j) \
  249. do { \
  250. if (x_or_y[i + j] != 0) retval -= VP8LFastSLog2(x_or_y[i + j]); \
  251. } while (0)
  252. // Checks whether the X + Y contribution is worth computing and adding.
  253. // Used in loop unrolling.
  254. #define ANALYZE_XY(j) \
  255. do { \
  256. if (tmp[j] != 0) { \
  257. retval -= VP8LFastSLog2(tmp[j]); \
  258. ANALYZE_X_OR_Y(X, j); \
  259. } \
  260. } while (0)
  261. static float CombinedShannonEntropy(const int X[256], const int Y[256]) {
  262. int i;
  263. double retval = 0.;
  264. int sumX, sumXY;
  265. int32_t tmp[4];
  266. __m128i zero = _mm_setzero_si128();
  267. // Sums up X + Y, 4 ints at a time (and will merge it at the end for sumXY).
  268. __m128i sumXY_128 = zero;
  269. __m128i sumX_128 = zero;
  270. for (i = 0; i < 256; i += 4) {
  271. const __m128i x = _mm_loadu_si128((const __m128i*)(X + i));
  272. const __m128i y = _mm_loadu_si128((const __m128i*)(Y + i));
  273. // Check if any X is non-zero: this actually provides a speedup as X is
  274. // usually sparse.
  275. if (_mm_movemask_epi8(_mm_cmpeq_epi32(x, zero)) != 0xFFFF) {
  276. const __m128i xy_128 = _mm_add_epi32(x, y);
  277. sumXY_128 = _mm_add_epi32(sumXY_128, xy_128);
  278. sumX_128 = _mm_add_epi32(sumX_128, x);
  279. // Analyze the different X + Y.
  280. _mm_storeu_si128((__m128i*)tmp, xy_128);
  281. ANALYZE_XY(0);
  282. ANALYZE_XY(1);
  283. ANALYZE_XY(2);
  284. ANALYZE_XY(3);
  285. } else {
  286. // X is fully 0, so only deal with Y.
  287. sumXY_128 = _mm_add_epi32(sumXY_128, y);
  288. ANALYZE_X_OR_Y(Y, 0);
  289. ANALYZE_X_OR_Y(Y, 1);
  290. ANALYZE_X_OR_Y(Y, 2);
  291. ANALYZE_X_OR_Y(Y, 3);
  292. }
  293. }
  294. // Sum up sumX_128 to get sumX.
  295. _mm_storeu_si128((__m128i*)tmp, sumX_128);
  296. sumX = tmp[3] + tmp[2] + tmp[1] + tmp[0];
  297. // Sum up sumXY_128 to get sumXY.
  298. _mm_storeu_si128((__m128i*)tmp, sumXY_128);
  299. sumXY = tmp[3] + tmp[2] + tmp[1] + tmp[0];
  300. retval += VP8LFastSLog2(sumX) + VP8LFastSLog2(sumXY);
  301. return (float)retval;
  302. }
  303. #undef ANALYZE_X_OR_Y
  304. #undef ANALYZE_XY
  305. //------------------------------------------------------------------------------
  306. static int VectorMismatch(const uint32_t* const array1,
  307. const uint32_t* const array2, int length) {
  308. int match_len;
  309. if (length >= 12) {
  310. __m128i A0 = _mm_loadu_si128((const __m128i*)&array1[0]);
  311. __m128i A1 = _mm_loadu_si128((const __m128i*)&array2[0]);
  312. match_len = 0;
  313. do {
  314. // Loop unrolling and early load both provide a speedup of 10% for the
  315. // current function. Also, max_limit can be MAX_LENGTH=4096 at most.
  316. const __m128i cmpA = _mm_cmpeq_epi32(A0, A1);
  317. const __m128i B0 =
  318. _mm_loadu_si128((const __m128i*)&array1[match_len + 4]);
  319. const __m128i B1 =
  320. _mm_loadu_si128((const __m128i*)&array2[match_len + 4]);
  321. if (_mm_movemask_epi8(cmpA) != 0xffff) break;
  322. match_len += 4;
  323. {
  324. const __m128i cmpB = _mm_cmpeq_epi32(B0, B1);
  325. A0 = _mm_loadu_si128((const __m128i*)&array1[match_len + 4]);
  326. A1 = _mm_loadu_si128((const __m128i*)&array2[match_len + 4]);
  327. if (_mm_movemask_epi8(cmpB) != 0xffff) break;
  328. match_len += 4;
  329. }
  330. } while (match_len + 12 < length);
  331. } else {
  332. match_len = 0;
  333. // Unroll the potential first two loops.
  334. if (length >= 4 &&
  335. _mm_movemask_epi8(_mm_cmpeq_epi32(
  336. _mm_loadu_si128((const __m128i*)&array1[0]),
  337. _mm_loadu_si128((const __m128i*)&array2[0]))) == 0xffff) {
  338. match_len = 4;
  339. if (length >= 8 &&
  340. _mm_movemask_epi8(_mm_cmpeq_epi32(
  341. _mm_loadu_si128((const __m128i*)&array1[4]),
  342. _mm_loadu_si128((const __m128i*)&array2[4]))) == 0xffff) {
  343. match_len = 8;
  344. }
  345. }
  346. }
  347. while (match_len < length && array1[match_len] == array2[match_len]) {
  348. ++match_len;
  349. }
  350. return match_len;
  351. }
  352. // Bundles multiple (1, 2, 4 or 8) pixels into a single pixel.
  353. static void BundleColorMap_SSE2(const uint8_t* const row, int width, int xbits,
  354. uint32_t* dst) {
  355. int x;
  356. assert(xbits >= 0);
  357. assert(xbits <= 3);
  358. switch (xbits) {
  359. case 0: {
  360. const __m128i ff = _mm_set1_epi16(0xff00);
  361. const __m128i zero = _mm_setzero_si128();
  362. // Store 0xff000000 | (row[x] << 8).
  363. for (x = 0; x + 16 <= width; x += 16, dst += 16) {
  364. const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]);
  365. const __m128i in_lo = _mm_unpacklo_epi8(zero, in);
  366. const __m128i dst0 = _mm_unpacklo_epi16(in_lo, ff);
  367. const __m128i dst1 = _mm_unpackhi_epi16(in_lo, ff);
  368. const __m128i in_hi = _mm_unpackhi_epi8(zero, in);
  369. const __m128i dst2 = _mm_unpacklo_epi16(in_hi, ff);
  370. const __m128i dst3 = _mm_unpackhi_epi16(in_hi, ff);
  371. _mm_storeu_si128((__m128i*)&dst[0], dst0);
  372. _mm_storeu_si128((__m128i*)&dst[4], dst1);
  373. _mm_storeu_si128((__m128i*)&dst[8], dst2);
  374. _mm_storeu_si128((__m128i*)&dst[12], dst3);
  375. }
  376. break;
  377. }
  378. case 1: {
  379. const __m128i ff = _mm_set1_epi16(0xff00);
  380. const __m128i mul = _mm_set1_epi16(0x110);
  381. for (x = 0; x + 16 <= width; x += 16, dst += 8) {
  382. // 0a0b | (where a/b are 4 bits).
  383. const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]);
  384. const __m128i tmp = _mm_mullo_epi16(in, mul); // aba0
  385. const __m128i pack = _mm_and_si128(tmp, ff); // ab00
  386. const __m128i dst0 = _mm_unpacklo_epi16(pack, ff);
  387. const __m128i dst1 = _mm_unpackhi_epi16(pack, ff);
  388. _mm_storeu_si128((__m128i*)&dst[0], dst0);
  389. _mm_storeu_si128((__m128i*)&dst[4], dst1);
  390. }
  391. break;
  392. }
  393. case 2: {
  394. const __m128i mask_or = _mm_set1_epi32(0xff000000);
  395. const __m128i mul_cst = _mm_set1_epi16(0x0104);
  396. const __m128i mask_mul = _mm_set1_epi16(0x0f00);
  397. for (x = 0; x + 16 <= width; x += 16, dst += 4) {
  398. // 000a000b000c000d | (where a/b/c/d are 2 bits).
  399. const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]);
  400. const __m128i mul = _mm_mullo_epi16(in, mul_cst); // 00ab00b000cd00d0
  401. const __m128i tmp = _mm_and_si128(mul, mask_mul); // 00ab000000cd0000
  402. const __m128i shift = _mm_srli_epi32(tmp, 12); // 00000000ab000000
  403. const __m128i pack = _mm_or_si128(shift, tmp); // 00000000abcd0000
  404. // Convert to 0xff00**00.
  405. const __m128i res = _mm_or_si128(pack, mask_or);
  406. _mm_storeu_si128((__m128i*)dst, res);
  407. }
  408. break;
  409. }
  410. default: {
  411. assert(xbits == 3);
  412. for (x = 0; x + 16 <= width; x += 16, dst += 2) {
  413. // 0000000a00000000b... | (where a/b are 1 bit).
  414. const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]);
  415. const __m128i shift = _mm_slli_epi64(in, 7);
  416. const uint32_t move = _mm_movemask_epi8(shift);
  417. dst[0] = 0xff000000 | ((move & 0xff) << 8);
  418. dst[1] = 0xff000000 | (move & 0xff00);
  419. }
  420. break;
  421. }
  422. }
  423. if (x != width) {
  424. VP8LBundleColorMap_C(row + x, width - x, xbits, dst);
  425. }
  426. }
  427. //------------------------------------------------------------------------------
  428. // Batch version of Predictor Transform subtraction
  429. static WEBP_INLINE void Average2_m128i(const __m128i* const a0,
  430. const __m128i* const a1,
  431. __m128i* const avg) {
  432. // (a + b) >> 1 = ((a + b + 1) >> 1) - ((a ^ b) & 1)
  433. const __m128i ones = _mm_set1_epi8(1);
  434. const __m128i avg1 = _mm_avg_epu8(*a0, *a1);
  435. const __m128i one = _mm_and_si128(_mm_xor_si128(*a0, *a1), ones);
  436. *avg = _mm_sub_epi8(avg1, one);
  437. }
  438. // Predictor0: ARGB_BLACK.
  439. static void PredictorSub0_SSE2(const uint32_t* in, const uint32_t* upper,
  440. int num_pixels, uint32_t* out) {
  441. int i;
  442. const __m128i black = _mm_set1_epi32(ARGB_BLACK);
  443. for (i = 0; i + 4 <= num_pixels; i += 4) {
  444. const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
  445. const __m128i res = _mm_sub_epi8(src, black);
  446. _mm_storeu_si128((__m128i*)&out[i], res);
  447. }
  448. if (i != num_pixels) {
  449. VP8LPredictorsSub_C[0](in + i, upper + i, num_pixels - i, out + i);
  450. }
  451. }
  452. #define GENERATE_PREDICTOR_1(X, IN) \
  453. static void PredictorSub##X##_SSE2(const uint32_t* in, const uint32_t* upper, \
  454. int num_pixels, uint32_t* out) { \
  455. int i; \
  456. for (i = 0; i + 4 <= num_pixels; i += 4) { \
  457. const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); \
  458. const __m128i pred = _mm_loadu_si128((const __m128i*)&(IN)); \
  459. const __m128i res = _mm_sub_epi8(src, pred); \
  460. _mm_storeu_si128((__m128i*)&out[i], res); \
  461. } \
  462. if (i != num_pixels) { \
  463. VP8LPredictorsSub_C[(X)](in + i, upper + i, num_pixels - i, out + i); \
  464. } \
  465. }
  466. GENERATE_PREDICTOR_1(1, in[i - 1]) // Predictor1: L
  467. GENERATE_PREDICTOR_1(2, upper[i]) // Predictor2: T
  468. GENERATE_PREDICTOR_1(3, upper[i + 1]) // Predictor3: TR
  469. GENERATE_PREDICTOR_1(4, upper[i - 1]) // Predictor4: TL
  470. #undef GENERATE_PREDICTOR_1
  471. // Predictor5: avg2(avg2(L, TR), T)
  472. static void PredictorSub5_SSE2(const uint32_t* in, const uint32_t* upper,
  473. int num_pixels, uint32_t* out) {
  474. int i;
  475. for (i = 0; i + 4 <= num_pixels; i += 4) {
  476. const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]);
  477. const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]);
  478. const __m128i TR = _mm_loadu_si128((const __m128i*)&upper[i + 1]);
  479. const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
  480. __m128i avg, pred, res;
  481. Average2_m128i(&L, &TR, &avg);
  482. Average2_m128i(&avg, &T, &pred);
  483. res = _mm_sub_epi8(src, pred);
  484. _mm_storeu_si128((__m128i*)&out[i], res);
  485. }
  486. if (i != num_pixels) {
  487. VP8LPredictorsSub_C[5](in + i, upper + i, num_pixels - i, out + i);
  488. }
  489. }
  490. #define GENERATE_PREDICTOR_2(X, A, B) \
  491. static void PredictorSub##X##_SSE2(const uint32_t* in, const uint32_t* upper, \
  492. int num_pixels, uint32_t* out) { \
  493. int i; \
  494. for (i = 0; i + 4 <= num_pixels; i += 4) { \
  495. const __m128i tA = _mm_loadu_si128((const __m128i*)&(A)); \
  496. const __m128i tB = _mm_loadu_si128((const __m128i*)&(B)); \
  497. const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); \
  498. __m128i pred, res; \
  499. Average2_m128i(&tA, &tB, &pred); \
  500. res = _mm_sub_epi8(src, pred); \
  501. _mm_storeu_si128((__m128i*)&out[i], res); \
  502. } \
  503. if (i != num_pixels) { \
  504. VP8LPredictorsSub_C[(X)](in + i, upper + i, num_pixels - i, out + i); \
  505. } \
  506. }
  507. GENERATE_PREDICTOR_2(6, in[i - 1], upper[i - 1]) // Predictor6: avg(L, TL)
  508. GENERATE_PREDICTOR_2(7, in[i - 1], upper[i]) // Predictor7: avg(L, T)
  509. GENERATE_PREDICTOR_2(8, upper[i - 1], upper[i]) // Predictor8: avg(TL, T)
  510. GENERATE_PREDICTOR_2(9, upper[i], upper[i + 1]) // Predictor9: average(T, TR)
  511. #undef GENERATE_PREDICTOR_2
  512. // Predictor10: avg(avg(L,TL), avg(T, TR)).
  513. static void PredictorSub10_SSE2(const uint32_t* in, const uint32_t* upper,
  514. int num_pixels, uint32_t* out) {
  515. int i;
  516. for (i = 0; i + 4 <= num_pixels; i += 4) {
  517. const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]);
  518. const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
  519. const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]);
  520. const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]);
  521. const __m128i TR = _mm_loadu_si128((const __m128i*)&upper[i + 1]);
  522. __m128i avgTTR, avgLTL, avg, res;
  523. Average2_m128i(&T, &TR, &avgTTR);
  524. Average2_m128i(&L, &TL, &avgLTL);
  525. Average2_m128i(&avgTTR, &avgLTL, &avg);
  526. res = _mm_sub_epi8(src, avg);
  527. _mm_storeu_si128((__m128i*)&out[i], res);
  528. }
  529. if (i != num_pixels) {
  530. VP8LPredictorsSub_C[10](in + i, upper + i, num_pixels - i, out + i);
  531. }
  532. }
  533. // Predictor11: select.
  534. static void GetSumAbsDiff32(const __m128i* const A, const __m128i* const B,
  535. __m128i* const out) {
  536. // We can unpack with any value on the upper 32 bits, provided it's the same
  537. // on both operands (to that their sum of abs diff is zero). Here we use *A.
  538. const __m128i A_lo = _mm_unpacklo_epi32(*A, *A);
  539. const __m128i B_lo = _mm_unpacklo_epi32(*B, *A);
  540. const __m128i A_hi = _mm_unpackhi_epi32(*A, *A);
  541. const __m128i B_hi = _mm_unpackhi_epi32(*B, *A);
  542. const __m128i s_lo = _mm_sad_epu8(A_lo, B_lo);
  543. const __m128i s_hi = _mm_sad_epu8(A_hi, B_hi);
  544. *out = _mm_packs_epi32(s_lo, s_hi);
  545. }
  546. static void PredictorSub11_SSE2(const uint32_t* in, const uint32_t* upper,
  547. int num_pixels, uint32_t* out) {
  548. int i;
  549. for (i = 0; i + 4 <= num_pixels; i += 4) {
  550. const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]);
  551. const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]);
  552. const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]);
  553. const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
  554. __m128i pa, pb;
  555. GetSumAbsDiff32(&T, &TL, &pa); // pa = sum |T-TL|
  556. GetSumAbsDiff32(&L, &TL, &pb); // pb = sum |L-TL|
  557. {
  558. const __m128i mask = _mm_cmpgt_epi32(pb, pa);
  559. const __m128i A = _mm_and_si128(mask, L);
  560. const __m128i B = _mm_andnot_si128(mask, T);
  561. const __m128i pred = _mm_or_si128(A, B); // pred = (L > T)? L : T
  562. const __m128i res = _mm_sub_epi8(src, pred);
  563. _mm_storeu_si128((__m128i*)&out[i], res);
  564. }
  565. }
  566. if (i != num_pixels) {
  567. VP8LPredictorsSub_C[11](in + i, upper + i, num_pixels - i, out + i);
  568. }
  569. }
  570. // Predictor12: ClampedSubSubtractFull.
  571. static void PredictorSub12_SSE2(const uint32_t* in, const uint32_t* upper,
  572. int num_pixels, uint32_t* out) {
  573. int i;
  574. const __m128i zero = _mm_setzero_si128();
  575. for (i = 0; i + 4 <= num_pixels; i += 4) {
  576. const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
  577. const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]);
  578. const __m128i L_lo = _mm_unpacklo_epi8(L, zero);
  579. const __m128i L_hi = _mm_unpackhi_epi8(L, zero);
  580. const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]);
  581. const __m128i T_lo = _mm_unpacklo_epi8(T, zero);
  582. const __m128i T_hi = _mm_unpackhi_epi8(T, zero);
  583. const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]);
  584. const __m128i TL_lo = _mm_unpacklo_epi8(TL, zero);
  585. const __m128i TL_hi = _mm_unpackhi_epi8(TL, zero);
  586. const __m128i diff_lo = _mm_sub_epi16(T_lo, TL_lo);
  587. const __m128i diff_hi = _mm_sub_epi16(T_hi, TL_hi);
  588. const __m128i pred_lo = _mm_add_epi16(L_lo, diff_lo);
  589. const __m128i pred_hi = _mm_add_epi16(L_hi, diff_hi);
  590. const __m128i pred = _mm_packus_epi16(pred_lo, pred_hi);
  591. const __m128i res = _mm_sub_epi8(src, pred);
  592. _mm_storeu_si128((__m128i*)&out[i], res);
  593. }
  594. if (i != num_pixels) {
  595. VP8LPredictorsSub_C[12](in + i, upper + i, num_pixels - i, out + i);
  596. }
  597. }
  598. // Predictors13: ClampedAddSubtractHalf
  599. static void PredictorSub13_SSE2(const uint32_t* in, const uint32_t* upper,
  600. int num_pixels, uint32_t* out) {
  601. int i;
  602. const __m128i zero = _mm_setzero_si128();
  603. for (i = 0; i + 2 <= num_pixels; i += 2) {
  604. // we can only process two pixels at a time
  605. const __m128i L = _mm_loadl_epi64((const __m128i*)&in[i - 1]);
  606. const __m128i src = _mm_loadl_epi64((const __m128i*)&in[i]);
  607. const __m128i T = _mm_loadl_epi64((const __m128i*)&upper[i]);
  608. const __m128i TL = _mm_loadl_epi64((const __m128i*)&upper[i - 1]);
  609. const __m128i L_lo = _mm_unpacklo_epi8(L, zero);
  610. const __m128i T_lo = _mm_unpacklo_epi8(T, zero);
  611. const __m128i TL_lo = _mm_unpacklo_epi8(TL, zero);
  612. const __m128i sum = _mm_add_epi16(T_lo, L_lo);
  613. const __m128i avg = _mm_srli_epi16(sum, 1);
  614. const __m128i A1 = _mm_sub_epi16(avg, TL_lo);
  615. const __m128i bit_fix = _mm_cmpgt_epi16(TL_lo, avg);
  616. const __m128i A2 = _mm_sub_epi16(A1, bit_fix);
  617. const __m128i A3 = _mm_srai_epi16(A2, 1);
  618. const __m128i A4 = _mm_add_epi16(avg, A3);
  619. const __m128i pred = _mm_packus_epi16(A4, A4);
  620. const __m128i res = _mm_sub_epi8(src, pred);
  621. _mm_storel_epi64((__m128i*)&out[i], res);
  622. }
  623. if (i != num_pixels) {
  624. VP8LPredictorsSub_C[13](in + i, upper + i, num_pixels - i, out + i);
  625. }
  626. }
  627. //------------------------------------------------------------------------------
  628. // Entry point
  629. extern void VP8LEncDspInitSSE2(void);
  630. WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitSSE2(void) {
  631. VP8LSubtractGreenFromBlueAndRed = SubtractGreenFromBlueAndRed;
  632. VP8LTransformColor = TransformColor;
  633. VP8LCollectColorBlueTransforms = CollectColorBlueTransforms;
  634. VP8LCollectColorRedTransforms = CollectColorRedTransforms;
  635. VP8LHistogramAdd = HistogramAdd;
  636. VP8LCombinedShannonEntropy = CombinedShannonEntropy;
  637. VP8LVectorMismatch = VectorMismatch;
  638. VP8LBundleColorMap = BundleColorMap_SSE2;
  639. VP8LPredictorsSub[0] = PredictorSub0_SSE2;
  640. VP8LPredictorsSub[1] = PredictorSub1_SSE2;
  641. VP8LPredictorsSub[2] = PredictorSub2_SSE2;
  642. VP8LPredictorsSub[3] = PredictorSub3_SSE2;
  643. VP8LPredictorsSub[4] = PredictorSub4_SSE2;
  644. VP8LPredictorsSub[5] = PredictorSub5_SSE2;
  645. VP8LPredictorsSub[6] = PredictorSub6_SSE2;
  646. VP8LPredictorsSub[7] = PredictorSub7_SSE2;
  647. VP8LPredictorsSub[8] = PredictorSub8_SSE2;
  648. VP8LPredictorsSub[9] = PredictorSub9_SSE2;
  649. VP8LPredictorsSub[10] = PredictorSub10_SSE2;
  650. VP8LPredictorsSub[11] = PredictorSub11_SSE2;
  651. VP8LPredictorsSub[12] = PredictorSub12_SSE2;
  652. VP8LPredictorsSub[13] = PredictorSub13_SSE2;
  653. VP8LPredictorsSub[14] = PredictorSub0_SSE2; // <- padding security sentinels
  654. VP8LPredictorsSub[15] = PredictorSub0_SSE2;
  655. }
  656. #else // !WEBP_USE_SSE2
  657. WEBP_DSP_INIT_STUB(VP8LEncDspInitSSE2)
  658. #endif // WEBP_USE_SSE2