123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388 |
- /* filter_neon_intrinsics.c - NEON optimised filter functions
- *
- * Copyright (c) 2014,2016 Glenn Randers-Pehrson
- * Written by James Yu <james.yu at linaro.org>, October 2013.
- * Based on filter_neon.S, written by Mans Rullgard, 2011.
- *
- * Last changed in libpng 1.6.22 [May 26, 2016]
- *
- * This code is released under the libpng license.
- * For conditions of distribution and use, see the disclaimer
- * and license in png.h
- */
- #include "../pngpriv.h"
- #ifdef PNG_READ_SUPPORTED
- /* This code requires -mfpu=neon on the command line: */
- #if PNG_ARM_NEON_IMPLEMENTATION == 1 /* intrinsics code from pngpriv.h */
- #include <arm_neon.h>
- /* libpng row pointers are not necessarily aligned to any particular boundary,
- * however this code will only work with appropriate alignment. arm/arm_init.c
- * checks for this (and will not compile unless it is done). This code uses
- * variants of png_aligncast to avoid compiler warnings.
- */
- #define png_ptr(type,pointer) png_aligncast(type *,pointer)
- #define png_ptrc(type,pointer) png_aligncastconst(const type *,pointer)
- /* The following relies on a variable 'temp_pointer' being declared with type
- * 'type'. This is written this way just to hide the GCC strict aliasing
- * warning; note that the code is safe because there never is an alias between
- * the input and output pointers.
- */
- #define png_ldr(type,pointer)\
- (temp_pointer = png_ptr(type,pointer), *temp_pointer)
- #if PNG_ARM_NEON_OPT > 0
- void
- png_read_filter_row_up_neon(png_row_infop row_info, png_bytep row,
- png_const_bytep prev_row)
- {
- png_bytep rp = row;
- png_bytep rp_stop = row + row_info->rowbytes;
- png_const_bytep pp = prev_row;
- png_debug(1, "in png_read_filter_row_up_neon");
- for (; rp < rp_stop; rp += 16, pp += 16)
- {
- uint8x16_t qrp, qpp;
- qrp = vld1q_u8(rp);
- qpp = vld1q_u8(pp);
- qrp = vaddq_u8(qrp, qpp);
- vst1q_u8(rp, qrp);
- }
- }
- void
- png_read_filter_row_sub3_neon(png_row_infop row_info, png_bytep row,
- png_const_bytep prev_row)
- {
- png_bytep rp = row;
- png_bytep rp_stop = row + row_info->rowbytes;
- uint8x16_t vtmp = vld1q_u8(rp);
- uint8x8x2_t *vrpt = png_ptr(uint8x8x2_t, &vtmp);
- uint8x8x2_t vrp = *vrpt;
- uint8x8x4_t vdest;
- vdest.val[3] = vdup_n_u8(0);
- png_debug(1, "in png_read_filter_row_sub3_neon");
- for (; rp < rp_stop;)
- {
- uint8x8_t vtmp1, vtmp2;
- uint32x2_t *temp_pointer;
- vtmp1 = vext_u8(vrp.val[0], vrp.val[1], 3);
- vdest.val[0] = vadd_u8(vdest.val[3], vrp.val[0]);
- vtmp2 = vext_u8(vrp.val[0], vrp.val[1], 6);
- vdest.val[1] = vadd_u8(vdest.val[0], vtmp1);
- vtmp1 = vext_u8(vrp.val[1], vrp.val[1], 1);
- vdest.val[2] = vadd_u8(vdest.val[1], vtmp2);
- vdest.val[3] = vadd_u8(vdest.val[2], vtmp1);
- vtmp = vld1q_u8(rp + 12);
- vrpt = png_ptr(uint8x8x2_t, &vtmp);
- vrp = *vrpt;
- vst1_lane_u32(png_ptr(uint32_t,rp), png_ldr(uint32x2_t,&vdest.val[0]), 0);
- rp += 3;
- vst1_lane_u32(png_ptr(uint32_t,rp), png_ldr(uint32x2_t,&vdest.val[1]), 0);
- rp += 3;
- vst1_lane_u32(png_ptr(uint32_t,rp), png_ldr(uint32x2_t,&vdest.val[2]), 0);
- rp += 3;
- vst1_lane_u32(png_ptr(uint32_t,rp), png_ldr(uint32x2_t,&vdest.val[3]), 0);
- rp += 3;
- }
- PNG_UNUSED(prev_row)
- }
- void
- png_read_filter_row_sub4_neon(png_row_infop row_info, png_bytep row,
- png_const_bytep prev_row)
- {
- png_bytep rp = row;
- png_bytep rp_stop = row + row_info->rowbytes;
- uint8x8x4_t vdest;
- vdest.val[3] = vdup_n_u8(0);
- png_debug(1, "in png_read_filter_row_sub4_neon");
- for (; rp < rp_stop; rp += 16)
- {
- uint32x2x4_t vtmp = vld4_u32(png_ptr(uint32_t,rp));
- uint8x8x4_t *vrpt = png_ptr(uint8x8x4_t,&vtmp);
- uint8x8x4_t vrp = *vrpt;
- uint32x2x4_t *temp_pointer;
- vdest.val[0] = vadd_u8(vdest.val[3], vrp.val[0]);
- vdest.val[1] = vadd_u8(vdest.val[0], vrp.val[1]);
- vdest.val[2] = vadd_u8(vdest.val[1], vrp.val[2]);
- vdest.val[3] = vadd_u8(vdest.val[2], vrp.val[3]);
- vst4_lane_u32(png_ptr(uint32_t,rp), png_ldr(uint32x2x4_t,&vdest), 0);
- }
- PNG_UNUSED(prev_row)
- }
- void
- png_read_filter_row_avg3_neon(png_row_infop row_info, png_bytep row,
- png_const_bytep prev_row)
- {
- png_bytep rp = row;
- png_const_bytep pp = prev_row;
- png_bytep rp_stop = row + row_info->rowbytes;
- uint8x16_t vtmp;
- uint8x8x2_t *vrpt;
- uint8x8x2_t vrp;
- uint8x8x4_t vdest;
- vdest.val[3] = vdup_n_u8(0);
- vtmp = vld1q_u8(rp);
- vrpt = png_ptr(uint8x8x2_t,&vtmp);
- vrp = *vrpt;
- png_debug(1, "in png_read_filter_row_avg3_neon");
- for (; rp < rp_stop; pp += 12)
- {
- uint8x8_t vtmp1, vtmp2, vtmp3;
- uint8x8x2_t *vppt;
- uint8x8x2_t vpp;
- uint32x2_t *temp_pointer;
- vtmp = vld1q_u8(pp);
- vppt = png_ptr(uint8x8x2_t,&vtmp);
- vpp = *vppt;
- vtmp1 = vext_u8(vrp.val[0], vrp.val[1], 3);
- vdest.val[0] = vhadd_u8(vdest.val[3], vpp.val[0]);
- vdest.val[0] = vadd_u8(vdest.val[0], vrp.val[0]);
- vtmp2 = vext_u8(vpp.val[0], vpp.val[1], 3);
- vtmp3 = vext_u8(vrp.val[0], vrp.val[1], 6);
- vdest.val[1] = vhadd_u8(vdest.val[0], vtmp2);
- vdest.val[1] = vadd_u8(vdest.val[1], vtmp1);
- vtmp2 = vext_u8(vpp.val[0], vpp.val[1], 6);
- vtmp1 = vext_u8(vrp.val[1], vrp.val[1], 1);
- vtmp = vld1q_u8(rp + 12);
- vrpt = png_ptr(uint8x8x2_t,&vtmp);
- vrp = *vrpt;
- vdest.val[2] = vhadd_u8(vdest.val[1], vtmp2);
- vdest.val[2] = vadd_u8(vdest.val[2], vtmp3);
- vtmp2 = vext_u8(vpp.val[1], vpp.val[1], 1);
- vdest.val[3] = vhadd_u8(vdest.val[2], vtmp2);
- vdest.val[3] = vadd_u8(vdest.val[3], vtmp1);
- vst1_lane_u32(png_ptr(uint32_t,rp), png_ldr(uint32x2_t,&vdest.val[0]), 0);
- rp += 3;
- vst1_lane_u32(png_ptr(uint32_t,rp), png_ldr(uint32x2_t,&vdest.val[1]), 0);
- rp += 3;
- vst1_lane_u32(png_ptr(uint32_t,rp), png_ldr(uint32x2_t,&vdest.val[2]), 0);
- rp += 3;
- vst1_lane_u32(png_ptr(uint32_t,rp), png_ldr(uint32x2_t,&vdest.val[3]), 0);
- rp += 3;
- }
- }
- void
- png_read_filter_row_avg4_neon(png_row_infop row_info, png_bytep row,
- png_const_bytep prev_row)
- {
- png_bytep rp = row;
- png_bytep rp_stop = row + row_info->rowbytes;
- png_const_bytep pp = prev_row;
- uint8x8x4_t vdest;
- vdest.val[3] = vdup_n_u8(0);
- png_debug(1, "in png_read_filter_row_avg4_neon");
- for (; rp < rp_stop; rp += 16, pp += 16)
- {
- uint32x2x4_t vtmp;
- uint8x8x4_t *vrpt, *vppt;
- uint8x8x4_t vrp, vpp;
- uint32x2x4_t *temp_pointer;
- vtmp = vld4_u32(png_ptr(uint32_t,rp));
- vrpt = png_ptr(uint8x8x4_t,&vtmp);
- vrp = *vrpt;
- vtmp = vld4_u32(png_ptrc(uint32_t,pp));
- vppt = png_ptr(uint8x8x4_t,&vtmp);
- vpp = *vppt;
- vdest.val[0] = vhadd_u8(vdest.val[3], vpp.val[0]);
- vdest.val[0] = vadd_u8(vdest.val[0], vrp.val[0]);
- vdest.val[1] = vhadd_u8(vdest.val[0], vpp.val[1]);
- vdest.val[1] = vadd_u8(vdest.val[1], vrp.val[1]);
- vdest.val[2] = vhadd_u8(vdest.val[1], vpp.val[2]);
- vdest.val[2] = vadd_u8(vdest.val[2], vrp.val[2]);
- vdest.val[3] = vhadd_u8(vdest.val[2], vpp.val[3]);
- vdest.val[3] = vadd_u8(vdest.val[3], vrp.val[3]);
- vst4_lane_u32(png_ptr(uint32_t,rp), png_ldr(uint32x2x4_t,&vdest), 0);
- }
- }
- static uint8x8_t
- paeth(uint8x8_t a, uint8x8_t b, uint8x8_t c)
- {
- uint8x8_t d, e;
- uint16x8_t p1, pa, pb, pc;
- p1 = vaddl_u8(a, b); /* a + b */
- pc = vaddl_u8(c, c); /* c * 2 */
- pa = vabdl_u8(b, c); /* pa */
- pb = vabdl_u8(a, c); /* pb */
- pc = vabdq_u16(p1, pc); /* pc */
- p1 = vcleq_u16(pa, pb); /* pa <= pb */
- pa = vcleq_u16(pa, pc); /* pa <= pc */
- pb = vcleq_u16(pb, pc); /* pb <= pc */
- p1 = vandq_u16(p1, pa); /* pa <= pb && pa <= pc */
- d = vmovn_u16(pb);
- e = vmovn_u16(p1);
- d = vbsl_u8(d, b, c);
- e = vbsl_u8(e, a, d);
- return e;
- }
- void
- png_read_filter_row_paeth3_neon(png_row_infop row_info, png_bytep row,
- png_const_bytep prev_row)
- {
- png_bytep rp = row;
- png_const_bytep pp = prev_row;
- png_bytep rp_stop = row + row_info->rowbytes;
- uint8x16_t vtmp;
- uint8x8x2_t *vrpt;
- uint8x8x2_t vrp;
- uint8x8_t vlast = vdup_n_u8(0);
- uint8x8x4_t vdest;
- vdest.val[3] = vdup_n_u8(0);
- vtmp = vld1q_u8(rp);
- vrpt = png_ptr(uint8x8x2_t,&vtmp);
- vrp = *vrpt;
- png_debug(1, "in png_read_filter_row_paeth3_neon");
- for (; rp < rp_stop; pp += 12)
- {
- uint8x8x2_t *vppt;
- uint8x8x2_t vpp;
- uint8x8_t vtmp1, vtmp2, vtmp3;
- uint32x2_t *temp_pointer;
- vtmp = vld1q_u8(pp);
- vppt = png_ptr(uint8x8x2_t,&vtmp);
- vpp = *vppt;
- vdest.val[0] = paeth(vdest.val[3], vpp.val[0], vlast);
- vdest.val[0] = vadd_u8(vdest.val[0], vrp.val[0]);
- vtmp1 = vext_u8(vrp.val[0], vrp.val[1], 3);
- vtmp2 = vext_u8(vpp.val[0], vpp.val[1], 3);
- vdest.val[1] = paeth(vdest.val[0], vtmp2, vpp.val[0]);
- vdest.val[1] = vadd_u8(vdest.val[1], vtmp1);
- vtmp1 = vext_u8(vrp.val[0], vrp.val[1], 6);
- vtmp3 = vext_u8(vpp.val[0], vpp.val[1], 6);
- vdest.val[2] = paeth(vdest.val[1], vtmp3, vtmp2);
- vdest.val[2] = vadd_u8(vdest.val[2], vtmp1);
- vtmp1 = vext_u8(vrp.val[1], vrp.val[1], 1);
- vtmp2 = vext_u8(vpp.val[1], vpp.val[1], 1);
- vtmp = vld1q_u8(rp + 12);
- vrpt = png_ptr(uint8x8x2_t,&vtmp);
- vrp = *vrpt;
- vdest.val[3] = paeth(vdest.val[2], vtmp2, vtmp3);
- vdest.val[3] = vadd_u8(vdest.val[3], vtmp1);
- vlast = vtmp2;
- vst1_lane_u32(png_ptr(uint32_t,rp), png_ldr(uint32x2_t,&vdest.val[0]), 0);
- rp += 3;
- vst1_lane_u32(png_ptr(uint32_t,rp), png_ldr(uint32x2_t,&vdest.val[1]), 0);
- rp += 3;
- vst1_lane_u32(png_ptr(uint32_t,rp), png_ldr(uint32x2_t,&vdest.val[2]), 0);
- rp += 3;
- vst1_lane_u32(png_ptr(uint32_t,rp), png_ldr(uint32x2_t,&vdest.val[3]), 0);
- rp += 3;
- }
- }
- void
- png_read_filter_row_paeth4_neon(png_row_infop row_info, png_bytep row,
- png_const_bytep prev_row)
- {
- png_bytep rp = row;
- png_bytep rp_stop = row + row_info->rowbytes;
- png_const_bytep pp = prev_row;
- uint8x8_t vlast = vdup_n_u8(0);
- uint8x8x4_t vdest;
- vdest.val[3] = vdup_n_u8(0);
- png_debug(1, "in png_read_filter_row_paeth4_neon");
- for (; rp < rp_stop; rp += 16, pp += 16)
- {
- uint32x2x4_t vtmp;
- uint8x8x4_t *vrpt, *vppt;
- uint8x8x4_t vrp, vpp;
- uint32x2x4_t *temp_pointer;
- vtmp = vld4_u32(png_ptr(uint32_t,rp));
- vrpt = png_ptr(uint8x8x4_t,&vtmp);
- vrp = *vrpt;
- vtmp = vld4_u32(png_ptrc(uint32_t,pp));
- vppt = png_ptr(uint8x8x4_t,&vtmp);
- vpp = *vppt;
- vdest.val[0] = paeth(vdest.val[3], vpp.val[0], vlast);
- vdest.val[0] = vadd_u8(vdest.val[0], vrp.val[0]);
- vdest.val[1] = paeth(vdest.val[0], vpp.val[1], vpp.val[0]);
- vdest.val[1] = vadd_u8(vdest.val[1], vrp.val[1]);
- vdest.val[2] = paeth(vdest.val[1], vpp.val[2], vpp.val[1]);
- vdest.val[2] = vadd_u8(vdest.val[2], vrp.val[2]);
- vdest.val[3] = paeth(vdest.val[2], vpp.val[3], vpp.val[2]);
- vdest.val[3] = vadd_u8(vdest.val[3], vrp.val[3]);
- vlast = vpp.val[3];
- vst4_lane_u32(png_ptr(uint32_t,rp), png_ldr(uint32x2x4_t,&vdest), 0);
- }
- }
- #endif /* PNG_ARM_NEON_OPT > 0 */
- #endif /* PNG_ARM_NEON_IMPLEMENTATION == 1 (intrinsics) */
- #endif /* READ */
|