123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486 |
- #include <linux/raid/pq.h>
- #include "x86.h"
- static const struct raid6_sse_constants {
- u64 x1d[2];
- } raid6_sse_constants __attribute__((aligned(16))) = {
- { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL },
- };
- static int raid6_have_sse2(void)
- {
-
- return boot_cpu_has(X86_FEATURE_MMX) &&
- boot_cpu_has(X86_FEATURE_FXSR) &&
- boot_cpu_has(X86_FEATURE_XMM) &&
- boot_cpu_has(X86_FEATURE_XMM2);
- }
- static void raid6_sse21_gen_syndrome(int disks, size_t bytes, void **ptrs)
- {
- u8 **dptr = (u8 **)ptrs;
- u8 *p, *q;
- int d, z, z0;
- z0 = disks - 3;
- p = dptr[z0+1];
- q = dptr[z0+2];
- kernel_fpu_begin();
- asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
- asm volatile("pxor %xmm5,%xmm5");
- for ( d = 0 ; d < bytes ; d += 16 ) {
- asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
- asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d]));
- asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));
- asm volatile("movdqa %xmm2,%xmm4");
- asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z0-1][d]));
- for ( z = z0-2 ; z >= 0 ; z-- ) {
- asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
- asm volatile("pcmpgtb %xmm4,%xmm5");
- asm volatile("paddb %xmm4,%xmm4");
- asm volatile("pand %xmm0,%xmm5");
- asm volatile("pxor %xmm5,%xmm4");
- asm volatile("pxor %xmm5,%xmm5");
- asm volatile("pxor %xmm6,%xmm2");
- asm volatile("pxor %xmm6,%xmm4");
- asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z][d]));
- }
- asm volatile("pcmpgtb %xmm4,%xmm5");
- asm volatile("paddb %xmm4,%xmm4");
- asm volatile("pand %xmm0,%xmm5");
- asm volatile("pxor %xmm5,%xmm4");
- asm volatile("pxor %xmm5,%xmm5");
- asm volatile("pxor %xmm6,%xmm2");
- asm volatile("pxor %xmm6,%xmm4");
- asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
- asm volatile("pxor %xmm2,%xmm2");
- asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
- asm volatile("pxor %xmm4,%xmm4");
- }
- asm volatile("sfence" : : : "memory");
- kernel_fpu_end();
- }
- static void raid6_sse21_xor_syndrome(int disks, int start, int stop,
- size_t bytes, void **ptrs)
- {
- u8 **dptr = (u8 **)ptrs;
- u8 *p, *q;
- int d, z, z0;
- z0 = stop;
- p = dptr[disks-2];
- q = dptr[disks-1];
- kernel_fpu_begin();
- asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
- for ( d = 0 ; d < bytes ; d += 16 ) {
- asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
- asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
- asm volatile("pxor %xmm4,%xmm2");
-
- for ( z = z0-1 ; z >= start ; z-- ) {
- asm volatile("pxor %xmm5,%xmm5");
- asm volatile("pcmpgtb %xmm4,%xmm5");
- asm volatile("paddb %xmm4,%xmm4");
- asm volatile("pand %xmm0,%xmm5");
- asm volatile("pxor %xmm5,%xmm4");
- asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
- asm volatile("pxor %xmm5,%xmm2");
- asm volatile("pxor %xmm5,%xmm4");
- }
-
- for ( z = start-1 ; z >= 0 ; z-- ) {
- asm volatile("pxor %xmm5,%xmm5");
- asm volatile("pcmpgtb %xmm4,%xmm5");
- asm volatile("paddb %xmm4,%xmm4");
- asm volatile("pand %xmm0,%xmm5");
- asm volatile("pxor %xmm5,%xmm4");
- }
- asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
-
- asm volatile("movdqa %%xmm4,%0" : "=m" (q[d]));
- asm volatile("movdqa %%xmm2,%0" : "=m" (p[d]));
- }
- asm volatile("sfence" : : : "memory");
- kernel_fpu_end();
- }
- const struct raid6_calls raid6_sse2x1 = {
- raid6_sse21_gen_syndrome,
- raid6_sse21_xor_syndrome,
- raid6_have_sse2,
- "sse2x1",
- 1
- };
- static void raid6_sse22_gen_syndrome(int disks, size_t bytes, void **ptrs)
- {
- u8 **dptr = (u8 **)ptrs;
- u8 *p, *q;
- int d, z, z0;
- z0 = disks - 3;
- p = dptr[z0+1];
- q = dptr[z0+2];
- kernel_fpu_begin();
- asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
- asm volatile("pxor %xmm5,%xmm5");
- asm volatile("pxor %xmm7,%xmm7");
-
- for ( d = 0 ; d < bytes ; d += 32 ) {
- asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
- asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d]));
- asm volatile("movdqa %0,%%xmm3" : : "m" (dptr[z0][d+16]));
- asm volatile("movdqa %xmm2,%xmm4");
- asm volatile("movdqa %xmm3,%xmm6");
- for ( z = z0-1 ; z >= 0 ; z-- ) {
- asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
- asm volatile("pcmpgtb %xmm4,%xmm5");
- asm volatile("pcmpgtb %xmm6,%xmm7");
- asm volatile("paddb %xmm4,%xmm4");
- asm volatile("paddb %xmm6,%xmm6");
- asm volatile("pand %xmm0,%xmm5");
- asm volatile("pand %xmm0,%xmm7");
- asm volatile("pxor %xmm5,%xmm4");
- asm volatile("pxor %xmm7,%xmm6");
- asm volatile("movdqa %0,%%xmm5" : : "m" (dptr[z][d]));
- asm volatile("movdqa %0,%%xmm7" : : "m" (dptr[z][d+16]));
- asm volatile("pxor %xmm5,%xmm2");
- asm volatile("pxor %xmm7,%xmm3");
- asm volatile("pxor %xmm5,%xmm4");
- asm volatile("pxor %xmm7,%xmm6");
- asm volatile("pxor %xmm5,%xmm5");
- asm volatile("pxor %xmm7,%xmm7");
- }
- asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
- asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
- asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
- asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
- }
- asm volatile("sfence" : : : "memory");
- kernel_fpu_end();
- }
- static void raid6_sse22_xor_syndrome(int disks, int start, int stop,
- size_t bytes, void **ptrs)
- {
- u8 **dptr = (u8 **)ptrs;
- u8 *p, *q;
- int d, z, z0;
- z0 = stop;
- p = dptr[disks-2];
- q = dptr[disks-1];
- kernel_fpu_begin();
- asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
- for ( d = 0 ; d < bytes ; d += 32 ) {
- asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
- asm volatile("movdqa %0,%%xmm6" :: "m" (dptr[z0][d+16]));
- asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
- asm volatile("movdqa %0,%%xmm3" : : "m" (p[d+16]));
- asm volatile("pxor %xmm4,%xmm2");
- asm volatile("pxor %xmm6,%xmm3");
-
- for ( z = z0-1 ; z >= start ; z-- ) {
- asm volatile("pxor %xmm5,%xmm5");
- asm volatile("pxor %xmm7,%xmm7");
- asm volatile("pcmpgtb %xmm4,%xmm5");
- asm volatile("pcmpgtb %xmm6,%xmm7");
- asm volatile("paddb %xmm4,%xmm4");
- asm volatile("paddb %xmm6,%xmm6");
- asm volatile("pand %xmm0,%xmm5");
- asm volatile("pand %xmm0,%xmm7");
- asm volatile("pxor %xmm5,%xmm4");
- asm volatile("pxor %xmm7,%xmm6");
- asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
- asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
- asm volatile("pxor %xmm5,%xmm2");
- asm volatile("pxor %xmm7,%xmm3");
- asm volatile("pxor %xmm5,%xmm4");
- asm volatile("pxor %xmm7,%xmm6");
- }
-
- for ( z = start-1 ; z >= 0 ; z-- ) {
- asm volatile("pxor %xmm5,%xmm5");
- asm volatile("pxor %xmm7,%xmm7");
- asm volatile("pcmpgtb %xmm4,%xmm5");
- asm volatile("pcmpgtb %xmm6,%xmm7");
- asm volatile("paddb %xmm4,%xmm4");
- asm volatile("paddb %xmm6,%xmm6");
- asm volatile("pand %xmm0,%xmm5");
- asm volatile("pand %xmm0,%xmm7");
- asm volatile("pxor %xmm5,%xmm4");
- asm volatile("pxor %xmm7,%xmm6");
- }
- asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
- asm volatile("pxor %0,%%xmm6" : : "m" (q[d+16]));
-
- asm volatile("movdqa %%xmm4,%0" : "=m" (q[d]));
- asm volatile("movdqa %%xmm6,%0" : "=m" (q[d+16]));
- asm volatile("movdqa %%xmm2,%0" : "=m" (p[d]));
- asm volatile("movdqa %%xmm3,%0" : "=m" (p[d+16]));
- }
- asm volatile("sfence" : : : "memory");
- kernel_fpu_end();
- }
- const struct raid6_calls raid6_sse2x2 = {
- raid6_sse22_gen_syndrome,
- raid6_sse22_xor_syndrome,
- raid6_have_sse2,
- "sse2x2",
- 1
- };
- #ifdef CONFIG_X86_64
- static void raid6_sse24_gen_syndrome(int disks, size_t bytes, void **ptrs)
- {
- u8 **dptr = (u8 **)ptrs;
- u8 *p, *q;
- int d, z, z0;
- z0 = disks - 3;
- p = dptr[z0+1];
- q = dptr[z0+2];
- kernel_fpu_begin();
- asm volatile("movdqa %0,%%xmm0" :: "m" (raid6_sse_constants.x1d[0]));
- asm volatile("pxor %xmm2,%xmm2");
- asm volatile("pxor %xmm3,%xmm3");
- asm volatile("pxor %xmm4,%xmm4");
- asm volatile("pxor %xmm5,%xmm5");
- asm volatile("pxor %xmm6,%xmm6");
- asm volatile("pxor %xmm7,%xmm7");
- asm volatile("pxor %xmm10,%xmm10");
- asm volatile("pxor %xmm11,%xmm11");
- asm volatile("pxor %xmm12,%xmm12");
- asm volatile("pxor %xmm13,%xmm13");
- asm volatile("pxor %xmm14,%xmm14");
- asm volatile("pxor %xmm15,%xmm15");
- for ( d = 0 ; d < bytes ; d += 64 ) {
- for ( z = z0 ; z >= 0 ; z-- ) {
-
- asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
- asm volatile("prefetchnta %0" :: "m" (dptr[z][d+32]));
- asm volatile("pcmpgtb %xmm4,%xmm5");
- asm volatile("pcmpgtb %xmm6,%xmm7");
- asm volatile("pcmpgtb %xmm12,%xmm13");
- asm volatile("pcmpgtb %xmm14,%xmm15");
- asm volatile("paddb %xmm4,%xmm4");
- asm volatile("paddb %xmm6,%xmm6");
- asm volatile("paddb %xmm12,%xmm12");
- asm volatile("paddb %xmm14,%xmm14");
- asm volatile("pand %xmm0,%xmm5");
- asm volatile("pand %xmm0,%xmm7");
- asm volatile("pand %xmm0,%xmm13");
- asm volatile("pand %xmm0,%xmm15");
- asm volatile("pxor %xmm5,%xmm4");
- asm volatile("pxor %xmm7,%xmm6");
- asm volatile("pxor %xmm13,%xmm12");
- asm volatile("pxor %xmm15,%xmm14");
- asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
- asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
- asm volatile("movdqa %0,%%xmm13" :: "m" (dptr[z][d+32]));
- asm volatile("movdqa %0,%%xmm15" :: "m" (dptr[z][d+48]));
- asm volatile("pxor %xmm5,%xmm2");
- asm volatile("pxor %xmm7,%xmm3");
- asm volatile("pxor %xmm13,%xmm10");
- asm volatile("pxor %xmm15,%xmm11");
- asm volatile("pxor %xmm5,%xmm4");
- asm volatile("pxor %xmm7,%xmm6");
- asm volatile("pxor %xmm13,%xmm12");
- asm volatile("pxor %xmm15,%xmm14");
- asm volatile("pxor %xmm5,%xmm5");
- asm volatile("pxor %xmm7,%xmm7");
- asm volatile("pxor %xmm13,%xmm13");
- asm volatile("pxor %xmm15,%xmm15");
- }
- asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
- asm volatile("pxor %xmm2,%xmm2");
- asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
- asm volatile("pxor %xmm3,%xmm3");
- asm volatile("movntdq %%xmm10,%0" : "=m" (p[d+32]));
- asm volatile("pxor %xmm10,%xmm10");
- asm volatile("movntdq %%xmm11,%0" : "=m" (p[d+48]));
- asm volatile("pxor %xmm11,%xmm11");
- asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
- asm volatile("pxor %xmm4,%xmm4");
- asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
- asm volatile("pxor %xmm6,%xmm6");
- asm volatile("movntdq %%xmm12,%0" : "=m" (q[d+32]));
- asm volatile("pxor %xmm12,%xmm12");
- asm volatile("movntdq %%xmm14,%0" : "=m" (q[d+48]));
- asm volatile("pxor %xmm14,%xmm14");
- }
- asm volatile("sfence" : : : "memory");
- kernel_fpu_end();
- }
- static void raid6_sse24_xor_syndrome(int disks, int start, int stop,
- size_t bytes, void **ptrs)
- {
- u8 **dptr = (u8 **)ptrs;
- u8 *p, *q;
- int d, z, z0;
- z0 = stop;
- p = dptr[disks-2];
- q = dptr[disks-1];
- kernel_fpu_begin();
- asm volatile("movdqa %0,%%xmm0" :: "m" (raid6_sse_constants.x1d[0]));
- for ( d = 0 ; d < bytes ; d += 64 ) {
- asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
- asm volatile("movdqa %0,%%xmm6" :: "m" (dptr[z0][d+16]));
- asm volatile("movdqa %0,%%xmm12" :: "m" (dptr[z0][d+32]));
- asm volatile("movdqa %0,%%xmm14" :: "m" (dptr[z0][d+48]));
- asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
- asm volatile("movdqa %0,%%xmm3" : : "m" (p[d+16]));
- asm volatile("movdqa %0,%%xmm10" : : "m" (p[d+32]));
- asm volatile("movdqa %0,%%xmm11" : : "m" (p[d+48]));
- asm volatile("pxor %xmm4,%xmm2");
- asm volatile("pxor %xmm6,%xmm3");
- asm volatile("pxor %xmm12,%xmm10");
- asm volatile("pxor %xmm14,%xmm11");
-
- for ( z = z0-1 ; z >= start ; z-- ) {
- asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
- asm volatile("prefetchnta %0" :: "m" (dptr[z][d+32]));
- asm volatile("pxor %xmm5,%xmm5");
- asm volatile("pxor %xmm7,%xmm7");
- asm volatile("pxor %xmm13,%xmm13");
- asm volatile("pxor %xmm15,%xmm15");
- asm volatile("pcmpgtb %xmm4,%xmm5");
- asm volatile("pcmpgtb %xmm6,%xmm7");
- asm volatile("pcmpgtb %xmm12,%xmm13");
- asm volatile("pcmpgtb %xmm14,%xmm15");
- asm volatile("paddb %xmm4,%xmm4");
- asm volatile("paddb %xmm6,%xmm6");
- asm volatile("paddb %xmm12,%xmm12");
- asm volatile("paddb %xmm14,%xmm14");
- asm volatile("pand %xmm0,%xmm5");
- asm volatile("pand %xmm0,%xmm7");
- asm volatile("pand %xmm0,%xmm13");
- asm volatile("pand %xmm0,%xmm15");
- asm volatile("pxor %xmm5,%xmm4");
- asm volatile("pxor %xmm7,%xmm6");
- asm volatile("pxor %xmm13,%xmm12");
- asm volatile("pxor %xmm15,%xmm14");
- asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
- asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
- asm volatile("movdqa %0,%%xmm13" :: "m" (dptr[z][d+32]));
- asm volatile("movdqa %0,%%xmm15" :: "m" (dptr[z][d+48]));
- asm volatile("pxor %xmm5,%xmm2");
- asm volatile("pxor %xmm7,%xmm3");
- asm volatile("pxor %xmm13,%xmm10");
- asm volatile("pxor %xmm15,%xmm11");
- asm volatile("pxor %xmm5,%xmm4");
- asm volatile("pxor %xmm7,%xmm6");
- asm volatile("pxor %xmm13,%xmm12");
- asm volatile("pxor %xmm15,%xmm14");
- }
- asm volatile("prefetchnta %0" :: "m" (q[d]));
- asm volatile("prefetchnta %0" :: "m" (q[d+32]));
-
- for ( z = start-1 ; z >= 0 ; z-- ) {
- asm volatile("pxor %xmm5,%xmm5");
- asm volatile("pxor %xmm7,%xmm7");
- asm volatile("pxor %xmm13,%xmm13");
- asm volatile("pxor %xmm15,%xmm15");
- asm volatile("pcmpgtb %xmm4,%xmm5");
- asm volatile("pcmpgtb %xmm6,%xmm7");
- asm volatile("pcmpgtb %xmm12,%xmm13");
- asm volatile("pcmpgtb %xmm14,%xmm15");
- asm volatile("paddb %xmm4,%xmm4");
- asm volatile("paddb %xmm6,%xmm6");
- asm volatile("paddb %xmm12,%xmm12");
- asm volatile("paddb %xmm14,%xmm14");
- asm volatile("pand %xmm0,%xmm5");
- asm volatile("pand %xmm0,%xmm7");
- asm volatile("pand %xmm0,%xmm13");
- asm volatile("pand %xmm0,%xmm15");
- asm volatile("pxor %xmm5,%xmm4");
- asm volatile("pxor %xmm7,%xmm6");
- asm volatile("pxor %xmm13,%xmm12");
- asm volatile("pxor %xmm15,%xmm14");
- }
- asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
- asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
- asm volatile("movntdq %%xmm10,%0" : "=m" (p[d+32]));
- asm volatile("movntdq %%xmm11,%0" : "=m" (p[d+48]));
- asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
- asm volatile("pxor %0,%%xmm6" : : "m" (q[d+16]));
- asm volatile("pxor %0,%%xmm12" : : "m" (q[d+32]));
- asm volatile("pxor %0,%%xmm14" : : "m" (q[d+48]));
- asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
- asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
- asm volatile("movntdq %%xmm12,%0" : "=m" (q[d+32]));
- asm volatile("movntdq %%xmm14,%0" : "=m" (q[d+48]));
- }
- asm volatile("sfence" : : : "memory");
- kernel_fpu_end();
- }
- const struct raid6_calls raid6_sse2x4 = {
- raid6_sse24_gen_syndrome,
- raid6_sse24_xor_syndrome,
- raid6_have_sse2,
- "sse2x4",
- 1
- };
- #endif
|