godot/thirdparty/libvpx/vpx_dsp/x86/inv_txfm_ssse3_x86_64.asm

;
;  Copyright (c) 2014 The WebM project authors. All Rights Reserved.
;
;  Use of this source code is governed by a BSD-style license
;  that can be found in the LICENSE file in the root of the source
;  tree. An additional intellectual property rights grant can be found
;  in the file PATENTS.  All contributing project authors may
;  be found in the AUTHORS file in the root of the source tree.
;

%include "third_party/x86inc/x86inc.asm"

; This file provides SSSE3 version of the inverse transformation. Part
; of the functions are originally derived from the ffmpeg project.
; Note that the current version applies to x86 64-bit only.

SECTION_RODATA

pw_11585x2: times 8 dw 23170

pw_m2404x2:  times 8 dw  -2404*2
pw_m4756x2:  times 8 dw  -4756*2
pw_m5520x2:  times 8 dw  -5520*2
pw_m8423x2:  times 8 dw  -8423*2
pw_m9102x2:  times 8 dw  -9102*2
pw_m10394x2: times 8 dw -10394*2
pw_m11003x2: times 8 dw -11003*2

pw_16364x2: times 8 dw 16364*2
pw_16305x2: times 8 dw 16305*2
pw_16207x2: times 8 dw 16207*2
pw_16069x2: times 8 dw 16069*2
pw_15893x2: times 8 dw 15893*2
pw_15679x2: times 8 dw 15679*2
pw_15426x2: times 8 dw 15426*2
pw_15137x2: times 8 dw 15137*2
pw_14811x2: times 8 dw 14811*2
pw_14449x2: times 8 dw 14449*2
pw_14053x2: times 8 dw 14053*2
pw_13623x2: times 8 dw 13623*2
pw_13160x2: times 8 dw 13160*2
pw_12665x2: times 8 dw 12665*2
pw_12140x2: times 8 dw 12140*2
pw__9760x2: times 8 dw  9760*2
pw__7723x2: times 8 dw  7723*2
pw__7005x2: times 8 dw  7005*2
pw__6270x2: times 8 dw  6270*2
pw__3981x2: times 8 dw  3981*2
pw__3196x2: times 8 dw  3196*2
pw__1606x2: times 8 dw  1606*2
pw___804x2: times 8 dw   804*2

pd_8192:    times 4 dd 8192
pw_32:      times 8 dw 32
pw_16:      times 8 dw 16

%macro TRANSFORM_COEFFS 2
pw_%1_%2:   dw  %1,  %2,  %1,  %2,  %1,  %2,  %1,  %2
pw_m%2_%1:  dw -%2,  %1, -%2,  %1, -%2,  %1, -%2,  %1
pw_m%1_m%2: dw -%1, -%2, -%1, -%2, -%1, -%2, -%1, -%2
%endmacro

TRANSFORM_COEFFS    6270, 15137
TRANSFORM_COEFFS    3196, 16069
TRANSFORM_COEFFS   13623,  9102

; constants for 32x32_34
TRANSFORM_COEFFS      804, 16364
TRANSFORM_COEFFS    15426,  5520
TRANSFORM_COEFFS     3981, 15893
TRANSFORM_COEFFS    16207,  2404
TRANSFORM_COEFFS     1606, 16305
TRANSFORM_COEFFS    15679,  4756
TRANSFORM_COEFFS    11585, 11585

; constants for 32x32_1024
TRANSFORM_COEFFS    12140, 11003
TRANSFORM_COEFFS     7005, 14811
TRANSFORM_COEFFS    14053,  8423
TRANSFORM_COEFFS     9760, 13160
TRANSFORM_COEFFS    12665, 10394
TRANSFORM_COEFFS     7723, 14449

%macro PAIR_PP_COEFFS 2
dpw_%1_%2:   dw  %1,  %1,  %1,  %1,  %2,  %2,  %2,  %2
%endmacro

%macro PAIR_MP_COEFFS 2
dpw_m%1_%2:  dw -%1, -%1, -%1, -%1,  %2,  %2,  %2,  %2
%endmacro

%macro PAIR_MM_COEFFS 2
dpw_m%1_m%2: dw -%1, -%1, -%1, -%1, -%2, -%2, -%2, -%2
%endmacro

PAIR_PP_COEFFS     30274, 12540
PAIR_PP_COEFFS      6392, 32138
PAIR_MP_COEFFS     18204, 27246

PAIR_PP_COEFFS     12540, 12540
PAIR_PP_COEFFS     30274, 30274
PAIR_PP_COEFFS      6392,  6392
PAIR_PP_COEFFS     32138, 32138
PAIR_MM_COEFFS     18204, 18204
PAIR_PP_COEFFS     27246, 27246

SECTION .text

%if ARCH_X86_64
%macro SUM_SUB 3
  psubw  m%3, m%1, m%2
  paddw  m%1, m%2
  SWAP    %2, %3
%endmacro

; butterfly operation
%macro MUL_ADD_2X 6 ; dst1, dst2, src, round, coefs1, coefs2
  pmaddwd            m%1, m%3, %5
  pmaddwd            m%2, m%3, %6
  paddd              m%1,  %4
  paddd              m%2,  %4
  psrad              m%1,  14
  psrad              m%2,  14
%endmacro

%macro BUTTERFLY_4X 7 ; dst1, dst2, coef1, coef2, round, tmp1, tmp2
  punpckhwd          m%6, m%2, m%1
  MUL_ADD_2X         %7,  %6,  %6,  %5, [pw_m%4_%3], [pw_%3_%4]
  punpcklwd          m%2, m%1
  MUL_ADD_2X         %1,  %2,  %2,  %5, [pw_m%4_%3], [pw_%3_%4]
  packssdw           m%1, m%7
  packssdw           m%2, m%6
%endmacro

%macro BUTTERFLY_4Xmm 7 ; dst1, dst2, coef1, coef2, round, tmp1, tmp2
  punpckhwd          m%6, m%2, m%1
  MUL_ADD_2X         %7,  %6,  %6,  %5, [pw_m%4_%3], [pw_m%3_m%4]
  punpcklwd          m%2, m%1
  MUL_ADD_2X         %1,  %2,  %2,  %5, [pw_m%4_%3], [pw_m%3_m%4]
  packssdw           m%1, m%7
  packssdw           m%2, m%6
%endmacro

; matrix transpose
%macro INTERLEAVE_2X 4
  punpckh%1          m%4, m%2, m%3
  punpckl%1          m%2, m%3
  SWAP               %3,  %4
%endmacro

%macro TRANSPOSE8X8 9
  INTERLEAVE_2X  wd, %1, %2, %9
  INTERLEAVE_2X  wd, %3, %4, %9
  INTERLEAVE_2X  wd, %5, %6, %9
  INTERLEAVE_2X  wd, %7, %8, %9

  INTERLEAVE_2X  dq, %1, %3, %9
  INTERLEAVE_2X  dq, %2, %4, %9
  INTERLEAVE_2X  dq, %5, %7, %9
  INTERLEAVE_2X  dq, %6, %8, %9

  INTERLEAVE_2X  qdq, %1, %5, %9
  INTERLEAVE_2X  qdq, %3, %7, %9
  INTERLEAVE_2X  qdq, %2, %6, %9
  INTERLEAVE_2X  qdq, %4, %8, %9

  SWAP  %2, %5
  SWAP  %4, %7
%endmacro

%macro IDCT8_1D 0
  SUM_SUB          0,    4,    9
  BUTTERFLY_4X     2,    6,    6270, 15137,  m8,  9,  10
  pmulhrsw        m0,  m12
  pmulhrsw        m4,  m12
  BUTTERFLY_4X     1,    7,    3196, 16069,  m8,  9,  10
  BUTTERFLY_4X     5,    3,   13623,  9102,  m8,  9,  10

  SUM_SUB          1,    5,    9
  SUM_SUB          7,    3,    9
  SUM_SUB          0,    6,    9
  SUM_SUB          4,    2,    9
  SUM_SUB          3,    5,    9
  pmulhrsw        m3,  m12
  pmulhrsw        m5,  m12

  SUM_SUB          0,    7,    9
  SUM_SUB          4,    3,    9
  SUM_SUB          2,    5,    9
  SUM_SUB          6,    1,    9

  SWAP             3,    6
  SWAP             1,    4
%endmacro

; This macro handles 8 pixels per line
%macro ADD_STORE_8P_2X 5;  src1, src2, tmp1, tmp2, zero
  paddw           m%1, m11
  paddw           m%2, m11
  psraw           m%1, 5
  psraw           m%2, 5

  movh            m%3, [outputq]
  movh            m%4, [outputq + strideq]
  punpcklbw       m%3, m%5
  punpcklbw       m%4, m%5
  paddw           m%3, m%1
  paddw           m%4, m%2
  packuswb        m%3, m%5
  packuswb        m%4, m%5
  movh               [outputq], m%3
  movh     [outputq + strideq], m%4
%endmacro

INIT_XMM ssse3
; full inverse 8x8 2D-DCT transform
cglobal idct8x8_64_add, 3, 5, 13, input, output, stride
  mova     m8, [pd_8192]
  mova    m11, [pw_16]
  mova    m12, [pw_11585x2]

  lea      r3, [2 * strideq]
%if CONFIG_VP9_HIGHBITDEPTH
  mova     m0, [inputq +   0]
  packssdw m0, [inputq +  16]
  mova     m1, [inputq +  32]
  packssdw m1, [inputq +  48]
  mova     m2, [inputq +  64]
  packssdw m2, [inputq +  80]
  mova     m3, [inputq +  96]
  packssdw m3, [inputq + 112]
  mova     m4, [inputq + 128]
  packssdw m4, [inputq + 144]
  mova     m5, [inputq + 160]
  packssdw m5, [inputq + 176]
  mova     m6, [inputq + 192]
  packssdw m6, [inputq + 208]
  mova     m7, [inputq + 224]
  packssdw m7, [inputq + 240]
%else
  mova     m0, [inputq +   0]
  mova     m1, [inputq +  16]
  mova     m2, [inputq +  32]
  mova     m3, [inputq +  48]
  mova     m4, [inputq +  64]
  mova     m5, [inputq +  80]
  mova     m6, [inputq +  96]
  mova     m7, [inputq + 112]
%endif
  TRANSPOSE8X8  0, 1, 2, 3, 4, 5, 6, 7, 9
  IDCT8_1D
  TRANSPOSE8X8  0, 1, 2, 3, 4, 5, 6, 7, 9
  IDCT8_1D

  pxor    m12, m12
  ADD_STORE_8P_2X  0, 1, 9, 10, 12
  lea              outputq, [outputq + r3]
  ADD_STORE_8P_2X  2, 3, 9, 10, 12
  lea              outputq, [outputq + r3]
  ADD_STORE_8P_2X  4, 5, 9, 10, 12
  lea              outputq, [outputq + r3]
  ADD_STORE_8P_2X  6, 7, 9, 10, 12

  RET

; inverse 8x8 2D-DCT transform with only first 10 coeffs non-zero
cglobal idct8x8_12_add, 3, 5, 13, input, output, stride
  mova       m8, [pd_8192]
  mova      m11, [pw_16]
  mova      m12, [pw_11585x2]

  lea        r3, [2 * strideq]

%if CONFIG_VP9_HIGHBITDEPTH
  mova       m0, [inputq +   0]
  packssdw   m0, [inputq +  16]
  mova       m1, [inputq +  32]
  packssdw   m1, [inputq +  48]
  mova       m2, [inputq +  64]
  packssdw   m2, [inputq +  80]
  mova       m3, [inputq +  96]
  packssdw   m3, [inputq + 112]
%else
  mova       m0, [inputq +  0]
  mova       m1, [inputq + 16]
  mova       m2, [inputq + 32]
  mova       m3, [inputq + 48]
%endif

  punpcklwd  m0, m1
  punpcklwd  m2, m3
  punpckhdq  m9, m0, m2
  punpckldq  m0, m2
  SWAP       2, 9

  ; m0 -> [0], [0]
  ; m1 -> [1], [1]
  ; m2 -> [2], [2]
  ; m3 -> [3], [3]
  punpckhqdq m10, m0, m0
  punpcklqdq m0,  m0
  punpckhqdq m9,  m2, m2
  punpcklqdq m2,  m2
  SWAP       1, 10
  SWAP       3,  9

  pmulhrsw   m0, m12
  pmulhrsw   m2, [dpw_30274_12540]
  pmulhrsw   m1, [dpw_6392_32138]
  pmulhrsw   m3, [dpw_m18204_27246]

  SUM_SUB    0, 2, 9
  SUM_SUB    1, 3, 9

  punpcklqdq m9, m3, m3
  punpckhqdq m5, m3, m9

  SUM_SUB    3, 5, 9
  punpckhqdq m5, m3
  pmulhrsw   m5, m12

  punpckhqdq m9, m1, m5
  punpcklqdq m1, m5
  SWAP       5, 9

  SUM_SUB    0, 5, 9
  SUM_SUB    2, 1, 9

  punpckhqdq m3, m0, m0
  punpckhqdq m4, m1, m1
  punpckhqdq m6, m5, m5
  punpckhqdq m7, m2, m2

  punpcklwd  m0, m3
  punpcklwd  m7, m2
  punpcklwd  m1, m4
  punpcklwd  m6, m5

  punpckhdq  m4, m0, m7
  punpckldq  m0, m7
  punpckhdq  m10, m1, m6
  punpckldq  m5, m1, m6

  punpckhqdq m1, m0, m5
  punpcklqdq m0, m5
  punpckhqdq m3, m4, m10
  punpcklqdq m2, m4, m10


  pmulhrsw   m0, m12
  pmulhrsw   m6, m2, [dpw_30274_30274]
  pmulhrsw   m4, m2, [dpw_12540_12540]

  pmulhrsw   m7, m1, [dpw_32138_32138]
  pmulhrsw   m1, [dpw_6392_6392]
  pmulhrsw   m5, m3, [dpw_m18204_m18204]
  pmulhrsw   m3, [dpw_27246_27246]

  mova       m2, m0
  SUM_SUB    0, 6, 9
  SUM_SUB    2, 4, 9
  SUM_SUB    1, 5, 9
  SUM_SUB    7, 3, 9

  SUM_SUB    3, 5, 9
  pmulhrsw   m3, m12
  pmulhrsw   m5, m12

  SUM_SUB    0, 7, 9
  SUM_SUB    2, 3, 9
  SUM_SUB    4, 5, 9
  SUM_SUB    6, 1, 9

  SWAP       3, 6
  SWAP       1, 2
  SWAP       2, 4


  pxor    m12, m12
  ADD_STORE_8P_2X  0, 1, 9, 10, 12
  lea              outputq, [outputq + r3]
  ADD_STORE_8P_2X  2, 3, 9, 10, 12
  lea              outputq, [outputq + r3]
  ADD_STORE_8P_2X  4, 5, 9, 10, 12
  lea              outputq, [outputq + r3]
  ADD_STORE_8P_2X  6, 7, 9, 10, 12

  RET

%define  idx0 16 * 0
%define  idx1 16 * 1
%define  idx2 16 * 2
%define  idx3 16 * 3
%define  idx4 16 * 4
%define  idx5 16 * 5
%define  idx6 16 * 6
%define  idx7 16 * 7
%define  idx8 16 * 0
%define  idx9 16 * 1
%define idx10 16 * 2
%define idx11 16 * 3
%define idx12 16 * 4
%define idx13 16 * 5
%define idx14 16 * 6
%define idx15 16 * 7
%define idx16 16 * 0
%define idx17 16 * 1
%define idx18 16 * 2
%define idx19 16 * 3
%define idx20 16 * 4
%define idx21 16 * 5
%define idx22 16 * 6
%define idx23 16 * 7
%define idx24 16 * 0
%define idx25 16 * 1
%define idx26 16 * 2
%define idx27 16 * 3
%define idx28 16 * 4
%define idx29 16 * 5
%define idx30 16 * 6
%define idx31 16 * 7

; FROM idct32x32_add_neon.asm
;
; Instead of doing the transforms stage by stage, it is done by loading
; some input values and doing as many stages as possible to minimize the
; storing/loading of intermediate results. To fit within registers, the
; final coefficients are cut into four blocks:
; BLOCK A: 16-19,28-31
; BLOCK B: 20-23,24-27
; BLOCK C: 8-11,12-15
; BLOCK D: 0-3,4-7
; Blocks A and C are straight calculation through the various stages. In
; block B, further calculations are performed using the results from
; block A. In block D, further calculations are performed using the results
; from block C and then the final calculations are done using results from
; block A and B which have been combined at the end of block B.
;

%macro IDCT32X32_34 4
  ; BLOCK A STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                m11, m1
  pmulhrsw             m1, [pw___804x2] ; stp1_16
  mova      [r4 +      0], m0
  pmulhrsw            m11, [pw_16364x2] ; stp2_31
  mova      [r4 + 16 * 2], m2
  mova                m12, m7
  pmulhrsw             m7, [pw_15426x2] ; stp1_28
  mova      [r4 + 16 * 4], m4
  pmulhrsw            m12, [pw_m5520x2] ; stp2_19
  mova      [r4 + 16 * 6], m6

  ; BLOCK A STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                 m2, m1   ; stp1_16
  mova                 m0, m11  ; stp1_31
  mova                 m4, m7   ; stp1_28
  mova                m15, m12  ; stp1_19

  ; BLOCK A STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  BUTTERFLY_4X          0,     2,   3196, 16069,  m8,  9,  10 ; stp1_17, stp1_30
  BUTTERFLY_4Xmm        4,    15,   3196, 16069,  m8,  9,  10 ; stp1_29, stp1_18

  ; BLOCK A STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               1, 12, 9 ; stp2_16, stp2_19
  SUM_SUB               0, 15, 9 ; stp2_17, stp2_18
  SUM_SUB              11,  7, 9 ; stp2_31, stp2_28
  SUM_SUB               2,  4, 9 ; stp2_30, stp2_29

  ; BLOCK A STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  BUTTERFLY_4X          4,    15,   6270, 15137,  m8,  9,  10 ; stp1_18, stp1_29
  BUTTERFLY_4X          7,    12,   6270, 15137,  m8,  9,  10 ; stp1_19, stp1_28

  ; BLOCK B STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                 m6, m5
  pmulhrsw             m5, [pw__3981x2] ; stp1_20
  mova [stp + %4 + idx28], m12
  mova [stp + %4 + idx29], m15
  pmulhrsw             m6, [pw_15893x2] ; stp2_27
  mova [stp + %4 + idx30], m2
  mova                 m2, m3
  pmulhrsw             m3, [pw_m2404x2] ; stp1_23
  mova [stp + %4 + idx31], m11
  pmulhrsw             m2, [pw_16207x2] ; stp2_24

  ; BLOCK B STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                m13, m5 ; stp1_20
  mova                m14, m6 ; stp1_27
  mova                m15, m3 ; stp1_23
  mova                m11, m2 ; stp1_24

  ; BLOCK B STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  BUTTERFLY_4X         14,    13,  13623,  9102,  m8,  9,  10 ; stp1_21, stp1_26
  BUTTERFLY_4Xmm       11,    15,  13623,  9102,  m8,  9,  10 ; stp1_25, stp1_22

  ; BLOCK B STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               3,  5, 9 ; stp2_23, stp2_20
  SUM_SUB              15, 14, 9 ; stp2_22, stp2_21
  SUM_SUB               2,  6, 9 ; stp2_24, stp2_27
  SUM_SUB              11, 13, 9 ; stp2_25, stp2_26

  ; BLOCK B STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  BUTTERFLY_4Xmm        6,     5,   6270, 15137,  m8,  9,  10 ; stp1_27, stp1_20
  BUTTERFLY_4Xmm       13,    14,   6270, 15137,  m8,  9,  10 ; stp1_26, stp1_21

  ; BLOCK B STAGE 6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               1,  3, 9 ; stp2_16, stp2_23
  SUM_SUB               0, 15, 9 ; stp2_17, stp2_22
  SUM_SUB               4, 14, 9 ; stp2_18, stp2_21
  SUM_SUB               7,  5, 9 ; stp2_19, stp2_20
  mova [stp + %3 + idx16], m1
  mova [stp + %3 + idx17], m0
  mova [stp + %3 + idx18], m4
  mova [stp + %3 + idx19], m7

  mova                 m4, [stp + %4 + idx28]
  mova                 m7, [stp + %4 + idx29]
  mova                m10, [stp + %4 + idx30]
  mova                m12, [stp + %4 + idx31]
  SUM_SUB               4,  6, 9 ; stp2_28, stp2_27
  SUM_SUB               7, 13, 9 ; stp2_29, stp2_26
  SUM_SUB              10, 11, 9 ; stp2_30, stp2_25
  SUM_SUB              12,  2, 9 ; stp2_31, stp2_24
  mova [stp + %4 + idx28], m4
  mova [stp + %4 + idx29], m7
  mova [stp + %4 + idx30], m10
  mova [stp + %4 + idx31], m12

  ; BLOCK B STAGE 7 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
%if 0 ; overflow occurs in SUM_SUB when using test streams
  mova                m10, [pw_11585x2]
  SUM_SUB               6,  5, 9
  pmulhrsw             m6, m10  ; stp1_27
  pmulhrsw             m5, m10  ; stp1_20
  SUM_SUB              13, 14,  9
  pmulhrsw            m13, m10  ; stp1_26
  pmulhrsw            m14, m10  ; stp1_21
  SUM_SUB              11, 15,  9
  pmulhrsw            m11, m10  ; stp1_25
  pmulhrsw            m15, m10  ; stp1_22
  SUM_SUB               2,  3,  9
  pmulhrsw             m2, m10  ; stp1_24
  pmulhrsw             m3, m10  ; stp1_23
%else
  BUTTERFLY_4X          6,     5,  11585, 11585,  m8,  9,  10 ; stp1_20, stp1_27
  SWAP 6, 5
  BUTTERFLY_4X         13,    14,  11585, 11585,  m8,  9,  10 ; stp1_21, stp1_26
  SWAP 13, 14
  BUTTERFLY_4X         11,    15,  11585, 11585,  m8,  9,  10 ; stp1_22, stp1_25
  SWAP 11, 15
  BUTTERFLY_4X          2,     3,  11585, 11585,  m8,  9,  10 ; stp1_23, stp1_24
  SWAP 2, 3
%endif

  mova [stp + %4 + idx24], m2
  mova [stp + %4 + idx25], m11
  mova [stp + %4 + idx26], m13
  mova [stp + %4 + idx27], m6

  ; BLOCK C STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  ;
  ; BLOCK C STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                 m0, [rsp + transposed_in + 16 *  2]
  mova                 m6, [rsp + transposed_in + 16 *  6]

  mova                 m1, m0
  pmulhrsw             m0, [pw__1606x2] ; stp1_8
  mova [stp + %3 + idx20], m5
  mova [stp + %3 + idx21], m14
  pmulhrsw             m1, [pw_16305x2] ; stp2_15
  mova [stp + %3 + idx22], m15
  mova                 m7, m6
  pmulhrsw             m7, [pw_m4756x2] ; stp2_11
  mova [stp + %3 + idx23], m3
  pmulhrsw             m6, [pw_15679x2] ; stp1_12

  ; BLOCK C STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                 m3, m0 ; stp1_8
  mova                 m2, m1 ; stp1_15

  ; BLOCK C STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  BUTTERFLY_4X          2,     3,   6270, 15137,  m8,  9,  10 ;  stp1_9, stp1_14
  mova                 m4, m7 ; stp1_11
  mova                 m5, m6 ; stp1_12
  BUTTERFLY_4Xmm        5,     4,   6270, 15137,  m8,  9,  10 ; stp1_13, stp1_10

  ; BLOCK C STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               0,  7, 9 ;  stp1_8, stp1_11
  SUM_SUB               2,  4, 9 ;  stp1_9, stp1_10
  SUM_SUB               1,  6, 9 ;  stp1_15, stp1_12
  SUM_SUB               3,  5, 9 ;  stp1_14, stp1_13

  ; BLOCK C STAGE 6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
%if 0 ; overflow occurs in SUM_SUB when using test streams
  mova                m10, [pw_11585x2]
  SUM_SUB               5,  4, 9
  pmulhrsw             m5, m10  ; stp1_13
  pmulhrsw             m4, m10  ; stp1_10
  SUM_SUB               6,  7, 9
  pmulhrsw             m6, m10  ; stp1_12
  pmulhrsw             m7, m10  ; stp1_11
%else
  BUTTERFLY_4X          5,     4,  11585, 11585,  m8,  9,  10 ; stp1_10, stp1_13
  SWAP 5, 4
  BUTTERFLY_4X          6,     7,  11585, 11585,  m8,  9,  10 ; stp1_11, stp1_12
  SWAP 6, 7
%endif

  ; BLOCK C STAGE 7 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova [stp + %2 +  idx8], m0
  mova [stp + %2 +  idx9], m2
  mova [stp + %2 + idx10], m4
  mova [stp + %2 + idx11], m7

  ; BLOCK D STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  ;
  ; BLOCK D STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  ;
  ; BLOCK D STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                m11, [rsp + transposed_in + 16 *  4]
  mova                m12, m11
  pmulhrsw            m11, [pw__3196x2] ; stp1_4
  pmulhrsw            m12, [pw_16069x2] ; stp1_7

  ; BLOCK D STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                 m0, [rsp + transposed_in + 16 *  0]
  mova                m10, [pw_11585x2]
  pmulhrsw             m0, m10  ; stp1_1

  mova                m14, m11 ; stp1_4
  mova                m13, m12 ; stp1_7

  ; BLOCK D STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
%if 0 ; overflow occurs in SUM_SUB when using test streams
  SUM_SUB              13,   14,  9
  pmulhrsw            m13, m10  ; stp1_6
  pmulhrsw            m14, m10  ; stp1_5
%else
  BUTTERFLY_4X         13,    14,  11585, 11585,  m8,  9,  10 ; stp1_5, stp1_6
  SWAP 13, 14
%endif
  mova                 m7, m0 ; stp1_0 = stp1_1
  mova                 m4, m0 ; stp1_1
  mova                 m2, m7 ; stp1_0

  ; BLOCK D STAGE 6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               0, 12, 9 ;  stp1_0, stp1_7
  SUM_SUB               7, 13, 9 ;  stp1_1, stp1_6
  SUM_SUB               2, 14, 9 ;  stp1_2, stp1_5
  SUM_SUB               4, 11, 9 ;  stp1_3, stp1_4

  ; BLOCK D STAGE 7 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               0,  1, 9 ;  stp1_0, stp1_15
  SUM_SUB               7,  3, 9 ;  stp1_1, stp1_14
  SUM_SUB               2,  5, 9 ;  stp1_2, stp1_13
  SUM_SUB               4,  6, 9 ;  stp1_3, stp1_12

  ; 0-3, 28-31 final stage
  mova                m15, [stp + %4 + idx30]
  mova                m10, [stp + %4 + idx31]
  SUM_SUB               0, 10, 9 ;  stp1_0, stp1_31
  SUM_SUB               7, 15, 9 ;  stp1_1, stp1_30
  mova [stp + %1 +  idx0], m0
  mova [stp + %1 +  idx1], m7
  mova [stp + %4 + idx30], m15
  mova [stp + %4 + idx31], m10
  mova                 m7, [stp + %4 + idx28]
  mova                 m0, [stp + %4 + idx29]
  SUM_SUB               2,  0, 9 ;  stp1_2, stp1_29
  SUM_SUB               4,  7, 9 ;  stp1_3, stp1_28
  mova [stp + %1 +  idx2], m2
  mova [stp + %1 +  idx3], m4
  mova [stp + %4 + idx28], m7
  mova [stp + %4 + idx29], m0

  ; 12-15, 16-19 final stage
  mova                 m0, [stp + %3 + idx16]
  mova                 m7, [stp + %3 + idx17]
  mova                 m2, [stp + %3 + idx18]
  mova                 m4, [stp + %3 + idx19]
  SUM_SUB               1,  0, 9 ;  stp1_15, stp1_16
  SUM_SUB               3,  7, 9 ;  stp1_14, stp1_17
  SUM_SUB               5,  2, 9 ;  stp1_13, stp1_18
  SUM_SUB               6,  4, 9 ;  stp1_12, stp1_19
  mova [stp + %2 + idx12], m6
  mova [stp + %2 + idx13], m5
  mova [stp + %2 + idx14], m3
  mova [stp + %2 + idx15], m1
  mova [stp + %3 + idx16], m0
  mova [stp + %3 + idx17], m7
  mova [stp + %3 + idx18], m2
  mova [stp + %3 + idx19], m4

  mova                 m4, [stp + %2 +  idx8]
  mova                 m5, [stp + %2 +  idx9]
  mova                 m6, [stp + %2 + idx10]
  mova                 m7, [stp + %2 + idx11]
  SUM_SUB              11,  7, 9 ;  stp1_4, stp1_11
  SUM_SUB              14,  6, 9 ;  stp1_5, stp1_10
  SUM_SUB              13,  5, 9 ;  stp1_6, stp1_9
  SUM_SUB              12,  4, 9 ;  stp1_7, stp1_8

  ; 4-7, 24-27 final stage
  mova                 m0, [stp + %4 + idx27]
  mova                 m1, [stp + %4 + idx26]
  mova                 m2, [stp + %4 + idx25]
  mova                 m3, [stp + %4 + idx24]
  SUM_SUB              11,  0, 9 ;  stp1_4, stp1_27
  SUM_SUB              14,  1, 9 ;  stp1_5, stp1_26
  SUM_SUB              13,  2, 9 ;  stp1_6, stp1_25
  SUM_SUB              12,  3, 9 ;  stp1_7, stp1_24
  mova [stp + %4 + idx27], m0
  mova [stp + %4 + idx26], m1
  mova [stp + %4 + idx25], m2
  mova [stp + %4 + idx24], m3
  mova [stp + %1 +  idx4], m11
  mova [stp + %1 +  idx5], m14
  mova [stp + %1 +  idx6], m13
  mova [stp + %1 +  idx7], m12

  ; 8-11, 20-23 final stage
  mova                 m0, [stp + %3 + idx20]
  mova                 m1, [stp + %3 + idx21]
  mova                 m2, [stp + %3 + idx22]
  mova                 m3, [stp + %3 + idx23]
  SUM_SUB               7,  0, 9 ;  stp1_11, stp_20
  SUM_SUB               6,  1, 9 ;  stp1_10, stp_21
  SUM_SUB               5,  2, 9 ;   stp1_9, stp_22
  SUM_SUB               4,  3, 9 ;   stp1_8, stp_23
  mova [stp + %2 +  idx8], m4
  mova [stp + %2 +  idx9], m5
  mova [stp + %2 + idx10], m6
  mova [stp + %2 + idx11], m7
  mova [stp + %3 + idx20], m0
  mova [stp + %3 + idx21], m1
  mova [stp + %3 + idx22], m2
  mova [stp + %3 + idx23], m3
%endmacro

%macro RECON_AND_STORE 1
  mova            m11, [pw_32]
  lea             stp, [rsp + %1]
  mov              r6, 32
  pxor             m8, m8
%%recon_and_store:
  mova             m0, [stp + 16 * 32 * 0]
  mova             m1, [stp + 16 * 32 * 1]
  mova             m2, [stp + 16 * 32 * 2]
  mova             m3, [stp + 16 * 32 * 3]
  add             stp, 16

  paddw            m0, m11
  paddw            m1, m11
  paddw            m2, m11
  paddw            m3, m11
  psraw            m0, 6
  psraw            m1, 6
  psraw            m2, 6
  psraw            m3, 6
  movh             m4, [outputq +  0]
  movh             m5, [outputq +  8]
  movh             m6, [outputq + 16]
  movh             m7, [outputq + 24]
  punpcklbw        m4, m8
  punpcklbw        m5, m8
  punpcklbw        m6, m8
  punpcklbw        m7, m8
  paddw            m0, m4
  paddw            m1, m5
  paddw            m2, m6
  paddw            m3, m7
  packuswb         m0, m1
  packuswb         m2, m3
  mova [outputq +  0], m0
  mova [outputq + 16], m2
  lea         outputq, [outputq + strideq]
  dec              r6
  jnz %%recon_and_store
%endmacro

%define i32x32_size     16*32*5
%define pass_two_start  16*32*0
%define transposed_in   16*32*4
%define pass_one_start  16*32*0
%define stp r8

INIT_XMM ssse3
cglobal idct32x32_34_add, 3, 11, 16, i32x32_size, input, output, stride
  mova            m8, [pd_8192]
  lea            stp, [rsp + pass_one_start]

idct32x32_34:
  mov             r3, inputq
  lea             r4, [rsp + transposed_in]

idct32x32_34_transpose:
%if CONFIG_VP9_HIGHBITDEPTH
  mova            m0, [r3 +       0]
  packssdw        m0, [r3 +      16]
  mova            m1, [r3 + 32 *  4]
  packssdw        m1, [r3 + 32 *  4 + 16]
  mova            m2, [r3 + 32 *  8]
  packssdw        m2, [r3 + 32 *  8 + 16]
  mova            m3, [r3 + 32 * 12]
  packssdw        m3, [r3 + 32 * 12 + 16]
  mova            m4, [r3 + 32 * 16]
  packssdw        m4, [r3 + 32 * 16 + 16]
  mova            m5, [r3 + 32 * 20]
  packssdw        m5, [r3 + 32 * 20 + 16]
  mova            m6, [r3 + 32 * 24]
  packssdw        m6, [r3 + 32 * 24 + 16]
  mova            m7, [r3 + 32 * 28]
  packssdw        m7, [r3 + 32 * 28 + 16]
%else
  mova            m0, [r3 +       0]
  mova            m1, [r3 + 16 *  4]
  mova            m2, [r3 + 16 *  8]
  mova            m3, [r3 + 16 * 12]
  mova            m4, [r3 + 16 * 16]
  mova            m5, [r3 + 16 * 20]
  mova            m6, [r3 + 16 * 24]
  mova            m7, [r3 + 16 * 28]
%endif

  TRANSPOSE8X8  0, 1, 2, 3, 4, 5, 6, 7, 9

  IDCT32X32_34  16*0, 16*32, 16*64, 16*96
  lea            stp, [stp + 16 * 8]
  mov             r6, 4
  lea            stp, [rsp + pass_one_start]
  lea             r9, [rsp + pass_one_start]

idct32x32_34_2:
  lea             r4, [rsp + transposed_in]
  mov             r3, r9

idct32x32_34_transpose_2:
  mova            m0, [r3 +      0]
  mova            m1, [r3 + 16 * 1]
  mova            m2, [r3 + 16 * 2]
  mova            m3, [r3 + 16 * 3]
  mova            m4, [r3 + 16 * 4]
  mova            m5, [r3 + 16 * 5]
  mova            m6, [r3 + 16 * 6]
  mova            m7, [r3 + 16 * 7]

  TRANSPOSE8X8  0, 1, 2, 3, 4, 5, 6, 7, 9

  IDCT32X32_34  16*0, 16*8, 16*16, 16*24

  lea            stp, [stp + 16 * 32]
  add             r9, 16 * 32
  dec             r6
  jnz idct32x32_34_2

  RECON_AND_STORE pass_two_start

  RET

%macro IDCT32X32_135 4
  ; BLOCK A STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                 m1, [rsp + transposed_in + 16 *  1]
  mova                m11, m1
  pmulhrsw             m1, [pw___804x2] ; stp1_16
  pmulhrsw            m11, [pw_16364x2] ; stp2_31

  mova                 m7, [rsp + transposed_in + 16 *  7]
  mova                m12, m7
  pmulhrsw             m7, [pw_15426x2] ; stp1_28
  pmulhrsw            m12, [pw_m5520x2] ; stp2_19

  mova                 m3, [rsp + transposed_in + 16 *  9]
  mova                 m4, m3
  pmulhrsw             m3, [pw__7005x2] ; stp1_18
  pmulhrsw             m4, [pw_14811x2] ; stp2_29

  mova                 m0, [rsp + transposed_in + 16 * 15]
  mova                 m2, m0
  pmulhrsw             m0, [pw_12140x2]  ; stp1_30
  pmulhrsw             m2, [pw_m11003x2] ; stp2_17

  ; BLOCK A STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               1,  2, 9 ; stp2_16, stp2_17
  SUM_SUB              12,  3, 9 ; stp2_19, stp2_18
  SUM_SUB               7,  4, 9 ; stp2_28, stp2_29
  SUM_SUB              11,  0, 9 ; stp2_31, stp2_30

  ; BLOCK A STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  BUTTERFLY_4X          0,     2,   3196, 16069,  m8,  9,  10 ; stp1_17, stp1_30
  BUTTERFLY_4Xmm        4,     3,   3196, 16069,  m8,  9,  10 ; stp1_29, stp1_18

  ; BLOCK A STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               1, 12, 9 ; stp2_16, stp2_19
  SUM_SUB               0,  3, 9 ; stp2_17, stp2_18
  SUM_SUB              11,  7, 9 ; stp2_31, stp2_28
  SUM_SUB               2,  4, 9 ; stp2_30, stp2_29

  ; BLOCK A STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  BUTTERFLY_4X          4,     3,   6270, 15137,  m8,  9,  10 ; stp1_18, stp1_29
  BUTTERFLY_4X          7,    12,   6270, 15137,  m8,  9,  10 ; stp1_19, stp1_28

  mova [stp + %3 + idx16], m1
  mova [stp + %3 + idx17], m0
  mova [stp + %3 + idx18], m4
  mova [stp + %3 + idx19], m7
  mova [stp + %4 + idx28], m12
  mova [stp + %4 + idx29], m3
  mova [stp + %4 + idx30], m2
  mova [stp + %4 + idx31], m11

  ; BLOCK B STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                 m2, [rsp + transposed_in + 16 *  3]
  mova                 m3, m2
  pmulhrsw             m3, [pw_m2404x2] ; stp1_23
  pmulhrsw             m2, [pw_16207x2] ; stp2_24

  mova                 m5, [rsp + transposed_in + 16 *  5]
  mova                 m6, m5
  pmulhrsw             m5, [pw__3981x2] ; stp1_20
  pmulhrsw             m6, [pw_15893x2] ; stp2_27

  mova                m14, [rsp + transposed_in + 16 * 11]
  mova                m13, m14
  pmulhrsw            m13, [pw_m8423x2] ; stp1_21
  pmulhrsw            m14, [pw_14053x2] ; stp2_26

  mova                 m0, [rsp + transposed_in + 16 * 13]
  mova                 m1, m0
  pmulhrsw             m0, [pw__9760x2] ; stp1_22
  pmulhrsw             m1, [pw_13160x2] ; stp2_25

  ; BLOCK B STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               5, 13, 9 ; stp2_20, stp2_21
  SUM_SUB               3,  0, 9 ; stp2_23, stp2_22
  SUM_SUB               2,  1, 9 ; stp2_24, stp2_25
  SUM_SUB               6, 14, 9 ; stp2_27, stp2_26

  ; BLOCK B STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  BUTTERFLY_4X         14,    13,  13623,  9102,  m8,  9,  10 ; stp1_21, stp1_26
  BUTTERFLY_4Xmm        1,     0,  13623,  9102,  m8,  9,  10 ; stp1_25, stp1_22

  ; BLOCK B STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               3,  5, 9 ; stp2_23, stp2_20
  SUM_SUB               0, 14, 9 ; stp2_22, stp2_21
  SUM_SUB               2,  6, 9 ; stp2_24, stp2_27
  SUM_SUB               1, 13, 9 ; stp2_25, stp2_26

  ; BLOCK B STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  BUTTERFLY_4Xmm        6,     5,   6270, 15137,  m8,  9,  10 ; stp1_27, stp1_20
  BUTTERFLY_4Xmm       13,    14,   6270, 15137,  m8,  9,  10 ; stp1_26, stp1_21

  ; BLOCK B STAGE 6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                 m4, [stp + %3 + idx16]
  mova                 m7, [stp + %3 + idx17]
  mova                m11, [stp + %3 + idx18]
  mova                m12, [stp + %3 + idx19]
  SUM_SUB               4,  3, 9 ; stp2_16, stp2_23
  SUM_SUB               7,  0, 9 ; stp2_17, stp2_22
  SUM_SUB              11, 14, 9 ; stp2_18, stp2_21
  SUM_SUB              12,  5, 9 ; stp2_19, stp2_20
  mova [stp + %3 + idx16], m4
  mova [stp + %3 + idx17], m7
  mova [stp + %3 + idx18], m11
  mova [stp + %3 + idx19], m12

  mova                 m4, [stp + %4 + idx28]
  mova                 m7, [stp + %4 + idx29]
  mova                m11, [stp + %4 + idx30]
  mova                m12, [stp + %4 + idx31]
  SUM_SUB               4,  6, 9 ; stp2_28, stp2_27
  SUM_SUB               7, 13, 9 ; stp2_29, stp2_26
  SUM_SUB              11,  1, 9 ; stp2_30, stp2_25
  SUM_SUB              12,  2, 9 ; stp2_31, stp2_24
  mova [stp + %4 + idx28], m4
  mova [stp + %4 + idx29], m7
  mova [stp + %4 + idx30], m11
  mova [stp + %4 + idx31], m12

  ; BLOCK B STAGE 7 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
%if 0 ; overflow occurs in SUM_SUB when using test streams
  mova                m10, [pw_11585x2]
  SUM_SUB               6,  5,  9
  pmulhrsw             m6, m10  ; stp1_27
  pmulhrsw             m5, m10  ; stp1_20
  SUM_SUB              13, 14,  9
  pmulhrsw            m13, m10  ; stp1_26
  pmulhrsw            m14, m10  ; stp1_21
  SUM_SUB               1,  0,  9
  pmulhrsw             m1, m10  ; stp1_25
  pmulhrsw             m0, m10  ; stp1_22
  SUM_SUB               2,  3,  9
  pmulhrsw             m2, m10  ; stp1_25
  pmulhrsw             m3, m10  ; stp1_22
%else
  BUTTERFLY_4X          6,     5,  11585, 11585,  m8,  9,  10 ; stp1_20, stp1_27
  SWAP  6, 5
  BUTTERFLY_4X         13,    14,  11585, 11585,  m8,  9,  10 ; stp1_21, stp1_26
  SWAP 13, 14
  BUTTERFLY_4X          1,     0,  11585, 11585,  m8,  9,  10 ; stp1_22, stp1_25
  SWAP  1, 0
  BUTTERFLY_4X          2,     3,  11585, 11585,  m8,  9,  10 ; stp1_23, stp1_24
  SWAP  2, 3
%endif
  mova [stp + %3 + idx20], m5
  mova [stp + %3 + idx21], m14
  mova [stp + %3 + idx22], m0
  mova [stp + %3 + idx23], m3
  mova [stp + %4 + idx24], m2
  mova [stp + %4 + idx25], m1
  mova [stp + %4 + idx26], m13
  mova [stp + %4 + idx27], m6

  ; BLOCK C STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  ;
  ; BLOCK C STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                 m0, [rsp + transposed_in + 16 *  2]
  mova                 m1, m0
  pmulhrsw             m0, [pw__1606x2] ; stp1_8
  pmulhrsw             m1, [pw_16305x2] ; stp2_15

  mova                 m6, [rsp + transposed_in + 16 *  6]
  mova                 m7, m6
  pmulhrsw             m7, [pw_m4756x2] ; stp2_11
  pmulhrsw             m6, [pw_15679x2] ; stp1_12

  mova                 m4, [rsp + transposed_in + 16 * 10]
  mova                 m5, m4
  pmulhrsw             m4, [pw__7723x2] ; stp1_10
  pmulhrsw             m5, [pw_14449x2] ; stp2_13

  mova                 m2, [rsp + transposed_in + 16 * 14]
  mova                 m3, m2
  pmulhrsw             m3, [pw_m10394x2] ; stp1_9
  pmulhrsw             m2, [pw_12665x2] ; stp2_14

  ; BLOCK C STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               0,  3, 9 ;  stp1_8, stp1_9
  SUM_SUB               7,  4, 9 ; stp1_11, stp1_10
  SUM_SUB               6,  5, 9 ; stp1_12, stp1_13
  SUM_SUB               1,  2, 9 ; stp1_15, stp1_14

  ; BLOCK C STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  BUTTERFLY_4X          2,     3,   6270, 15137,  m8,  9,  10 ;  stp1_9, stp1_14
  BUTTERFLY_4Xmm        5,     4,   6270, 15137,  m8,  9,  10 ; stp1_13, stp1_10

  ; BLOCK C STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               0,  7, 9 ;  stp1_8, stp1_11
  SUM_SUB               2,  4, 9 ;  stp1_9, stp1_10
  SUM_SUB               1,  6, 9 ;  stp1_15, stp1_12
  SUM_SUB               3,  5, 9 ;  stp1_14, stp1_13

  ; BLOCK C STAGE 6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
%if 0 ; overflow occurs in SUM_SUB when using test streams
  mova                m10, [pw_11585x2]
  SUM_SUB               5,    4,  9
  pmulhrsw             m5, m10  ; stp1_13
  pmulhrsw             m4, m10  ; stp1_10
  SUM_SUB               6,    7,  9
  pmulhrsw             m6, m10  ; stp1_12
  pmulhrsw             m7, m10  ; stp1_11
%else
  BUTTERFLY_4X       5,     4,  11585,  11585,  m8,  9,  10 ; stp1_10, stp1_13
  SWAP  5, 4
  BUTTERFLY_4X       6,     7,  11585,  11585,  m8,  9,  10 ; stp1_11, stp1_12
  SWAP  6, 7
%endif
  ; BLOCK C STAGE 7 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova [stp + %2 +  idx8], m0
  mova [stp + %2 +  idx9], m2
  mova [stp + %2 + idx10], m4
  mova [stp + %2 + idx11], m7
  mova [stp + %2 + idx12], m6
  mova [stp + %2 + idx13], m5
  mova [stp + %2 + idx14], m3
  mova [stp + %2 + idx15], m1

  ; BLOCK D STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  ;
  ; BLOCK D STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  ;
  ; BLOCK D STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                m11, [rsp + transposed_in + 16 *  4]
  mova                m12, m11
  pmulhrsw            m11, [pw__3196x2] ; stp1_4
  pmulhrsw            m12, [pw_16069x2] ; stp1_7

  mova                m13, [rsp + transposed_in + 16 * 12]
  mova                m14, m13
  pmulhrsw            m13, [pw_13623x2] ; stp1_6
  pmulhrsw            m14, [pw_m9102x2] ; stp1_5

  ; BLOCK D STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                 m0, [rsp + transposed_in + 16 *  0]
  mova                 m2, [rsp + transposed_in + 16 *  8]
  pmulhrsw             m0, [pw_11585x2]  ; stp1_1
  mova                 m3, m2
  pmulhrsw             m2, [pw__6270x2]  ; stp1_2
  pmulhrsw             m3, [pw_15137x2]  ; stp1_3

  SUM_SUB              11, 14, 9 ;  stp1_4, stp1_5
  SUM_SUB              12, 13, 9 ;  stp1_7, stp1_6

  ; BLOCK D STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
%if 0 ; overflow occurs in SUM_SUB when using test streams
  mova                m10, [pw_11585x2]
  SUM_SUB              13,   14,  9
  pmulhrsw            m13, m10  ; stp1_6
  pmulhrsw            m14, m10  ; stp1_5
%else
  BUTTERFLY_4X         13,    14,  11585, 11585,  m8,  9,  10 ; stp1_5, stp1_6
  SWAP 13, 14
%endif
  mova                 m1, m0    ; stp1_0 = stp1_1
  SUM_SUB               0,  3, 9 ;  stp1_0, stp1_3
  SUM_SUB               1,  2, 9 ;  stp1_1, stp1_2

  ; BLOCK D STAGE 6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               0, 12, 9 ;  stp1_0, stp1_7
  SUM_SUB               1, 13, 9 ;  stp1_1, stp1_6
  SUM_SUB               2, 14, 9 ;  stp1_2, stp1_5
  SUM_SUB               3, 11, 9 ;  stp1_3, stp1_4

  ; BLOCK D STAGE 7 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                 m4, [stp + %2 + idx12]
  mova                 m5, [stp + %2 + idx13]
  mova                 m6, [stp + %2 + idx14]
  mova                 m7, [stp + %2 + idx15]
  SUM_SUB               0,  7, 9 ;  stp1_0, stp1_15
  SUM_SUB               1,  6, 9 ;  stp1_1, stp1_14
  SUM_SUB               2,  5, 9 ;  stp1_2, stp1_13
  SUM_SUB               3,  4, 9 ;  stp1_3, stp1_12

  ; 0-3, 28-31 final stage
  mova                m10, [stp + %4 + idx31]
  mova                m15, [stp + %4 + idx30]
  SUM_SUB               0, 10, 9 ;  stp1_0, stp1_31
  SUM_SUB               1, 15, 9 ;  stp1_1, stp1_30
  mova [stp + %1 +  idx0], m0
  mova [stp + %1 +  idx1], m1
  mova [stp + %4 + idx31], m10
  mova [stp + %4 + idx30], m15
  mova                 m0, [stp + %4 + idx29]
  mova                 m1, [stp + %4 + idx28]
  SUM_SUB               2,  0, 9 ;  stp1_2, stp1_29
  SUM_SUB               3,  1, 9 ;  stp1_3, stp1_28
  mova [stp + %1 +  idx2], m2
  mova [stp + %1 +  idx3], m3
  mova [stp + %4 + idx29], m0
  mova [stp + %4 + idx28], m1

  ; 12-15, 16-19 final stage
  mova                 m0, [stp + %3 + idx16]
  mova                 m1, [stp + %3 + idx17]
  mova                 m2, [stp + %3 + idx18]
  mova                 m3, [stp + %3 + idx19]
  SUM_SUB               7,  0, 9 ;  stp1_15, stp1_16
  SUM_SUB               6,  1, 9 ;  stp1_14, stp1_17
  SUM_SUB               5,  2, 9 ;  stp1_13, stp1_18
  SUM_SUB               4,  3, 9 ;  stp1_12, stp1_19
  mova [stp + %2 + idx12], m4
  mova [stp + %2 + idx13], m5
  mova [stp + %2 + idx14], m6
  mova [stp + %2 + idx15], m7
  mova [stp + %3 + idx16], m0
  mova [stp + %3 + idx17], m1
  mova [stp + %3 + idx18], m2
  mova [stp + %3 + idx19], m3

  mova                 m4, [stp + %2 +  idx8]
  mova                 m5, [stp + %2 +  idx9]
  mova                 m6, [stp + %2 + idx10]
  mova                 m7, [stp + %2 + idx11]
  SUM_SUB              11,  7, 9 ;  stp1_4, stp1_11
  SUM_SUB              14,  6, 9 ;  stp1_5, stp1_10
  SUM_SUB              13,  5, 9 ;  stp1_6, stp1_9
  SUM_SUB              12,  4, 9 ;  stp1_7, stp1_8

  ; 4-7, 24-27 final stage
  mova                 m3, [stp + %4 + idx24]
  mova                 m2, [stp + %4 + idx25]
  mova                 m1, [stp + %4 + idx26]
  mova                 m0, [stp + %4 + idx27]
  SUM_SUB              12,  3, 9 ;  stp1_7, stp1_24
  SUM_SUB              13,  2, 9 ;  stp1_6, stp1_25
  SUM_SUB              14,  1, 9 ;  stp1_5, stp1_26
  SUM_SUB              11,  0, 9 ;  stp1_4, stp1_27
  mova [stp + %4 + idx24], m3
  mova [stp + %4 + idx25], m2
  mova [stp + %4 + idx26], m1
  mova [stp + %4 + idx27], m0
  mova [stp + %1 +  idx4], m11
  mova [stp + %1 +  idx5], m14
  mova [stp + %1 +  idx6], m13
  mova [stp + %1 +  idx7], m12

  ; 8-11, 20-23 final stage
  mova                 m0, [stp + %3 + idx20]
  mova                 m1, [stp + %3 + idx21]
  mova                 m2, [stp + %3 + idx22]
  mova                 m3, [stp + %3 + idx23]
  SUM_SUB               7,  0, 9 ;  stp1_11, stp_20
  SUM_SUB               6,  1, 9 ;  stp1_10, stp_21
  SUM_SUB               5,  2, 9 ;   stp1_9, stp_22
  SUM_SUB               4,  3, 9 ;   stp1_8, stp_23
  mova [stp + %2 +  idx8], m4
  mova [stp + %2 +  idx9], m5
  mova [stp + %2 + idx10], m6
  mova [stp + %2 + idx11], m7
  mova [stp + %3 + idx20], m0
  mova [stp + %3 + idx21], m1
  mova [stp + %3 + idx22], m2
  mova [stp + %3 + idx23], m3
%endmacro

INIT_XMM ssse3
cglobal idct32x32_135_add, 3, 11, 16, i32x32_size, input, output, stride
  mova            m8, [pd_8192]
  mov             r6, 2
  lea            stp, [rsp + pass_one_start]

idct32x32_135:
  mov             r3, inputq
  lea             r4, [rsp + transposed_in]
  mov             r7, 2

idct32x32_135_transpose:
%if CONFIG_VP9_HIGHBITDEPTH
  mova            m0, [r3 +       0]
  packssdw        m0, [r3 +      16]
  mova            m1, [r3 + 32 *  4]
  packssdw        m1, [r3 + 32 *  4 + 16]
  mova            m2, [r3 + 32 *  8]
  packssdw        m2, [r3 + 32 *  8 + 16]
  mova            m3, [r3 + 32 * 12]
  packssdw        m3, [r3 + 32 * 12 + 16]
  mova            m4, [r3 + 32 * 16]
  packssdw        m4, [r3 + 32 * 16 + 16]
  mova            m5, [r3 + 32 * 20]
  packssdw        m5, [r3 + 32 * 20 + 16]
  mova            m6, [r3 + 32 * 24]
  packssdw        m6, [r3 + 32 * 24 + 16]
  mova            m7, [r3 + 32 * 28]
  packssdw        m7, [r3 + 32 * 28 + 16]
%else
  mova            m0, [r3 +       0]
  mova            m1, [r3 + 16 *  4]
  mova            m2, [r3 + 16 *  8]
  mova            m3, [r3 + 16 * 12]
  mova            m4, [r3 + 16 * 16]
  mova            m5, [r3 + 16 * 20]
  mova            m6, [r3 + 16 * 24]
  mova            m7, [r3 + 16 * 28]
%endif
  TRANSPOSE8X8  0, 1, 2, 3, 4, 5, 6, 7, 9

  mova [r4 +      0], m0
  mova [r4 + 16 * 1], m1
  mova [r4 + 16 * 2], m2
  mova [r4 + 16 * 3], m3
  mova [r4 + 16 * 4], m4
  mova [r4 + 16 * 5], m5
  mova [r4 + 16 * 6], m6
  mova [r4 + 16 * 7], m7

%if CONFIG_VP9_HIGHBITDEPTH
  add             r3, 32
%else
  add             r3, 16
%endif
  add             r4, 16 * 8
  dec             r7
  jne idct32x32_135_transpose

  IDCT32X32_135 16*0, 16*32, 16*64, 16*96
  lea            stp, [stp + 16 * 8]
%if CONFIG_VP9_HIGHBITDEPTH
  lea         inputq, [inputq + 32 * 32]
%else
  lea         inputq, [inputq + 16 * 32]
%endif
  dec             r6
  jnz idct32x32_135

  mov             r6, 4
  lea            stp, [rsp + pass_one_start]
  lea             r9, [rsp + pass_one_start]

idct32x32_135_2:
  lea             r4, [rsp + transposed_in]
  mov             r3, r9
  mov             r7, 2

idct32x32_135_transpose_2:
  mova            m0, [r3 +      0]
  mova            m1, [r3 + 16 * 1]
  mova            m2, [r3 + 16 * 2]
  mova            m3, [r3 + 16 * 3]
  mova            m4, [r3 + 16 * 4]
  mova            m5, [r3 + 16 * 5]
  mova            m6, [r3 + 16 * 6]
  mova            m7, [r3 + 16 * 7]

  TRANSPOSE8X8  0, 1, 2, 3, 4, 5, 6, 7, 9

  mova [r4 +      0], m0
  mova [r4 + 16 * 1], m1
  mova [r4 + 16 * 2], m2
  mova [r4 + 16 * 3], m3
  mova [r4 + 16 * 4], m4
  mova [r4 + 16 * 5], m5
  mova [r4 + 16 * 6], m6
  mova [r4 + 16 * 7], m7

  add             r3, 16 * 8
  add             r4, 16 * 8
  dec             r7
  jne idct32x32_135_transpose_2

  IDCT32X32_135 16*0, 16*8, 16*16, 16*24

  lea            stp, [stp + 16 * 32]
  add             r9, 16 * 32
  dec             r6
  jnz idct32x32_135_2

  RECON_AND_STORE pass_two_start

  RET

%macro IDCT32X32_1024 4
  ; BLOCK A STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                 m1, [rsp + transposed_in + 16 *  1]
  mova                m11, [rsp + transposed_in + 16 * 31]
  BUTTERFLY_4X          1,    11,    804, 16364,  m8,  9,  10 ; stp1_16, stp1_31

  mova                 m0, [rsp + transposed_in + 16 * 15]
  mova                 m2, [rsp + transposed_in + 16 * 17]
  BUTTERFLY_4X          2,     0,  12140, 11003,  m8,  9,  10 ; stp1_17, stp1_30

  mova                 m7, [rsp + transposed_in + 16 *  7]
  mova                m12, [rsp + transposed_in + 16 * 25]
  BUTTERFLY_4X         12,     7,  15426,  5520,  m8,  9,  10 ; stp1_19, stp1_28

  mova                 m3, [rsp + transposed_in + 16 *  9]
  mova                 m4, [rsp + transposed_in + 16 * 23]
  BUTTERFLY_4X          3,     4,   7005, 14811,  m8,  9,  10 ; stp1_18, stp1_29

  ; BLOCK A STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               1,  2, 9 ; stp2_16, stp2_17
  SUM_SUB              12,  3, 9 ; stp2_19, stp2_18
  SUM_SUB               7,  4, 9 ; stp2_28, stp2_29
  SUM_SUB              11,  0, 9 ; stp2_31, stp2_30

  ; BLOCK A STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  BUTTERFLY_4X          0,     2,   3196, 16069,  m8,  9,  10 ; stp1_17, stp1_30
  BUTTERFLY_4Xmm        4,     3,   3196, 16069,  m8,  9,  10 ; stp1_29, stp1_18

  ; BLOCK A STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               1, 12, 9 ; stp2_16, stp2_19
  SUM_SUB               0,  3, 9 ; stp2_17, stp2_18
  SUM_SUB              11,  7, 9 ; stp2_31, stp2_28
  SUM_SUB               2,  4, 9 ; stp2_30, stp2_29

  ; BLOCK A STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  BUTTERFLY_4X          4,     3,   6270, 15137,  m8,  9,  10 ; stp1_18, stp1_29
  BUTTERFLY_4X          7,    12,   6270, 15137,  m8,  9,  10 ; stp1_19, stp1_28

  mova [stp + %3 + idx16], m1
  mova [stp + %3 + idx17], m0
  mova [stp + %3 + idx18], m4
  mova [stp + %3 + idx19], m7
  mova [stp + %4 + idx28], m12
  mova [stp + %4 + idx29], m3
  mova [stp + %4 + idx30], m2
  mova [stp + %4 + idx31], m11

  ; BLOCK B STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                 m5, [rsp + transposed_in + 16 *  5]
  mova                 m6, [rsp + transposed_in + 16 * 27]
  BUTTERFLY_4X          5,     6,   3981, 15893,  m8,  9,  10 ; stp1_20, stp1_27

  mova                m13, [rsp + transposed_in + 16 * 21]
  mova                m14, [rsp + transposed_in + 16 * 11]
  BUTTERFLY_4X         13,    14,  14053,  8423,  m8,  9,  10 ; stp1_21, stp1_26

  mova                 m0, [rsp + transposed_in + 16 * 13]
  mova                 m1, [rsp + transposed_in + 16 * 19]
  BUTTERFLY_4X          0,     1,   9760, 13160,  m8,  9,  10 ; stp1_22, stp1_25

  mova                 m2, [rsp + transposed_in + 16 *  3]
  mova                 m3, [rsp + transposed_in + 16 * 29]
  BUTTERFLY_4X          3,     2,  16207,  2404,  m8,  9,  10 ; stp1_23, stp1_24

  ; BLOCK B STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               5, 13, 9 ; stp2_20, stp2_21
  SUM_SUB               3,  0, 9 ; stp2_23, stp2_22
  SUM_SUB               2,  1, 9 ; stp2_24, stp2_25
  SUM_SUB               6, 14, 9 ; stp2_27, stp2_26

  ; BLOCK B STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  BUTTERFLY_4X         14,    13,  13623,  9102,  m8,  9,  10 ; stp1_21, stp1_26
  BUTTERFLY_4Xmm        1,     0,  13623,  9102,  m8,  9,  10 ; stp1_25, stp1_22

  ; BLOCK B STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               3,  5, 9 ; stp2_23, stp2_20
  SUM_SUB               0, 14, 9 ; stp2_22, stp2_21
  SUM_SUB               2,  6, 9 ; stp2_24, stp2_27
  SUM_SUB               1, 13, 9 ; stp2_25, stp2_26

  ; BLOCK B STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  BUTTERFLY_4Xmm        6,     5,   6270, 15137,  m8,  9,  10 ; stp1_27, stp1_20
  BUTTERFLY_4Xmm       13,    14,   6270, 15137,  m8,  9,  10 ; stp1_26, stp1_21

  ; BLOCK B STAGE 6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                 m4, [stp + %3 + idx16]
  mova                 m7, [stp + %3 + idx17]
  mova                m11, [stp + %3 + idx18]
  mova                m12, [stp + %3 + idx19]
  SUM_SUB               4,  3, 9 ; stp2_16, stp2_23
  SUM_SUB               7,  0, 9 ; stp2_17, stp2_22
  SUM_SUB              11, 14, 9 ; stp2_18, stp2_21
  SUM_SUB              12,  5, 9 ; stp2_19, stp2_20
  mova [stp + %3 + idx16], m4
  mova [stp + %3 + idx17], m7
  mova [stp + %3 + idx18], m11
  mova [stp + %3 + idx19], m12

  mova                 m4, [stp + %4 + idx28]
  mova                 m7, [stp + %4 + idx29]
  mova                m11, [stp + %4 + idx30]
  mova                m12, [stp + %4 + idx31]
  SUM_SUB               4,  6, 9 ; stp2_28, stp2_27
  SUM_SUB               7, 13, 9 ; stp2_29, stp2_26
  SUM_SUB              11,  1, 9 ; stp2_30, stp2_25
  SUM_SUB              12,  2, 9 ; stp2_31, stp2_24
  mova [stp + %4 + idx28], m4
  mova [stp + %4 + idx29], m7
  mova [stp + %4 + idx30], m11
  mova [stp + %4 + idx31], m12

  ; BLOCK B STAGE 7 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
%if 0 ; overflow occurs in SUM_SUB when using test streams
  mova                m10, [pw_11585x2]
  SUM_SUB               6,  5,  9
  pmulhrsw             m6, m10  ; stp1_27
  pmulhrsw             m5, m10  ; stp1_20
  SUM_SUB              13, 14,  9
  pmulhrsw            m13, m10  ; stp1_26
  pmulhrsw            m14, m10  ; stp1_21
  SUM_SUB               1,  0,  9
  pmulhrsw             m1, m10  ; stp1_25
  pmulhrsw             m0, m10  ; stp1_22
  SUM_SUB               2,  3,  9
  pmulhrsw             m2, m10  ; stp1_25
  pmulhrsw             m3, m10  ; stp1_22
%else
  BUTTERFLY_4X          6,     5,  11585, 11585,  m8,  9,  10 ; stp1_20, stp1_27
  SWAP  6, 5
  BUTTERFLY_4X         13,    14,  11585, 11585,  m8,  9,  10 ; stp1_21, stp1_26
  SWAP 13, 14
  BUTTERFLY_4X          1,     0,  11585, 11585,  m8,  9,  10 ; stp1_22, stp1_25
  SWAP  1, 0
  BUTTERFLY_4X          2,     3,  11585, 11585,  m8,  9,  10 ; stp1_23, stp1_24
  SWAP  2, 3
%endif
  mova [stp + %3 + idx20], m5
  mova [stp + %3 + idx21], m14
  mova [stp + %3 + idx22], m0
  mova [stp + %3 + idx23], m3
  mova [stp + %4 + idx24], m2
  mova [stp + %4 + idx25], m1
  mova [stp + %4 + idx26], m13
  mova [stp + %4 + idx27], m6

  ; BLOCK C STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  ;
  ; BLOCK C STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                 m0, [rsp + transposed_in + 16 *  2]
  mova                 m1, [rsp + transposed_in + 16 * 30]
  BUTTERFLY_4X          0,     1,   1606, 16305,  m8,  9,  10 ; stp1_8, stp1_15

  mova                 m2, [rsp + transposed_in + 16 * 14]
  mova                 m3, [rsp + transposed_in + 16 * 18]
  BUTTERFLY_4X          3,     2,  12665, 10394,  m8,  9,  10 ; stp1_9, stp1_14

  mova                 m4, [rsp + transposed_in + 16 * 10]
  mova                 m5, [rsp + transposed_in + 16 * 22]
  BUTTERFLY_4X          4,     5,   7723, 14449,  m8,  9,  10 ; stp1_10, stp1_13

  mova                 m6, [rsp + transposed_in + 16 *  6]
  mova                 m7, [rsp + transposed_in + 16 * 26]
  BUTTERFLY_4X          7,     6,  15679,  4756,  m8,  9,  10 ; stp1_11, stp1_12

  ; BLOCK C STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               0,  3, 9 ;  stp1_8, stp1_9
  SUM_SUB               7,  4, 9 ; stp1_11, stp1_10
  SUM_SUB               6,  5, 9 ; stp1_12, stp1_13
  SUM_SUB               1,  2, 9 ; stp1_15, stp1_14

  ; BLOCK C STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  BUTTERFLY_4X          2,     3,   6270, 15137,  m8,  9,  10 ;  stp1_9, stp1_14
  BUTTERFLY_4Xmm        5,     4,   6270, 15137,  m8,  9,  10 ; stp1_13, stp1_10

  ; BLOCK C STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               0,  7, 9 ;  stp1_8, stp1_11
  SUM_SUB               2,  4, 9 ;  stp1_9, stp1_10
  SUM_SUB               1,  6, 9 ;  stp1_15, stp1_12
  SUM_SUB               3,  5, 9 ;  stp1_14, stp1_13

  ; BLOCK C STAGE 6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
%if 0 ; overflow occurs in SUM_SUB when using test streams
  mova                m10, [pw_11585x2]
  SUM_SUB               5,    4,  9
  pmulhrsw             m5, m10  ; stp1_13
  pmulhrsw             m4, m10  ; stp1_10
  SUM_SUB               6,    7,  9
  pmulhrsw             m6, m10  ; stp1_12
  pmulhrsw             m7, m10  ; stp1_11
%else
  BUTTERFLY_4X       5,     4,  11585,  11585,  m8,  9,  10 ; stp1_10, stp1_13
  SWAP  5, 4
  BUTTERFLY_4X       6,     7,  11585,  11585,  m8,  9,  10 ; stp1_11, stp1_12
  SWAP  6, 7
%endif
  ; BLOCK C STAGE 7 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova [stp + %2 +  idx8], m0
  mova [stp + %2 +  idx9], m2
  mova [stp + %2 + idx10], m4
  mova [stp + %2 + idx11], m7
  mova [stp + %2 + idx12], m6
  mova [stp + %2 + idx13], m5
  mova [stp + %2 + idx14], m3
  mova [stp + %2 + idx15], m1

  ; BLOCK D STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  ;
  ; BLOCK D STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  ;
  ; BLOCK D STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                m11, [rsp + transposed_in + 16 *  4]
  mova                m12, [rsp + transposed_in + 16 * 28]
  BUTTERFLY_4X         11,    12,   3196, 16069,  m8,  9,  10 ; stp1_4, stp1_7

  mova                m13, [rsp + transposed_in + 16 * 12]
  mova                m14, [rsp + transposed_in + 16 * 20]
  BUTTERFLY_4X         14,    13,  13623,  9102,  m8,  9,  10 ; stp1_5, stp1_6

  ; BLOCK D STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                 m0, [rsp + transposed_in + 16 *  0]
  mova                 m1, [rsp + transposed_in + 16 * 16]

%if 0 ; overflow occurs in SUM_SUB when using test streams
  mova                m10, [pw_11585x2]
  SUM_SUB               0,    1,  9
  pmulhrsw             m0, m10  ; stp1_1
  pmulhrsw             m1, m10  ; stp1_0
%else
  BUTTERFLY_4X          0,     1,  11585, 11585,  m8,  9,  10 ; stp1_1, stp1_0
  SWAP  0, 1
%endif
  mova                 m2, [rsp + transposed_in + 16 *  8]
  mova                 m3, [rsp + transposed_in + 16 * 24]
  BUTTERFLY_4X          2,     3,   6270, 15137,  m8,  9,  10 ;  stp1_2, stp1_3

  mova                m10, [pw_11585x2]
  SUM_SUB              11, 14, 9 ;  stp1_4, stp1_5
  SUM_SUB              12, 13, 9 ;  stp1_7, stp1_6

  ; BLOCK D STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
%if 0 ; overflow occurs in SUM_SUB when using test streams
  SUM_SUB              13,   14,  9
  pmulhrsw            m13, m10  ; stp1_6
  pmulhrsw            m14, m10  ; stp1_5
%else
  BUTTERFLY_4X         13,    14,  11585, 11585,  m8,  9,  10 ; stp1_5, stp1_6
  SWAP 13, 14
%endif
  SUM_SUB               0,  3, 9 ;  stp1_0, stp1_3
  SUM_SUB               1,  2, 9 ;  stp1_1, stp1_2

  ; BLOCK D STAGE 6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  SUM_SUB               0, 12, 9 ;  stp1_0, stp1_7
  SUM_SUB               1, 13, 9 ;  stp1_1, stp1_6
  SUM_SUB               2, 14, 9 ;  stp1_2, stp1_5
  SUM_SUB               3, 11, 9 ;  stp1_3, stp1_4

  ; BLOCK D STAGE 7 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  mova                 m4, [stp + %2 + idx12]
  mova                 m5, [stp + %2 + idx13]
  mova                 m6, [stp + %2 + idx14]
  mova                 m7, [stp + %2 + idx15]
  SUM_SUB               0,  7, 9 ;  stp1_0, stp1_15
  SUM_SUB               1,  6, 9 ;  stp1_1, stp1_14
  SUM_SUB               2,  5, 9 ;  stp1_2, stp1_13
  SUM_SUB               3,  4, 9 ;  stp1_3, stp1_12

  ; 0-3, 28-31 final stage
  mova                m10, [stp + %4 + idx31]
  mova                m15, [stp + %4 + idx30]
  SUM_SUB               0, 10, 9 ;  stp1_0, stp1_31
  SUM_SUB               1, 15, 9 ;  stp1_1, stp1_30
  mova [stp + %1 +  idx0], m0
  mova [stp + %1 +  idx1], m1
  mova [stp + %4 + idx31], m10
  mova [stp + %4 + idx30], m15
  mova                 m0, [stp + %4 + idx29]
  mova                 m1, [stp + %4 + idx28]
  SUM_SUB               2,  0, 9 ;  stp1_2, stp1_29
  SUM_SUB               3,  1, 9 ;  stp1_3, stp1_28
  mova [stp + %1 +  idx2], m2
  mova [stp + %1 +  idx3], m3
  mova [stp + %4 + idx29], m0
  mova [stp + %4 + idx28], m1

  ; 12-15, 16-19 final stage
  mova                 m0, [stp + %3 + idx16]
  mova                 m1, [stp + %3 + idx17]
  mova                 m2, [stp + %3 + idx18]
  mova                 m3, [stp + %3 + idx19]
  SUM_SUB               7,  0, 9 ;  stp1_15, stp1_16
  SUM_SUB               6,  1, 9 ;  stp1_14, stp1_17
  SUM_SUB               5,  2, 9 ;  stp1_13, stp1_18
  SUM_SUB               4,  3, 9 ;  stp1_12, stp1_19
  mova [stp + %2 + idx12], m4
  mova [stp + %2 + idx13], m5
  mova [stp + %2 + idx14], m6
  mova [stp + %2 + idx15], m7
  mova [stp + %3 + idx16], m0
  mova [stp + %3 + idx17], m1
  mova [stp + %3 + idx18], m2
  mova [stp + %3 + idx19], m3

  mova                 m4, [stp + %2 +  idx8]
  mova                 m5, [stp + %2 +  idx9]
  mova                 m6, [stp + %2 + idx10]
  mova                 m7, [stp + %2 + idx11]
  SUM_SUB              11,  7, 9 ;  stp1_4, stp1_11
  SUM_SUB              14,  6, 9 ;  stp1_5, stp1_10
  SUM_SUB              13,  5, 9 ;  stp1_6, stp1_9
  SUM_SUB              12,  4, 9 ;  stp1_7, stp1_8

  ; 4-7, 24-27 final stage
  mova                 m3, [stp + %4 + idx24]
  mova                 m2, [stp + %4 + idx25]
  mova                 m1, [stp + %4 + idx26]
  mova                 m0, [stp + %4 + idx27]
  SUM_SUB              12,  3, 9 ;  stp1_7, stp1_24
  SUM_SUB              13,  2, 9 ;  stp1_6, stp1_25
  SUM_SUB              14,  1, 9 ;  stp1_5, stp1_26
  SUM_SUB              11,  0, 9 ;  stp1_4, stp1_27
  mova [stp + %4 + idx24], m3
  mova [stp + %4 + idx25], m2
  mova [stp + %4 + idx26], m1
  mova [stp + %4 + idx27], m0
  mova [stp + %1 +  idx4], m11
  mova [stp + %1 +  idx5], m14
  mova [stp + %1 +  idx6], m13
  mova [stp + %1 +  idx7], m12

  ; 8-11, 20-23 final stage
  mova                 m0, [stp + %3 + idx20]
  mova                 m1, [stp + %3 + idx21]
  mova                 m2, [stp + %3 + idx22]
  mova                 m3, [stp + %3 + idx23]
  SUM_SUB               7,  0, 9 ;  stp1_11, stp_20
  SUM_SUB               6,  1, 9 ;  stp1_10, stp_21
  SUM_SUB               5,  2, 9 ;   stp1_9, stp_22
  SUM_SUB               4,  3, 9 ;   stp1_8, stp_23
  mova [stp + %2 +  idx8], m4
  mova [stp + %2 +  idx9], m5
  mova [stp + %2 + idx10], m6
  mova [stp + %2 + idx11], m7
  mova [stp + %3 + idx20], m0
  mova [stp + %3 + idx21], m1
  mova [stp + %3 + idx22], m2
  mova [stp + %3 + idx23], m3
%endmacro

INIT_XMM ssse3
cglobal idct32x32_1024_add, 3, 11, 16, i32x32_size, input, output, stride
  mova            m8, [pd_8192]
  mov             r6, 4
  lea            stp, [rsp + pass_one_start]

idct32x32_1024:
  mov             r3, inputq
  lea             r4, [rsp + transposed_in]
  mov             r7, 4

idct32x32_1024_transpose:
%if CONFIG_VP9_HIGHBITDEPTH
  mova            m0, [r3 +       0]
  packssdw        m0, [r3 +      16]
  mova            m1, [r3 + 32 *  4]
  packssdw        m1, [r3 + 32 *  4 + 16]
  mova            m2, [r3 + 32 *  8]
  packssdw        m2, [r3 + 32 *  8 + 16]
  mova            m3, [r3 + 32 * 12]
  packssdw        m3, [r3 + 32 * 12 + 16]
  mova            m4, [r3 + 32 * 16]
  packssdw        m4, [r3 + 32 * 16 + 16]
  mova            m5, [r3 + 32 * 20]
  packssdw        m5, [r3 + 32 * 20 + 16]
  mova            m6, [r3 + 32 * 24]
  packssdw        m6, [r3 + 32 * 24 + 16]
  mova            m7, [r3 + 32 * 28]
  packssdw        m7, [r3 + 32 * 28 + 16]
%else
  mova            m0, [r3 +       0]
  mova            m1, [r3 + 16 *  4]
  mova            m2, [r3 + 16 *  8]
  mova            m3, [r3 + 16 * 12]
  mova            m4, [r3 + 16 * 16]
  mova            m5, [r3 + 16 * 20]
  mova            m6, [r3 + 16 * 24]
  mova            m7, [r3 + 16 * 28]
%endif

  TRANSPOSE8X8  0, 1, 2, 3, 4, 5, 6, 7, 9

  mova [r4 +      0], m0
  mova [r4 + 16 * 1], m1
  mova [r4 + 16 * 2], m2
  mova [r4 + 16 * 3], m3
  mova [r4 + 16 * 4], m4
  mova [r4 + 16 * 5], m5
  mova [r4 + 16 * 6], m6
  mova [r4 + 16 * 7], m7
%if CONFIG_VP9_HIGHBITDEPTH
  add             r3, 32
%else
  add             r3, 16
%endif
  add             r4, 16 * 8
  dec             r7
  jne idct32x32_1024_transpose

  IDCT32X32_1024 16*0, 16*32, 16*64, 16*96

  lea            stp, [stp + 16 * 8]
%if CONFIG_VP9_HIGHBITDEPTH
  lea         inputq, [inputq + 32 * 32]
%else
  lea         inputq, [inputq + 16 * 32]
%endif
  dec             r6
  jnz idct32x32_1024

  mov             r6, 4
  lea            stp, [rsp + pass_one_start]
  lea             r9, [rsp + pass_one_start]

idct32x32_1024_2:
  lea             r4, [rsp + transposed_in]
  mov             r3, r9
  mov             r7, 4

idct32x32_1024_transpose_2:
  mova            m0, [r3 +      0]
  mova            m1, [r3 + 16 * 1]
  mova            m2, [r3 + 16 * 2]
  mova            m3, [r3 + 16 * 3]
  mova            m4, [r3 + 16 * 4]
  mova            m5, [r3 + 16 * 5]
  mova            m6, [r3 + 16 * 6]
  mova            m7, [r3 + 16 * 7]

  TRANSPOSE8X8  0, 1, 2, 3, 4, 5, 6, 7, 9

  mova [r4 +      0], m0
  mova [r4 + 16 * 1], m1
  mova [r4 + 16 * 2], m2
  mova [r4 + 16 * 3], m3
  mova [r4 + 16 * 4], m4
  mova [r4 + 16 * 5], m5
  mova [r4 + 16 * 6], m6
  mova [r4 + 16 * 7], m7

  add             r3, 16 * 8
  add             r4, 16 * 8
  dec             r7
  jne idct32x32_1024_transpose_2

  IDCT32X32_1024 16*0, 16*8, 16*16, 16*24

  lea            stp, [stp + 16 * 32]
  add             r9, 16 * 32
  dec             r6
  jnz idct32x32_1024_2

  RECON_AND_STORE pass_two_start

  RET
%endif