xref: /aosp_15_r20/external/libdav1d/src/x86/cdef_avx2.asm (revision c09093415860a1c2373dacd84c4fde00c507cdfd)

; Copyright © 2018, VideoLAN and dav1d authors
; Copyright © 2018, Two Orioles, LLC
; All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
;
; 1. Redistributions of source code must retain the above copyright notice, this
;    list of conditions and the following disclaimer.
;
; 2. Redistributions in binary form must reproduce the above copyright notice,
;    this list of conditions and the following disclaimer in the documentation
;    and/or other materials provided with the distribution.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

%include "config.asm"
%include "ext/x86/x86inc.asm"

%if ARCH_X86_64

%macro JMP_TABLE 2-*
 %xdefine %1_jmptable %%table
 %xdefine %%base mangle(private_prefix %+ _%1_avx2)
 %%table:
 %rep %0 - 1
    dd %%base %+ .%2 - %%table
  %rotate 1
 %endrep
%endmacro

%macro CDEF_FILTER_JMP_TABLE 1
JMP_TABLE cdef_filter_%1_8bpc, \
    d6k0, d6k1, d7k0, d7k1, \
    d0k0, d0k1, d1k0, d1k1, d2k0, d2k1, d3k0, d3k1, \
    d4k0, d4k1, d5k0, d5k1, d6k0, d6k1, d7k0, d7k1, \
    d0k0, d0k1, d1k0, d1k1
%endmacro

SECTION_RODATA 32

pd_47130256:   dd  4,  7,  1,  3,  0,  2,  5,  6
blend_4x4:     dd 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00
               dd 0x80, 0x00, 0x00
blend_4x8_0:   dd 0x00, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
blend_4x8_1:   dd 0x00, 0x00, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
               dd 0x00, 0x00
blend_4x8_2:   dd 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080
               dd 0x0000
blend_4x8_3:   dd 0x0000, 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080
               dd 0x0000, 0x0000
blend_8x8_0:   dq 0x00, 0x00, 0x80, 0x80, 0x80, 0x80
blend_8x8_1:   dq 0x0000, 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x0000, 0x0000
div_table:     dd 840, 420, 280, 210, 168, 140, 120, 105, 420, 210, 140, 105
shufw_6543210x:db 12, 13, 10, 11,  8,  9,  6,  7,  4,  5,  2,  3,  0,  1, 14, 15
shufb_lohi:    db  0,  8,  1,  9,  2, 10,  3, 11,  4, 12,  5, 13,  6, 14,  7, 15
pw_128:        times 2 dw 128
pw_2048:       times 2 dw 2048
tap_table:     ; masks for 8 bit shifts
               db 0xFF, 0x7F, 0x3F, 0x1F, 0x0F, 0x07, 0x03, 0x01
               ; weights
               db  4,  2,  3,  3,  2,  1
               db -1 * 16 + 1, -2 * 16 + 2
               db  0 * 16 + 1, -1 * 16 + 2
               db  0 * 16 + 1,  0 * 16 + 2
               db  0 * 16 + 1,  1 * 16 + 2
               db  1 * 16 + 1,  2 * 16 + 2
               db  1 * 16 + 0,  2 * 16 + 1
               db  1 * 16 + 0,  2 * 16 + 0
               db  1 * 16 + 0,  2 * 16 - 1
               ; the last 6 are repeats of the first 6 so we don't need to & 7
               db -1 * 16 + 1, -2 * 16 + 2
               db  0 * 16 + 1, -1 * 16 + 2
               db  0 * 16 + 1,  0 * 16 + 2
               db  0 * 16 + 1,  1 * 16 + 2
               db  1 * 16 + 1,  2 * 16 + 2
               db  1 * 16 + 0,  2 * 16 + 1

CDEF_FILTER_JMP_TABLE 4x4
CDEF_FILTER_JMP_TABLE 4x8
CDEF_FILTER_JMP_TABLE 8x8

SECTION .text

%macro PREP_REGS 2 ; w, h
    ; off1/2/3[k] [6 total] from [tapq+12+(dir+0/2/6)*2+k]
    mov           dird, r7m
    lea         tableq, [cdef_filter_%1x%2_8bpc_jmptable]
    lea           dirq, [tableq+dirq*2*4]
%if %1 == 4
 %if %2 == 4
  DEFINE_ARGS dst, stride, left, top, bot, pri, sec, \
              table, dir, dirjmp, stride3, k
 %else
  DEFINE_ARGS dst, stride, left, top, bot, pri, sec, \
              table, dir, dirjmp, dst4, stride3, k
    lea          dst4q, [dstq+strideq*4]
 %endif
%else
  DEFINE_ARGS dst, stride, h, top1, bot, pri, sec, \
              table, dir, dirjmp, top2, stride3, k
    mov             hq, -8
    lea          top1q, [top1q+strideq*0]
    lea          top2q, [top1q+strideq*1]
%endif
%if %1 == 4
    lea       stride3q, [strideq*3]
%endif
%endmacro

%macro LOAD_BLOCK 2-3 0 ; w, h, init_min_max
    mov             kd, 1
    pxor           m15, m15                     ; sum
%if %2 == 8
    pxor           m12, m12
 %if %1 == 4
    movd           xm4, [dstq +strideq*0]
    movd           xm6, [dstq +strideq*1]
    movd           xm5, [dstq +strideq*2]
    movd           xm7, [dstq +stride3q ]
    vinserti128     m4, [dst4q+strideq*0], 1
    vinserti128     m6, [dst4q+strideq*1], 1
    vinserti128     m5, [dst4q+strideq*2], 1
    vinserti128     m7, [dst4q+stride3q ], 1
    punpckldq       m4, m6
    punpckldq       m5, m7
 %else
    movq           xm4, [dstq+strideq*0]
    movq           xm5, [dstq+strideq*1]
    vinserti128     m4, [dstq+strideq*2], 1
    vinserti128     m5, [dstq+stride3q ], 1
 %endif
    punpcklqdq      m4, m5
%else
    movd           xm4, [dstq+strideq*0]
    movd           xm5, [dstq+strideq*1]
    vinserti128     m4, [dstq+strideq*2], 1
    vinserti128     m5, [dstq+stride3q ], 1
    punpckldq       m4, m5
%endif
%if %3 == 1
    mova            m7, m4                      ; min
    mova            m8, m4                      ; max
%endif
%endmacro

%macro ACCUMULATE_TAP_BYTE 7-8 0 ; tap_offset, shift, mask, strength
                                 ; mul_tap, w, h, clip
    ; load p0/p1
    movsxd     dirjmpq, [dirq+kq*4+%1*2*4]
    add        dirjmpq, tableq
    call       dirjmpq

%if %8 == 1
    pmaxub          m7, m5
    pminub          m8, m5
    pmaxub          m7, m6
    pminub          m8, m6
%endif

    ; accumulate sum[m15] over p0/p1
%if %7 == 4
    punpcklbw       m5, m6
    punpcklbw       m6, m4, m4
    psubusb         m9, m5, m6
    psubusb         m5, m6, m5
    por             m9, m5     ; abs_diff_p01(p01 - px)
    pcmpeqb         m5, m9
    por             m5, %5
    psignb          m6, %5, m5
    psrlw           m5, m9, %2 ; emulate 8-bit shift
    pand            m5, %3
    psubusb         m5, %4, m5
    pminub          m5, m9
    pmaddubsw       m5, m6
    paddw          m15, m5
%else
    psubusb         m9, m5, m4
    psubusb         m5, m4, m5
    psubusb        m11, m6, m4
    psubusb         m6, m4, m6
    por             m9, m5      ; abs_diff_p0(p0 - px)
    por            m11, m6      ; abs_diff_p1(p1 - px)
    pcmpeqb         m5, m9
    pcmpeqb         m6, m11
    punpckhbw      m10, m9, m11
    punpcklbw       m9, m11
    por             m5, %5
    por            m11, m6, %5
    punpckhbw       m6, m5, m11
    punpcklbw       m5, m11
    psignb         m11, %5, m6
    psrlw           m6, m10, %2 ; emulate 8-bit shift
    pand            m6, %3
    psubusb         m6, %4, m6
    pminub          m6, m10
    pmaddubsw       m6, m11
    paddw          m12, m6
    psignb         m11, %5, m5
    psrlw           m5, m9, %2  ; emulate 8-bit shift
    pand            m5, %3
    psubusb         m5, %4, m5
    pminub          m5, m9
    pmaddubsw       m5, m11
    paddw          m15, m5
%endif
%endmacro

%macro ADJUST_PIXEL 4-5 0 ; w, h, zero, pw_2048, clip
%if %2 == 4
 %if %5 == 1
    punpcklbw       m4, %3
 %endif
    pcmpgtw         %3, m15
    paddw          m15, %3
    pmulhrsw       m15, %4
 %if %5 == 0
    packsswb       m15, m15
    paddb           m4, m15
 %else
    paddw           m4, m15
    packuswb        m4, m4 ; clip px in [0x0,0xff]
    pminub          m4, m7
    pmaxub          m4, m8
 %endif
    vextracti128   xm5, m4, 1
    movd   [dstq+strideq*0], xm4
    movd   [dstq+strideq*2], xm5
    pextrd [dstq+strideq*1], xm4, 1
    pextrd [dstq+stride3q ], xm5, 1
%else
    pcmpgtw         m6, %3, m12
    pcmpgtw         m5, %3, m15
    paddw          m12, m6
    paddw          m15, m5
 %if %5 == 1
    punpckhbw       m5, m4, %3
    punpcklbw       m4, %3
 %endif
    pmulhrsw       m12, %4
    pmulhrsw       m15, %4
 %if %5 == 0
    packsswb       m15, m12
    paddb           m4, m15
 %else
    paddw           m5, m12
    paddw           m4, m15
    packuswb        m4, m5 ; clip px in [0x0,0xff]
    pminub          m4, m7
    pmaxub          m4, m8
 %endif
    vextracti128   xm5, m4, 1
 %if %1 == 4
    movd   [dstq +strideq*0], xm4
    movd   [dst4q+strideq*0], xm5
    pextrd [dstq +strideq*1], xm4, 1
    pextrd [dst4q+strideq*1], xm5, 1
    pextrd [dstq +strideq*2], xm4, 2
    pextrd [dst4q+strideq*2], xm5, 2
    pextrd [dstq +stride3q ], xm4, 3
    pextrd [dst4q+stride3q ], xm5, 3
 %else
    movq   [dstq+strideq*0], xm4
    movq   [dstq+strideq*2], xm5
    movhps [dstq+strideq*1], xm4
    movhps [dstq+stride3q ], xm5
 %endif
%endif
%endmacro

%macro BORDER_PREP_REGS 2 ; w, h
    ; off1/2/3[k] [6 total] from [tapq+12+(dir+0/2/6)*2+k]
    mov           dird, r7m
    lea           dirq, [tableq+dirq*2+14]
%if %1*%2*2/mmsize > 1
 %if %1 == 4
    DEFINE_ARGS dst, stride, k, dir, stk, pri, sec, stride3, h, off
 %else
    DEFINE_ARGS dst, stride, k, dir, stk, pri, sec, h, off
 %endif
    mov             hd, %1*%2*2/mmsize
%else
    DEFINE_ARGS dst, stride, k, dir, stk, pri, sec, stride3, off
%endif
    lea           stkq, [px]
    pxor           m11, m11
%endmacro

%macro BORDER_LOAD_BLOCK 2-3 0 ; w, h, init_min_max
    mov             kd, 1
%if %1 == 4
    movq           xm4, [stkq+32*0]
    movhps         xm4, [stkq+32*1]
    movq           xm5, [stkq+32*2]
    movhps         xm5, [stkq+32*3]
    vinserti128     m4, xm5, 1
%else
    mova           xm4, [stkq+32*0]             ; px
    vinserti128     m4, [stkq+32*1], 1
%endif
    pxor           m15, m15                     ; sum
%if %3 == 1
    mova            m7, m4                      ; max
    mova            m8, m4                      ; min
%endif
%endmacro

%macro ACCUMULATE_TAP_WORD 6-7 0 ; tap_offset, shift, mask, strength
                                 ; mul_tap, w, clip
    ; load p0/p1
    movsx         offq, byte [dirq+kq+%1]       ; off1
%if %6 == 4
    movq           xm5, [stkq+offq*2+32*0]      ; p0
    movq           xm6, [stkq+offq*2+32*2]
    movhps         xm5, [stkq+offq*2+32*1]
    movhps         xm6, [stkq+offq*2+32*3]
    vinserti128     m5, xm6, 1
%else
    movu           xm5, [stkq+offq*2+32*0]      ; p0
    vinserti128     m5, [stkq+offq*2+32*1], 1
%endif
    neg           offq                          ; -off1
%if %6 == 4
    movq           xm6, [stkq+offq*2+32*0]      ; p1
    movq           xm9, [stkq+offq*2+32*2]
    movhps         xm6, [stkq+offq*2+32*1]
    movhps         xm9, [stkq+offq*2+32*3]
    vinserti128     m6, xm9, 1
%else
    movu           xm6, [stkq+offq*2+32*0]      ; p1
    vinserti128     m6, [stkq+offq*2+32*1], 1
%endif
%if %7 == 1
    ; out of bounds values are set to a value that is a both a large unsigned
    ; value and a negative signed value.
    ; use signed max and unsigned min to remove them
    pmaxsw          m7, m5                      ; max after p0
    pminuw          m8, m5                      ; min after p0
    pmaxsw          m7, m6                      ; max after p1
    pminuw          m8, m6                      ; min after p1
%endif

    ; accumulate sum[m15] over p0/p1
    ; calculate difference before converting
    psubw           m5, m4                      ; diff_p0(p0 - px)
    psubw           m6, m4                      ; diff_p1(p1 - px)

    ; convert to 8-bits with signed saturation
    ; saturating to large diffs has no impact on the results
    packsswb        m5, m6

    ; group into pairs so we can accumulate using maddubsw
    pshufb          m5, m12
    pabsb           m9, m5
    psignb         m10, %5, m5
    psrlw           m5, m9, %2                  ; emulate 8-bit shift
    pand            m5, %3
    psubusb         m5, %4, m5

    ; use unsigned min since abs diff can equal 0x80
    pminub          m5, m9
    pmaddubsw       m5, m10
    paddw          m15, m5
%endmacro

%macro BORDER_ADJUST_PIXEL 2-3 0 ; w, pw_2048, clip
    pcmpgtw         m9, m11, m15
    paddw          m15, m9
    pmulhrsw       m15, %2
    paddw           m4, m15
%if %3 == 1
    pminsw          m4, m7
    pmaxsw          m4, m8
%endif
    packuswb        m4, m4
    vextracti128   xm5, m4, 1
%if %1 == 4
    movd   [dstq+strideq*0], xm4
    pextrd [dstq+strideq*1], xm4, 1
    movd   [dstq+strideq*2], xm5
    pextrd [dstq+stride3q ], xm5, 1
%else
    movq [dstq+strideq*0], xm4
    movq [dstq+strideq*1], xm5
%endif
%endmacro

%macro CDEF_FILTER 2 ; w, h
INIT_YMM avx2
cglobal cdef_filter_%1x%2_8bpc, 5, 11, 0, dst, stride, left, top, bot, \
                                          pri, sec, dir, damping, edge
    mov          edged, edgem
    cmp          edged, 0xf
    jne .border_block

    PUSH           r11
    PUSH           r12
%if %2 == 4
%assign regs_used 13
    ALLOC_STACK   0x60, 16
    pmovzxbw       xm0, [leftq+1]
    vpermq          m0, m0, q0110
    psrldq          m1, m0, 4
    vpalignr        m2, m0, m0, 12
    movu    [rsp+0x10], m0
    movu    [rsp+0x28], m1
    movu    [rsp+0x40], m2
%elif %1 == 4
%assign regs_used 14
    PUSH           r13
    ALLOC_STACK 8*2+%1*%2*1, 16
    pmovzxwd        m0, [leftq]
    mova    [rsp+0x10], m0
%else
%assign regs_used 15
    PUSH           r13
    PUSH           r14
    ALLOC_STACK 8*4+%1*%2*2+32, 16
    lea            r11, [strideq*3]
    movu           xm4, [dstq+strideq*2]
    pmovzxwq        m0, [leftq+0]
    pmovzxwq        m1, [leftq+8]
    vinserti128     m4, [dstq+r11], 1
    pmovzxbd        m2, [leftq+1]
    pmovzxbd        m3, [leftq+9]
    mov       [rsp+16], botq
    mova    [rsp+0x20], m0
    mova    [rsp+0x40], m1
    mova    [rsp+0x60], m2
    mova    [rsp+0x80], m3
    mova    [rsp+0xa0], m4
    lea           botq, [dstq+strideq*4]
%endif

 DEFINE_ARGS dst, stride, left, top, bot, pri, secdmp, zero, pridmp, damping
    mov       dampingd, r8m
    xor          zerod, zerod
    movifnidn     prid, prim
    sub       dampingd, 31
    movifnidn  secdmpd, secdmpm
    test          prid, prid
    jz .sec_only
    movd           xm0, prid
    lzcnt      pridmpd, prid
    add        pridmpd, dampingd
    cmovs      pridmpd, zerod
    mov        [rsp+0], pridmpq                 ; pri_shift
    test       secdmpd, secdmpd
    jz .pri_only
    movd           xm1, secdmpd
    lzcnt      secdmpd, secdmpd
    add        secdmpd, dampingd
    mov        [rsp+8], secdmpq                 ; sec_shift

 DEFINE_ARGS dst, stride, left, top, bot, pri, secdmp, table, pridmp
    lea         tableq, [tap_table]
    vpbroadcastb   m13, [tableq+pridmpq]        ; pri_shift_mask
    vpbroadcastb   m14, [tableq+secdmpq]        ; sec_shift_mask

    ; pri/sec_taps[k] [4 total]
 DEFINE_ARGS dst, stride, left, top, bot, pri, sec, table, dir
    vpbroadcastb    m0, xm0                     ; pri_strength
    vpbroadcastb    m1, xm1                     ; sec_strength
    and           prid, 1
    lea           priq, [tableq+priq*2+8]       ; pri_taps
    lea           secq, [tableq+12]             ; sec_taps

    PREP_REGS       %1, %2
%if %1*%2 > mmsize
.v_loop:
%endif
    LOAD_BLOCK      %1, %2, 1
.k_loop:
    vpbroadcastb    m2, [priq+kq]                          ; pri_taps
    vpbroadcastb    m3, [secq+kq]                          ; sec_taps
    ACCUMULATE_TAP_BYTE 2, [rsp+0], m13, m0, m2, %1, %2, 1 ; dir + 0
    ACCUMULATE_TAP_BYTE 4, [rsp+8], m14, m1, m3, %1, %2, 1 ; dir + 2
    ACCUMULATE_TAP_BYTE 0, [rsp+8], m14, m1, m3, %1, %2, 1 ; dir - 2
    dec             kq
    jge .k_loop

    vpbroadcastd   m10, [pw_2048]
    pxor            m9, m9
    ADJUST_PIXEL    %1, %2, m9, m10, 1
%if %1*%2 > mmsize
    lea           dstq, [dstq+strideq*4]
    lea          top1q, [rsp+0xa0]
    lea          top2q, [rsp+0xb0]
    mov           botq, [rsp+16]
    add             hq, 4
    jl .v_loop
%endif
    RET

.pri_only:
 DEFINE_ARGS dst, stride, left, top, bot, pri, _, table, pridmp
    lea         tableq, [tap_table]
    vpbroadcastb   m13, [tableq+pridmpq]        ; pri_shift_mask
    ; pri/sec_taps[k] [4 total]
 DEFINE_ARGS dst, stride, left, top, bot, pri, _, table, dir
    vpbroadcastb    m0, xm0                     ; pri_strength
    and           prid, 1
    lea           priq, [tableq+priq*2+8]       ; pri_taps
    PREP_REGS       %1, %2
    vpbroadcastd    m3, [pw_2048]
    pxor            m1, m1
%if %1*%2 > mmsize
.pri_v_loop:
%endif
    LOAD_BLOCK      %1, %2
.pri_k_loop:
    vpbroadcastb    m2, [priq+kq]                       ; pri_taps
    ACCUMULATE_TAP_BYTE 2, [rsp+0], m13, m0, m2, %1, %2 ; dir + 0
    dec             kq
    jge .pri_k_loop
    ADJUST_PIXEL    %1, %2, m1, m3
%if %1*%2 > mmsize
    lea           dstq, [dstq+strideq*4]
    lea          top1q, [rsp+0xa0]
    lea          top2q, [rsp+0xb0]
    mov           botq, [rsp+16]
    add             hq, 4
    jl .pri_v_loop
%endif
    RET

.sec_only:
 DEFINE_ARGS dst, stride, left, top, bot, _, secdmp, zero, _, damping
    movd           xm1, secdmpd
    lzcnt      secdmpd, secdmpd
    add        secdmpd, dampingd
    mov        [rsp+8], secdmpq                 ; sec_shift
 DEFINE_ARGS dst, stride, left, top, bot, _, secdmp, table
    lea         tableq, [tap_table]
    vpbroadcastb   m14, [tableq+secdmpq]        ; sec_shift_mask
    ; pri/sec_taps[k] [4 total]
 DEFINE_ARGS dst, stride, left, top, bot, _, sec, table, dir
    vpbroadcastb    m1, xm1                     ; sec_strength
    lea           secq, [tableq+12]             ; sec_taps
    PREP_REGS       %1, %2
    vpbroadcastd    m2, [pw_2048]
    pxor            m0, m0
%if %1*%2 > mmsize
.sec_v_loop:
%endif
    LOAD_BLOCK      %1, %2
.sec_k_loop:
    vpbroadcastb    m3, [secq+kq]                       ; sec_taps
    ACCUMULATE_TAP_BYTE 4, [rsp+8], m14, m1, m3, %1, %2 ; dir + 2
    ACCUMULATE_TAP_BYTE 0, [rsp+8], m14, m1, m3, %1, %2 ; dir - 2
    dec             kq
    jge .sec_k_loop
    ADJUST_PIXEL    %1, %2, m0, m2
%if %1*%2 > mmsize
    lea           dstq, [dstq+strideq*4]
    lea          top1q, [rsp+0xa0]
    lea          top2q, [rsp+0xb0]
    mov           botq, [rsp+16]
    add             hq, 4
    jl .sec_v_loop
%endif
    RET

.d0k0:
%if %1 == 4
 %if %2 == 4
    vpbroadcastq    m6, [dstq+strideq*1-1]
    vpbroadcastq   m10, [dstq+strideq*2-1]
    movd           xm5, [topq+strideq*1+1]
    movd           xm9, [dstq+strideq*0+1]
    psrldq         m11, m6, 2
    psrldq         m12, m10, 2
    vinserti128     m6, [dstq+stride3q -1], 1
    vinserti128    m10, [botq          -1], 1
    vpblendd        m5, m11, 0x10
    vpblendd        m9, m12, 0x10
    movu           m11, [blend_4x4+16]
    punpckldq       m6, m10
    punpckldq       m5, m9
    vpblendvb       m6, [rsp+gprsize+0x28], m11
 %else
    movd           xm5, [topq +strideq*1+1]
    movq           xm6, [dstq +strideq*1-1]
    movq          xm10, [dstq +stride3q -1]
    movq          xm11, [dst4q+strideq*1-1]
    pinsrd         xm5, [dstq +strideq*0+1], 1
    movhps         xm6, [dstq +strideq*2-1]
    movhps        xm10, [dst4q+strideq*0-1]
    movhps        xm11, [dst4q+strideq*2-1]
    psrldq         xm9, xm6, 2
    shufps         xm5, xm9, q2010   ; -1 +0 +1 +2
    shufps         xm6, xm10, q2020  ; +1 +2 +3 +4
    psrldq         xm9, xm11, 2
    psrldq        xm10, 2
    shufps        xm10, xm9, q2020   ; +3 +4 +5 +6
    movd           xm9, [dst4q+stride3q -1]
    pinsrd         xm9, [botq           -1], 1
    shufps        xm11, xm9, q1020   ; +5 +6 +7 +8
    pmovzxbw        m9, [leftq+3]
    vinserti128     m6, xm11, 1
    movu           m11, [blend_4x8_0+4]
    vinserti128     m5, xm10, 1
    vpblendvb       m6, m9, m11
 %endif
%else
    lea            r13, [blend_8x8_0+16]
    movq           xm5, [top2q         +1]
    vbroadcasti128 m10, [dstq+strideq*1-1]
    vbroadcasti128 m11, [dstq+strideq*2-1]
    movhps         xm5, [dstq+strideq*0+1]
    vinserti128     m6, m10, [dstq+stride3q-1], 1
    vinserti128     m9, m11, [botq         -1], 1
    psrldq         m10, 2
    psrldq         m11, 2
    punpcklqdq      m6, m9
    movu            m9, [r13+hq*2*1+16*1]
    punpcklqdq     m10, m11
    vpblendd        m5, m10, 0xF0
    vpblendvb       m6, [rsp+gprsize+0x60+hq*8+64+8*1], m9
%endif
    ret
.d1k0:
.d2k0:
.d3k0:
%if %1 == 4
 %if %2 == 4
    movq           xm6, [dstq+strideq*0-1]
    movq           xm9, [dstq+strideq*1-1]
    vinserti128     m6, [dstq+strideq*2-1], 1
    vinserti128     m9, [dstq+stride3q -1], 1
    movu           m11, [rsp+gprsize+0x10]
    pcmpeqd        m12, m12
    psrldq          m5, m6, 2
    psrldq         m10, m9, 2
    psrld          m12, 24
    punpckldq       m6, m9
    punpckldq       m5, m10
    vpblendvb       m6, m11, m12
 %else
    movq           xm6, [dstq +strideq*0-1]
    movq           xm9, [dstq +strideq*2-1]
    movhps         xm6, [dstq +strideq*1-1]
    movhps         xm9, [dstq +stride3q -1]
    movq          xm10, [dst4q+strideq*0-1]
    movhps        xm10, [dst4q+strideq*1-1]
    psrldq         xm5, xm6, 2
    psrldq        xm11, xm9, 2
    shufps         xm5, xm11, q2020
    movq          xm11, [dst4q+strideq*2-1]
    movhps        xm11, [dst4q+stride3q -1]
    shufps         xm6, xm9, q2020
    shufps         xm9, xm10, xm11, q2020
    vinserti128     m6, xm9, 1
    pmovzxbw        m9, [leftq+1]
    psrldq        xm10, 2
    psrldq        xm11, 2
    shufps        xm10, xm11, q2020
    vpbroadcastd   m11, [blend_4x8_0+4]
    vinserti128     m5, xm10, 1
    vpblendvb       m6, m9, m11
 %endif
%else
    movu           xm5, [dstq+strideq*0-1]
    movu           xm9, [dstq+strideq*1-1]
    vinserti128     m5, [dstq+strideq*2-1], 1
    vinserti128     m9, [dstq+stride3q -1], 1
    movu           m10, [blend_8x8_0+16]
    punpcklqdq      m6, m5, m9
    vpblendvb       m6, [rsp+gprsize+0x60+hq*8+64], m10
    psrldq          m5, 2
    psrldq          m9, 2
    punpcklqdq      m5, m9
%endif
    ret
.d4k0:
%if %1 == 4
 %if %2 == 4
    vpbroadcastq   m10, [dstq+strideq*1-1]
    vpbroadcastq   m11, [dstq+strideq*2-1]
    movd           xm6, [topq+strideq*1-1]
    movd           xm9, [dstq+strideq*0-1]
    psrldq          m5, m10, 2
    psrldq         m12, m11, 2
    vpblendd        m6, m10, 0x10
    vpblendd        m9, m11, 0x10
    movu           m10, [blend_4x4]
    vinserti128     m5, [dstq+stride3q +1], 1
    vinserti128    m12, [botq          +1], 1
    punpckldq       m6, m9
    punpckldq       m5, m12
    vpblendvb       m6, [rsp+gprsize+0x40], m10
 %else
    movd           xm6, [topq +strideq*1-1]
    movq           xm9, [dstq +strideq*1-1]
    movq          xm10, [dstq +stride3q -1]
    movq          xm11, [dst4q+strideq*1-1]
    pinsrd         xm6, [dstq +strideq*0-1], 1
    movhps         xm9, [dstq +strideq*2-1]
    movhps        xm10, [dst4q+strideq*0-1]
    movhps        xm11, [dst4q+strideq*2-1]
    psrldq         xm5, xm9, 2
    shufps         xm6, xm9, q2010
    psrldq         xm9, xm10, 2
    shufps         xm5, xm9, q2020
    shufps        xm10, xm11, q2020
    movd           xm9, [dst4q+stride3q +1]
    vinserti128     m6, xm10, 1
    pinsrd         xm9, [botq           +1], 1
    psrldq        xm11, 2
    pmovzxbw       m10, [leftq-1]
    shufps        xm11, xm9, q1020
    movu            m9, [blend_4x8_0]
    vinserti128     m5, xm11, 1
    vpblendvb       m6, m10, m9
 %endif
%else
    lea            r13, [blend_8x8_0+8]
    movq           xm6, [top2q         -1]
    vbroadcasti128  m5, [dstq+strideq*1-1]
    vbroadcasti128  m9, [dstq+strideq*2-1]
    movhps         xm6, [dstq+strideq*0-1]
    movu           m11, [r13+hq*2*1+16*1]
    punpcklqdq     m10, m5, m9
    vinserti128     m5, [dstq+stride3q -1], 1
    vinserti128     m9, [botq          -1], 1
    vpblendd        m6, m10, 0xF0
    vpblendvb       m6, [rsp+gprsize+0x60+hq*8+64-8*1], m11
    psrldq          m5, 2
    psrldq          m9, 2
    punpcklqdq      m5, m9
%endif
    ret
.d5k0:
.d6k0:
.d7k0:
%if %1 == 4
 %if %2 == 4
    movd           xm6, [topq+strideq*1  ]
    vpbroadcastd    m5, [dstq+strideq*1  ]
    vpbroadcastd    m9, [dstq+strideq*2  ]
    vpblendd       xm6, [dstq+strideq*0-4], 0x2
    vpblendd        m5, m9, 0x22
    vpblendd        m6, m5, 0x30
    vinserti128     m5, [dstq+stride3q   ], 1
    vpblendd        m5, [botq         -20], 0x20
 %else
    movd           xm6, [topq +strideq*1]
    movd           xm5, [dstq +strideq*1]
    movd           xm9, [dstq +stride3q ]
    movd          xm10, [dst4q+strideq*1]
    movd          xm11, [dst4q+stride3q ]
    pinsrd         xm6, [dstq +strideq*0], 1
    pinsrd         xm5, [dstq +strideq*2], 1
    pinsrd         xm9, [dst4q+strideq*0], 1
    pinsrd        xm10, [dst4q+strideq*2], 1
    pinsrd        xm11, [botq           ], 1
    punpcklqdq     xm6, xm5
    punpcklqdq     xm5, xm9
    punpcklqdq     xm9, xm10
    punpcklqdq    xm10, xm11
    vinserti128     m6, xm9, 1
    vinserti128     m5, xm10, 1
 %endif
%else
    movq           xm6, [top2q         ]
    movq           xm5, [dstq+strideq*1]
    movq           xm9, [dstq+stride3q ]
    movhps         xm6, [dstq+strideq*0]
    movhps         xm5, [dstq+strideq*2]
    movhps         xm9, [botq          ]
    vinserti128     m6, xm5, 1
    vinserti128     m5, xm9, 1
%endif
    ret
.d0k1:
%if %1 == 4
 %if %2 == 4
    movd           xm6, [dstq+strideq*2-2]
    movd           xm9, [dstq+stride3q -2]
    movd           xm5, [topq+strideq*0+2]
    movd          xm10, [topq+strideq*1+2]
    pinsrw         xm6, [leftq+4], 0
    pinsrw         xm9, [leftq+6], 0
    vinserti128     m5, [dstq+strideq*0+2], 1
    vinserti128    m10, [dstq+strideq*1+2], 1
    vinserti128     m6, [botq+strideq*0-2], 1
    vinserti128     m9, [botq+strideq*1-2], 1
    punpckldq       m5, m10
    punpckldq       m6, m9
 %else
    movq           xm6, [dstq +strideq*2-2]
    movd          xm10, [dst4q+strideq*2-2]
    movd           xm5, [topq +strideq*0+2]
    movq           xm9, [dst4q+strideq*0-2]
    movhps         xm6, [dstq +stride3q -2]
    pinsrw        xm10, [dst4q+stride3q   ], 3
    pinsrd         xm5, [topq +strideq*1+2], 1
    movhps         xm9, [dst4q+strideq*1-2]
    pinsrd        xm10, [botq +strideq*0-2], 2
    pinsrd         xm5, [dstq +strideq*0+2], 2
    pinsrd        xm10, [botq +strideq*1-2], 3
    pinsrd         xm5, [dstq +strideq*1+2], 3
    shufps        xm11, xm6, xm9, q3131
    shufps         xm6, xm9, q2020
    movu            m9, [blend_4x8_3+8]
    vinserti128     m6, xm10, 1
    vinserti128     m5, xm11, 1
    vpblendvb       m6, [rsp+gprsize+0x10+8], m9
 %endif
%else
    lea            r13, [blend_8x8_1+16]
    movq           xm6, [dstq+strideq*2-2]
    movq           xm9, [dstq+stride3q -2]
    movq           xm5, [top1q         +2]
    movq          xm10, [top2q         +2]
    movu           m11, [r13+hq*2*2+16*2]
    vinserti128     m6, [botq+strideq*0-2], 1
    vinserti128     m9, [botq+strideq*1-2], 1
    vinserti128     m5, [dstq+strideq*0+2], 1
    vinserti128    m10, [dstq+strideq*1+2], 1
    punpcklqdq      m6, m9
    punpcklqdq      m5, m10
    vpblendvb       m6, [rsp+gprsize+0x20+hq*8+64+8*2], m11
%endif
    ret
.d1k1:
%if %1 == 4
 %if %2 == 4
    vpbroadcastq    m6, [dstq+strideq*1-2]
    vpbroadcastq    m9, [dstq+strideq*2-2]
    movd           xm5, [topq+strideq*1+2]
    movd          xm10, [dstq+strideq*0+2]
    psrldq         m11, m6, 4
    psrldq         m12, m9, 4
    vpblendd        m5, m11, 0x10
    movq          xm11, [leftq+2]
    vinserti128     m6, [dstq+stride3q-2], 1
    punpckldq     xm11, xm11
    vpblendd       m10, m12, 0x10
    pcmpeqd        m12, m12
    pmovzxwd       m11, xm11
    psrld          m12, 16
    punpckldq       m6, m9
    vpbroadcastd    m9, [botq-2]
    vpblendvb       m6, m11, m12
    punpckldq       m5, m10
    vpblendd        m6, m9, 0x20
 %else
    movd           xm5, [topq +strideq*1+2]
    movq           xm6, [dstq +strideq*1-2]
    movq           xm9, [dstq +stride3q -2]
    movq          xm10, [dst4q+strideq*1-2]
    movd          xm11, [dst4q+stride3q -2]
    pinsrd         xm5, [dstq +strideq*0+2], 1
    movhps         xm6, [dstq +strideq*2-2]
    movhps         xm9, [dst4q+strideq*0-2]
    movhps        xm10, [dst4q+strideq*2-2]
    pinsrd        xm11, [botq           -2], 1
    shufps         xm5, xm6, q3110
    shufps         xm6, xm9, q2020
    shufps         xm9, xm10, q3131
    shufps        xm10, xm11, q1020
    movu           m11, [blend_4x8_2+4]
    vinserti128     m6, xm10, 1
    vinserti128     m5, xm9, 1
    vpblendvb       m6, [rsp+gprsize+0x10+4], m11
 %endif
%else
    lea            r13, [blend_8x8_1+16]
    movq           xm5, [top2q         +2]
    vbroadcasti128  m6, [dstq+strideq*1-2]
    vbroadcasti128  m9, [dstq+strideq*2-2]
    movhps         xm5, [dstq+strideq*0+2]
    shufps         m10, m6, m9, q2121
    vinserti128     m6, [dstq+stride3q -2], 1
    vinserti128     m9, [botq          -2], 1
    movu           m11, [r13+hq*2*1+16*1]
    vpblendd        m5, m10, 0xF0
    punpcklqdq      m6, m9
    vpblendvb       m6, [rsp+gprsize+0x20+hq*8+64+8*1], m11
%endif
    ret
.d2k1:
%if %1 == 4
 %if %2 == 4
    movq          xm11, [leftq]
    movq           xm6, [dstq+strideq*0-2]
    movq           xm9, [dstq+strideq*1-2]
    vinserti128     m6, [dstq+strideq*2-2], 1
    vinserti128     m9, [dstq+stride3q -2], 1
    punpckldq     xm11, xm11
    psrldq          m5, m6, 4
    psrldq         m10, m9, 4
    pmovzxwd       m11, xm11
    punpckldq       m6, m9
    punpckldq       m5, m10
    pblendw         m6, m11, 0x05
 %else
    movq           xm5, [dstq +strideq*0-2]
    movq           xm9, [dstq +strideq*2-2]
    movq          xm10, [dst4q+strideq*0-2]
    movq          xm11, [dst4q+strideq*2-2]
    movhps         xm5, [dstq +strideq*1-2]
    movhps         xm9, [dstq +stride3q -2]
    movhps        xm10, [dst4q+strideq*1-2]
    movhps        xm11, [dst4q+stride3q -2]
    shufps         xm6, xm5, xm9, q2020
    shufps         xm5, xm9, q3131
    shufps         xm9, xm10, xm11, q2020
    shufps        xm10, xm11, q3131
    pmovzxwd       m11, [leftq]
    vinserti128     m6, xm9, 1
    vinserti128     m5, xm10, 1
    pblendw         m6, m11, 0x55
 %endif
%else
    mova           m11, [rsp+gprsize+0x20+hq*8+64]
    movu           xm5, [dstq+strideq*0-2]
    movu           xm9, [dstq+strideq*1-2]
    vinserti128     m5, [dstq+strideq*2-2], 1
    vinserti128     m9, [dstq+stride3q -2], 1
    shufps          m6, m5, m9, q1010
    shufps          m5, m9, q2121
    pblendw         m6, m11, 0x11
%endif
    ret
.d3k1:
%if %1 == 4
 %if %2 == 4
    vpbroadcastq   m11, [dstq+strideq*1-2]
    vpbroadcastq   m12, [dstq+strideq*2-2]
    movd           xm6, [topq+strideq*1-2]
    movd           xm9, [dstq+strideq*0-2]
    pblendw        m11, [leftq-16+2], 0x01
    pblendw        m12, [leftq-16+4], 0x01
    pinsrw         xm9, [leftq- 0+0], 0
    psrldq          m5, m11, 4
    psrldq         m10, m12, 4
    vinserti128     m5, [dstq+stride3q +2], 1
    vinserti128    m10, [botq          +2], 1
    vpblendd        m6, m11, 0x10
    vpblendd        m9, m12, 0x10
    punpckldq       m6, m9
    punpckldq       m5, m10
 %else
    movd           xm6, [topq +strideq*1-2]
    movq           xm5, [dstq +strideq*1-2]
    movq           xm9, [dstq +stride3q -2]
    movq          xm10, [dst4q+strideq*1-2]
    movd          xm11, [dst4q+stride3q +2]
    pinsrw         xm6, [dstq +strideq*0  ], 3
    movhps         xm5, [dstq +strideq*2-2]
    movhps         xm9, [dst4q+strideq*0-2]
    movhps        xm10, [dst4q+strideq*2-2]
    pinsrd        xm11, [botq           +2], 1
    shufps         xm6, xm5, q2010
    shufps         xm5, xm9, q3131
    shufps         xm9, xm10, q2020
    shufps        xm10, xm11, q1031
    movu           m11, [blend_4x8_2]
    vinserti128     m6, xm9, 1
    vinserti128     m5, xm10, 1
    vpblendvb       m6, [rsp+gprsize+0x10-4], m11
 %endif
%else
    lea            r13, [blend_8x8_1+8]
    movq           xm6, [top2q         -2]
    vbroadcasti128  m5, [dstq+strideq*1-2]
    vbroadcasti128 m10, [dstq+strideq*2-2]
    movhps         xm6, [dstq+strideq*0-2]
    punpcklqdq      m9, m5, m10
    vinserti128     m5, [dstq+stride3q -2], 1
    vinserti128    m10, [botq          -2], 1
    movu           m11, [r13+hq*2*1+16*1]
    vpblendd        m6, m9, 0xF0
    shufps          m5, m10, q2121
    vpblendvb       m6, [rsp+gprsize+0x20+hq*8+64-8*1], m11
%endif
    ret
.d4k1:
%if %1 == 4
 %if %2 == 4
    vinserti128     m6, [dstq+strideq*0-2], 1
    vinserti128     m9, [dstq+strideq*1-2], 1
    movd           xm5, [dstq+strideq*2+2]
    movd          xm10, [dstq+stride3q +2]
    pblendw         m6, [leftq-16+0], 0x01
    pblendw         m9, [leftq-16+2], 0x01
    vinserti128     m5, [botq+strideq*0+2], 1
    vinserti128    m10, [botq+strideq*1+2], 1
    vpblendd        m6, [topq+strideq*0-2], 0x01
    vpblendd        m9, [topq+strideq*1-2], 0x01
    punpckldq       m5, m10
    punpckldq       m6, m9
 %else
    movd           xm6, [topq +strideq*0-2]
    movq           xm5, [dstq +strideq*2-2]
    movq           xm9, [dst4q+strideq*0-2]
    movd          xm10, [dst4q+strideq*2+2]
    pinsrd         xm6, [topq +strideq*1-2], 1
    movhps         xm5, [dstq +stride3q -2]
    movhps         xm9, [dst4q+strideq*1-2]
    pinsrd        xm10, [dst4q+stride3q +2], 1
    pinsrd         xm6, [dstq +strideq*0-2], 2
    pinsrd        xm10, [botq +strideq*0+2], 2
    pinsrd         xm6, [dstq +strideq*1-2], 3
    pinsrd        xm10, [botq +strideq*1+2], 3
    shufps        xm11, xm5, xm9, q2020
    shufps         xm5, xm9, q3131
    movu            m9, [blend_4x8_3]
    vinserti128     m6, xm11, 1
    vinserti128     m5, xm10, 1
    vpblendvb       m6, [rsp+gprsize+0x10-8], m9
 %endif
%else
    lea            r13, [blend_8x8_1]
    movu           m11, [r13+hq*2*2+16*2]
    movq           xm6, [top1q         -2]
    movq           xm9, [top2q         -2]
    movq           xm5, [dstq+strideq*2+2]
    movq          xm10, [dstq+stride3q +2]
    vinserti128     m6, [dstq+strideq*0-2], 1
    vinserti128     m9, [dstq+strideq*1-2], 1
    vinserti128     m5, [botq+strideq*0+2], 1
    vinserti128    m10, [botq+strideq*1+2], 1
    punpcklqdq      m6, m9
    vpblendvb       m6, [rsp+gprsize+0x20+hq*8+64-8*2], m11
    punpcklqdq      m5, m10
%endif
    ret
.d5k1:
%if %1 == 4
 %if %2 == 4
    movd           xm6, [topq+strideq*0-1]
    movd           xm9, [topq+strideq*1-1]
    movd           xm5, [dstq+strideq*2+1]
    movd          xm10, [dstq+stride3q +1]
    pcmpeqd        m12, m12
    pmovzxbw       m11, [leftq-8+1]
    psrld          m12, 24
    vinserti128     m6, [dstq+strideq*0-1], 1
    vinserti128     m9, [dstq+strideq*1-1], 1
    vinserti128     m5, [botq+strideq*0+1], 1
    vinserti128    m10, [botq+strideq*1+1], 1
    punpckldq       m6, m9
    pxor            m9, m9
    vpblendd       m12, m9, 0x0F
    punpckldq       m5, m10
    vpblendvb       m6, m11, m12
 %else
    movd           xm6, [topq +strideq*0-1]
    movq           xm5, [dstq +strideq*2-1]
    movq           xm9, [dst4q+strideq*0-1]
    movd          xm10, [dst4q+strideq*2+1]
    pinsrd         xm6, [topq +strideq*1-1], 1
    movhps         xm5, [dstq +stride3q -1]
    movhps         xm9, [dst4q+strideq*1-1]
    pinsrd        xm10, [dst4q+stride3q +1], 1
    pinsrd         xm6, [dstq +strideq*0-1], 2
    pinsrd        xm10, [botq +strideq*0+1], 2
    pinsrd         xm6, [dstq +strideq*1-1], 3
    pinsrd        xm10, [botq +strideq*1+1], 3
    shufps        xm11, xm5, xm9, q2020
    vinserti128     m6, xm11, 1
    pmovzxbw       m11, [leftq-3]
    psrldq         xm5, 2
    psrldq         xm9, 2
    shufps         xm5, xm9, q2020
    movu            m9, [blend_4x8_1]
    vinserti128     m5, xm10, 1
    vpblendvb       m6, m11, m9
 %endif
%else
    lea            r13, [blend_8x8_0]
    movu           m11, [r13+hq*2*2+16*2]
    movq           xm6, [top1q         -1]
    movq           xm9, [top2q         -1]
    movq           xm5, [dstq+strideq*2+1]
    movq          xm10, [dstq+stride3q +1]
    vinserti128     m6, [dstq+strideq*0-1], 1
    vinserti128     m9, [dstq+strideq*1-1], 1
    vinserti128     m5, [botq+strideq*0+1], 1
    vinserti128    m10, [botq+strideq*1+1], 1
    punpcklqdq      m6, m9
    punpcklqdq      m5, m10
    vpblendvb       m6, [rsp+gprsize+0x60+hq*8+64-8*2], m11
%endif
    ret
.d6k1:
%if %1 == 4
 %if %2 == 4
    movd           xm6, [topq+strideq*0]
    movd           xm9, [topq+strideq*1]
    movd           xm5, [dstq+strideq*2]
    movd          xm10, [dstq+stride3q ]
    vinserti128     m6, [dstq+strideq*0], 1
    vinserti128     m9, [dstq+strideq*1], 1
    vinserti128     m5, [botq+strideq*0], 1
    vinserti128    m10, [botq+strideq*1], 1
    punpckldq       m6, m9
    punpckldq       m5, m10
 %else
    movd           xm5, [dstq +strideq*2]
    movd           xm6, [topq +strideq*0]
    movd           xm9, [dst4q+strideq*2]
    pinsrd         xm5, [dstq +stride3q ], 1
    pinsrd         xm6, [topq +strideq*1], 1
    pinsrd         xm9, [dst4q+stride3q ], 1
    pinsrd         xm5, [dst4q+strideq*0], 2
    pinsrd         xm6, [dstq +strideq*0], 2
    pinsrd         xm9, [botq +strideq*0], 2
    pinsrd         xm5, [dst4q+strideq*1], 3
    pinsrd         xm6, [dstq +strideq*1], 3
    pinsrd         xm9, [botq +strideq*1], 3
    vinserti128     m6, xm5, 1
    vinserti128     m5, xm9, 1
 %endif
%else
    movq           xm5, [dstq+strideq*2]
    movq           xm9, [botq+strideq*0]
    movq           xm6, [top1q         ]
    movq          xm10, [dstq+strideq*0]
    movhps         xm5, [dstq+stride3q ]
    movhps         xm9, [botq+strideq*1]
    movhps         xm6, [top2q         ]
    movhps        xm10, [dstq+strideq*1]
    vinserti128     m5, xm9, 1
    vinserti128     m6, xm10, 1
%endif
    ret
.d7k1:
%if %1 == 4
 %if %2 == 4
    movd           xm5, [dstq+strideq*2-1]
    movd           xm9, [dstq+stride3q -1]
    movd           xm6, [topq+strideq*0+1]
    movd          xm10, [topq+strideq*1+1]
    pinsrb         xm5, [leftq+ 5], 0
    pinsrb         xm9, [leftq+ 7], 0
    vinserti128     m6, [dstq+strideq*0+1], 1
    vinserti128    m10, [dstq+strideq*1+1], 1
    vinserti128     m5, [botq+strideq*0-1], 1
    vinserti128     m9, [botq+strideq*1-1], 1
    punpckldq       m6, m10
    punpckldq       m5, m9
 %else
    movd           xm6, [topq +strideq*0+1]
    movq           xm9, [dstq +strideq*2-1]
    movq          xm10, [dst4q+strideq*0-1]
    movd          xm11, [dst4q+strideq*2-1]
    pinsrd         xm6, [topq +strideq*1+1], 1
    movhps         xm9, [dstq +stride3q -1]
    movhps        xm10, [dst4q+strideq*1-1]
    pinsrd        xm11, [dst4q+stride3q -1], 1
    pinsrd         xm6, [dstq +strideq*0+1], 2
    pinsrd        xm11, [botq +strideq*0-1], 2
    pinsrd         xm6, [dstq +strideq*1+1], 3
    pinsrd        xm11, [botq +strideq*1-1], 3
    shufps         xm5, xm9, xm10, q2020
    vinserti128     m5, xm11, 1
    pmovzxbw       m11, [leftq+5]
    psrldq         xm9, 2
    psrldq        xm10, 2
    shufps         xm9, xm10, q2020
    movu           m10, [blend_4x8_1+8]
    vinserti128     m6, xm9, 1
    vpblendvb       m5, m11, m10
 %endif
%else
    lea            r13, [blend_8x8_0+16]
    movq           xm5, [dstq+strideq*2-1]
    movq           xm9, [botq+strideq*0-1]
    movq           xm6, [top1q         +1]
    movq          xm10, [dstq+strideq*0+1]
    movhps         xm5, [dstq+stride3q -1]
    movhps         xm9, [botq+strideq*1-1]
    movhps         xm6, [top2q         +1]
    movhps        xm10, [dstq+strideq*1+1]
    movu           m11, [r13+hq*2*2+16*2]
    vinserti128     m5, xm9, 1
    vinserti128     m6, xm10, 1
    vpblendvb       m5, [rsp+gprsize+0x60+hq*8+64+8*2], m11
%endif
    ret

.border_block:
 DEFINE_ARGS dst, stride, left, top, bot, pri, sec, stride3, dst4, edge
    RESET_STACK_STATE
    %assign stack_offset stack_offset - (regs_used - 11) * gprsize
    %assign regs_used 11
    ALLOC_STACK 2*16+(%2+4)*32, 16
%define px rsp+2*16+2*32

    pcmpeqw        m14, m14
    psllw          m14, 15                  ; 0x8000

    ; prepare pixel buffers - body/right
%if %1 == 4
    INIT_XMM avx2
%endif
%if %2 == 8
    lea          dst4q, [dstq+strideq*4]
%endif
    lea       stride3q, [strideq*3]
    test         edgeb, 2                   ; have_right
    jz .no_right
    pmovzxbw        m1, [dstq+strideq*0]
    pmovzxbw        m2, [dstq+strideq*1]
    pmovzxbw        m3, [dstq+strideq*2]
    pmovzxbw        m4, [dstq+stride3q]
    mova     [px+0*32], m1
    mova     [px+1*32], m2
    mova     [px+2*32], m3
    mova     [px+3*32], m4
%if %2 == 8
    pmovzxbw        m1, [dst4q+strideq*0]
    pmovzxbw        m2, [dst4q+strideq*1]
    pmovzxbw        m3, [dst4q+strideq*2]
    pmovzxbw        m4, [dst4q+stride3q]
    mova     [px+4*32], m1
    mova     [px+5*32], m2
    mova     [px+6*32], m3
    mova     [px+7*32], m4
%endif
    jmp .body_done
.no_right:
%if %1 == 4
    movd           xm1, [dstq+strideq*0]
    movd           xm2, [dstq+strideq*1]
    movd           xm3, [dstq+strideq*2]
    movd           xm4, [dstq+stride3q]
    pmovzxbw       xm1, xm1
    pmovzxbw       xm2, xm2
    pmovzxbw       xm3, xm3
    pmovzxbw       xm4, xm4
    movq     [px+0*32], xm1
    movq     [px+1*32], xm2
    movq     [px+2*32], xm3
    movq     [px+3*32], xm4
%else
    pmovzxbw       xm1, [dstq+strideq*0]
    pmovzxbw       xm2, [dstq+strideq*1]
    pmovzxbw       xm3, [dstq+strideq*2]
    pmovzxbw       xm4, [dstq+stride3q]
    mova     [px+0*32], xm1
    mova     [px+1*32], xm2
    mova     [px+2*32], xm3
    mova     [px+3*32], xm4
%endif
    movd [px+0*32+%1*2], xm14
    movd [px+1*32+%1*2], xm14
    movd [px+2*32+%1*2], xm14
    movd [px+3*32+%1*2], xm14
%if %2 == 8
 %if %1 == 4
    movd           xm1, [dst4q+strideq*0]
    movd           xm2, [dst4q+strideq*1]
    movd           xm3, [dst4q+strideq*2]
    movd           xm4, [dst4q+stride3q]
    pmovzxbw       xm1, xm1
    pmovzxbw       xm2, xm2
    pmovzxbw       xm3, xm3
    pmovzxbw       xm4, xm4
    movq     [px+4*32], xm1
    movq     [px+5*32], xm2
    movq     [px+6*32], xm3
    movq     [px+7*32], xm4
 %else
    pmovzxbw       xm1, [dst4q+strideq*0]
    pmovzxbw       xm2, [dst4q+strideq*1]
    pmovzxbw       xm3, [dst4q+strideq*2]
    pmovzxbw       xm4, [dst4q+stride3q]
    mova     [px+4*32], xm1
    mova     [px+5*32], xm2
    mova     [px+6*32], xm3
    mova     [px+7*32], xm4
 %endif
    movd [px+4*32+%1*2], xm14
    movd [px+5*32+%1*2], xm14
    movd [px+6*32+%1*2], xm14
    movd [px+7*32+%1*2], xm14
%endif
.body_done:

    ; top
    test         edgeb, 4                    ; have_top
    jz .no_top
    test         edgeb, 1                    ; have_left
    jz .top_no_left
    test         edgeb, 2                    ; have_right
    jz .top_no_right
    pmovzxbw        m1, [topq+strideq*0-(%1/2)]
    pmovzxbw        m2, [topq+strideq*1-(%1/2)]
    movu  [px-2*32-%1], m1
    movu  [px-1*32-%1], m2
    jmp .top_done
.top_no_right:
    pmovzxbw        m1, [topq+strideq*0-%1]
    pmovzxbw        m2, [topq+strideq*1-%1]
    movu [px-2*32-%1*2], m1
    movu [px-1*32-%1*2], m2
    movd [px-2*32+%1*2], xm14
    movd [px-1*32+%1*2], xm14
    jmp .top_done
.top_no_left:
    test         edgeb, 2                   ; have_right
    jz .top_no_left_right
    pmovzxbw        m1, [topq+strideq*0]
    pmovzxbw        m2, [topq+strideq*1]
    mova   [px-2*32+0], m1
    mova   [px-1*32+0], m2
    movd   [px-2*32-4], xm14
    movd   [px-1*32-4], xm14
    jmp .top_done
.top_no_left_right:
%if %1 == 4
    movd           xm1, [topq+strideq*0]
    pinsrd         xm1, [topq+strideq*1], 1
    pmovzxbw       xm1, xm1
    movq   [px-2*32+0], xm1
    movhps [px-1*32+0], xm1
%else
    pmovzxbw       xm1, [topq+strideq*0]
    pmovzxbw       xm2, [topq+strideq*1]
    mova   [px-2*32+0], xm1
    mova   [px-1*32+0], xm2
%endif
    movd   [px-2*32-4], xm14
    movd   [px-1*32-4], xm14
    movd [px-2*32+%1*2], xm14
    movd [px-1*32+%1*2], xm14
    jmp .top_done
.no_top:
    movu   [px-2*32-%1], m14
    movu   [px-1*32-%1], m14
.top_done:

    ; left
    test         edgeb, 1                   ; have_left
    jz .no_left
    pmovzxbw       xm1, [leftq+ 0]
%if %2 == 8
    pmovzxbw       xm2, [leftq+ 8]
%endif
    movd   [px+0*32-4], xm1
    pextrd [px+1*32-4], xm1, 1
    pextrd [px+2*32-4], xm1, 2
    pextrd [px+3*32-4], xm1, 3
%if %2 == 8
    movd   [px+4*32-4], xm2
    pextrd [px+5*32-4], xm2, 1
    pextrd [px+6*32-4], xm2, 2
    pextrd [px+7*32-4], xm2, 3
%endif
    jmp .left_done
.no_left:
    movd   [px+0*32-4], xm14
    movd   [px+1*32-4], xm14
    movd   [px+2*32-4], xm14
    movd   [px+3*32-4], xm14
%if %2 == 8
    movd   [px+4*32-4], xm14
    movd   [px+5*32-4], xm14
    movd   [px+6*32-4], xm14
    movd   [px+7*32-4], xm14
%endif
.left_done:

    ; bottom
 DEFINE_ARGS dst, stride, _, _, bot, pri, sec, stride3, _, edge
    test         edgeb, 8                   ; have_bottom
    jz .no_bottom
    test         edgeb, 1                   ; have_left
    jz .bottom_no_left
    test         edgeb, 2                   ; have_right
    jz .bottom_no_right
    pmovzxbw        m1, [botq+strideq*0-(%1/2)]
    pmovzxbw        m2, [botq+strideq*1-(%1/2)]
    movu   [px+(%2+0)*32-%1], m1
    movu   [px+(%2+1)*32-%1], m2
    jmp .bottom_done
.bottom_no_right:
    pmovzxbw        m1, [botq+strideq*0-%1]
    pmovzxbw        m2, [botq+strideq*1-%1]
    movu  [px+(%2+0)*32-%1*2], m1
    movu  [px+(%2+1)*32-%1*2], m2
%if %1 == 8
    movd  [px+(%2-1)*32+%1*2], xm14                ; overwritten by previous movu
%endif
    movd  [px+(%2+0)*32+%1*2], xm14
    movd  [px+(%2+1)*32+%1*2], xm14
    jmp .bottom_done
.bottom_no_left:
    test          edgeb, 2                  ; have_right
    jz .bottom_no_left_right
    pmovzxbw        m1, [botq+strideq*0]
    pmovzxbw        m2, [botq+strideq*1]
    mova   [px+(%2+0)*32+0], m1
    mova   [px+(%2+1)*32+0], m2
    movd   [px+(%2+0)*32-4], xm14
    movd   [px+(%2+1)*32-4], xm14
    jmp .bottom_done
.bottom_no_left_right:
%if %1 == 4
    movd           xm1, [botq+strideq*0]
    pinsrd         xm1, [botq+strideq*1], 1
    pmovzxbw       xm1, xm1
    movq   [px+(%2+0)*32+0], xm1
    movhps [px+(%2+1)*32+0], xm1
%else
    pmovzxbw       xm1, [botq+strideq*0]
    pmovzxbw       xm2, [botq+strideq*1]
    mova   [px+(%2+0)*32+0], xm1
    mova   [px+(%2+1)*32+0], xm2
%endif
    movd   [px+(%2+0)*32-4], xm14
    movd   [px+(%2+1)*32-4], xm14
    movd  [px+(%2+0)*32+%1*2], xm14
    movd  [px+(%2+1)*32+%1*2], xm14
    jmp .bottom_done
.no_bottom:
    movu   [px+(%2+0)*32-%1], m14
    movu   [px+(%2+1)*32-%1], m14
.bottom_done:

    ; actual filter
 INIT_YMM avx2
 DEFINE_ARGS dst, stride, _, pridmp, damping, pri, secdmp, stride3, zero
%undef edged
    ; register to shuffle values into after packing
    vbroadcasti128 m12, [shufb_lohi]

    mov       dampingd, r8m
    xor          zerod, zerod
    movifnidn     prid, prim
    sub       dampingd, 31
    movifnidn  secdmpd, secdmpm
    test          prid, prid
    jz .border_sec_only
    movd           xm0, prid
    lzcnt      pridmpd, prid
    add        pridmpd, dampingd
    cmovs      pridmpd, zerod
    mov        [rsp+0], pridmpq                 ; pri_shift
    test       secdmpd, secdmpd
    jz .border_pri_only
    movd           xm1, secdmpd
    lzcnt      secdmpd, secdmpd
    add        secdmpd, dampingd
    mov        [rsp+8], secdmpq                 ; sec_shift

 DEFINE_ARGS dst, stride, _, pridmp, table, pri, secdmp, stride3
    lea         tableq, [tap_table]
    vpbroadcastb   m13, [tableq+pridmpq]        ; pri_shift_mask
    vpbroadcastb   m14, [tableq+secdmpq]        ; sec_shift_mask

    ; pri/sec_taps[k] [4 total]
 DEFINE_ARGS dst, stride, _, dir, table, pri, sec, stride3
    vpbroadcastb    m0, xm0                     ; pri_strength
    vpbroadcastb    m1, xm1                     ; sec_strength
    and           prid, 1
    lea           priq, [tableq+priq*2+8]       ; pri_taps
    lea           secq, [tableq+12]             ; sec_taps

    BORDER_PREP_REGS %1, %2
%if %1*%2*2/mmsize > 1
.border_v_loop:
%endif
    BORDER_LOAD_BLOCK %1, %2, 1
.border_k_loop:
    vpbroadcastb    m2, [priq+kq]               ; pri_taps
    vpbroadcastb    m3, [secq+kq]               ; sec_taps
    ACCUMULATE_TAP_WORD 0*2, [rsp+0], m13, m0, m2, %1, 1
    ACCUMULATE_TAP_WORD 2*2, [rsp+8], m14, m1, m3, %1, 1
    ACCUMULATE_TAP_WORD 6*2, [rsp+8], m14, m1, m3, %1, 1
    dec             kq
    jge .border_k_loop

    vpbroadcastd   m10, [pw_2048]
    BORDER_ADJUST_PIXEL %1, m10, 1
%if %1*%2*2/mmsize > 1
 %define vloop_lines (mmsize/(%1*2))
    lea           dstq, [dstq+strideq*vloop_lines]
    add           stkq, 32*vloop_lines
    dec             hd
    jg .border_v_loop
%endif
    RET

.border_pri_only:
 DEFINE_ARGS dst, stride, _, pridmp, table, pri, _, stride3
    lea         tableq, [tap_table]
    vpbroadcastb   m13, [tableq+pridmpq]        ; pri_shift_mask
 DEFINE_ARGS dst, stride, _, dir, table, pri, _, stride3
    vpbroadcastb    m0, xm0                     ; pri_strength
    and           prid, 1
    lea           priq, [tableq+priq*2+8]       ; pri_taps
    BORDER_PREP_REGS %1, %2
    vpbroadcastd    m1, [pw_2048]
%if %1*%2*2/mmsize > 1
.border_pri_v_loop:
%endif
    BORDER_LOAD_BLOCK %1, %2
.border_pri_k_loop:
    vpbroadcastb    m2, [priq+kq]               ; pri_taps
    ACCUMULATE_TAP_WORD 0*2, [rsp+0], m13, m0, m2, %1
    dec             kq
    jge .border_pri_k_loop
    BORDER_ADJUST_PIXEL %1, m1
%if %1*%2*2/mmsize > 1
 %define vloop_lines (mmsize/(%1*2))
    lea           dstq, [dstq+strideq*vloop_lines]
    add           stkq, 32*vloop_lines
    dec             hd
    jg .border_pri_v_loop
%endif
    RET

.border_sec_only:
 DEFINE_ARGS dst, stride, _, _, damping, _, secdmp, stride3
    movd           xm1, secdmpd
    lzcnt      secdmpd, secdmpd
    add        secdmpd, dampingd
    mov        [rsp+8], secdmpq                 ; sec_shift
 DEFINE_ARGS dst, stride, _, _, table, _, secdmp, stride3
    lea         tableq, [tap_table]
    vpbroadcastb   m14, [tableq+secdmpq]        ; sec_shift_mask
 DEFINE_ARGS dst, stride, _, dir, table, _, sec, stride3
    vpbroadcastb    m1, xm1                     ; sec_strength
    lea           secq, [tableq+12]             ; sec_taps
    BORDER_PREP_REGS %1, %2
    vpbroadcastd    m0, [pw_2048]
%if %1*%2*2/mmsize > 1
.border_sec_v_loop:
%endif
    BORDER_LOAD_BLOCK %1, %2
.border_sec_k_loop:
    vpbroadcastb    m3, [secq+kq]               ; sec_taps
    ACCUMULATE_TAP_WORD 2*2, [rsp+8], m14, m1, m3, %1
    ACCUMULATE_TAP_WORD 6*2, [rsp+8], m14, m1, m3, %1
    dec             kq
    jge .border_sec_k_loop
    BORDER_ADJUST_PIXEL %1, m0
%if %1*%2*2/mmsize > 1
 %define vloop_lines (mmsize/(%1*2))
    lea           dstq, [dstq+strideq*vloop_lines]
    add           stkq, 32*vloop_lines
    dec             hd
    jg .border_sec_v_loop
%endif
    RET
%endmacro

CDEF_FILTER 8, 8
CDEF_FILTER 4, 8
CDEF_FILTER 4, 4

INIT_YMM avx2
cglobal cdef_dir_8bpc, 3, 4, 6, src, stride, var, stride3
    lea       stride3q, [strideq*3]
    movq           xm0, [srcq+strideq*0]
    movq           xm1, [srcq+strideq*1]
    movq           xm2, [srcq+strideq*2]
    movq           xm3, [srcq+stride3q ]
    lea           srcq, [srcq+strideq*4]
    vpbroadcastq    m4, [srcq+stride3q ]
    vpbroadcastq    m5, [srcq+strideq*2]
    vpblendd        m0, m4, 0xf0
    vpblendd        m1, m5, 0xf0
    vpbroadcastq    m4, [srcq+strideq*1]
    vpbroadcastq    m5, [srcq+strideq*0]
    vpblendd        m2, m4, 0xf0
    vpblendd        m3, m5, 0xf0
    pxor            m4, m4
    punpcklbw       m0, m4
    punpcklbw       m1, m4
    punpcklbw       m2, m4
    punpcklbw       m3, m4
cglobal_label .main
    vpbroadcastd    m4, [pw_128]
    PROLOGUE 3, 4, 15
    psubw           m0, m4
    psubw           m1, m4
    psubw           m2, m4
    psubw           m3, m4

    ; shuffle registers to generate partial_sum_diag[0-1] together
    vperm2i128      m7, m0, m0, 0x01
    vperm2i128      m6, m1, m1, 0x01
    vperm2i128      m5, m2, m2, 0x01
    vperm2i128      m4, m3, m3, 0x01

    ; start with partial_sum_hv[0-1]
    paddw           m8, m0, m1
    paddw           m9, m2, m3
    phaddw         m10, m0, m1
    phaddw         m11, m2, m3
    paddw           m8, m9
    phaddw         m10, m11
    vextracti128   xm9, m8, 1
    vextracti128  xm11, m10, 1
    paddw          xm8, xm9                 ; partial_sum_hv[1]
    phaddw        xm10, xm11                ; partial_sum_hv[0]
    vinserti128     m8, xm10, 1
    vpbroadcastd    m9, [div_table+44]
    pmaddwd         m8, m8
    pmulld          m8, m9                  ; cost6[2a-d] | cost2[a-d]

    ; create aggregates [lower half]:
    ; m9 = m0:01234567+m1:x0123456+m2:xx012345+m3:xxx01234+
    ;      m4:xxxx0123+m5:xxxxx012+m6:xxxxxx01+m7:xxxxxxx0
    ; m10=             m1:7xxxxxxx+m2:67xxxxxx+m3:567xxxxx+
    ;      m4:4567xxxx+m5:34567xxx+m6:234567xx+m7:1234567x
    ; and [upper half]:
    ; m9 = m0:xxxxxxx0+m1:xxxxxx01+m2:xxxxx012+m3:xxxx0123+
    ;      m4:xxx01234+m5:xx012345+m6:x0123456+m7:01234567
    ; m10= m0:1234567x+m1:234567xx+m2:34567xxx+m3:4567xxxx+
    ;      m4:567xxxxx+m5:67xxxxxx+m6:7xxxxxxx
    ; and then shuffle m11 [shufw_6543210x], unpcklwd, pmaddwd, pmulld, paddd

    pslldq          m9, m1, 2
    psrldq         m10, m1, 14
    pslldq         m11, m2, 4
    psrldq         m12, m2, 12
    pslldq         m13, m3, 6
    psrldq         m14, m3, 10
    paddw           m9, m11
    paddw          m10, m12
    paddw           m9, m13
    paddw          m10, m14
    pslldq         m11, m4, 8
    psrldq         m12, m4, 8
    pslldq         m13, m5, 10
    psrldq         m14, m5, 6
    paddw           m9, m11
    paddw          m10, m12
    paddw           m9, m13
    paddw          m10, m14
    pslldq         m11, m6, 12
    psrldq         m12, m6, 4
    pslldq         m13, m7, 14
    psrldq         m14, m7, 2
    paddw           m9, m11
    paddw          m10, m12
    paddw           m9, m13
    paddw          m10, m14                 ; partial_sum_diag[0/1][8-14,zero]
    vbroadcasti128 m14, [shufw_6543210x]
    vbroadcasti128 m13, [div_table+16]
    vbroadcasti128 m12, [div_table+0]
    paddw           m9, m0                  ; partial_sum_diag[0/1][0-7]
    pshufb         m10, m14
    punpckhwd      m11, m9, m10
    punpcklwd       m9, m10
    pmaddwd        m11, m11
    pmaddwd         m9, m9
    pmulld         m11, m13
    pmulld          m9, m12
    paddd           m9, m11                 ; cost0[a-d] | cost4[a-d]

    ; merge horizontally and vertically for partial_sum_alt[0-3]
    paddw          m10, m0, m1
    paddw          m11, m2, m3
    paddw          m12, m4, m5
    paddw          m13, m6, m7
    phaddw          m0, m4
    phaddw          m1, m5
    phaddw          m2, m6
    phaddw          m3, m7

    ; create aggregates [lower half]:
    ; m4 = m10:01234567+m11:x0123456+m12:xx012345+m13:xxx01234
    ; m11=              m11:7xxxxxxx+m12:67xxxxxx+m13:567xxxxx
    ; and [upper half]:
    ; m4 = m10:xxx01234+m11:xx012345+m12:x0123456+m13:01234567
    ; m11= m10:567xxxxx+m11:67xxxxxx+m12:7xxxxxxx
    ; and then pshuflw m11 3012, unpcklwd, pmaddwd, pmulld, paddd

    pslldq          m4, m11, 2
    psrldq         m11, 14
    pslldq          m5, m12, 4
    psrldq         m12, 12
    pslldq          m6, m13, 6
    psrldq         m13, 10
    paddw           m4, m10
    paddw          m11, m12
    vpbroadcastd   m12, [div_table+44]
    paddw           m5, m6
    paddw          m11, m13                 ; partial_sum_alt[3/2] right
    vbroadcasti128 m13, [div_table+32]
    paddw           m4, m5                  ; partial_sum_alt[3/2] left
    pshuflw         m5, m11, q3012
    punpckhwd       m6, m11, m4
    punpcklwd       m4, m5
    pmaddwd         m6, m6
    pmaddwd         m4, m4
    pmulld          m6, m12
    pmulld          m4, m13
    paddd           m4, m6                  ; cost7[a-d] | cost5[a-d]

    ; create aggregates [lower half]:
    ; m5 = m0:01234567+m1:x0123456+m2:xx012345+m3:xxx01234
    ; m1 =             m1:7xxxxxxx+m2:67xxxxxx+m3:567xxxxx
    ; and [upper half]:
    ; m5 = m0:xxx01234+m1:xx012345+m2:x0123456+m3:01234567
    ; m1 = m0:567xxxxx+m1:67xxxxxx+m2:7xxxxxxx
    ; and then pshuflw m1 3012, unpcklwd, pmaddwd, pmulld, paddd

    pslldq          m5, m1, 2
    psrldq          m1, 14
    pslldq          m6, m2, 4
    psrldq          m2, 12
    pslldq          m7, m3, 6
    psrldq          m3, 10
    paddw           m5, m0
    paddw           m1, m2
    paddw           m6, m7
    paddw           m1, m3                  ; partial_sum_alt[0/1] right
    paddw           m5, m6                  ; partial_sum_alt[0/1] left
    pshuflw         m0, m1, q3012
    punpckhwd       m1, m5
    punpcklwd       m5, m0
    pmaddwd         m1, m1
    pmaddwd         m5, m5
    pmulld          m1, m12
    pmulld          m5, m13
    paddd           m5, m1                  ; cost1[a-d] | cost3[a-d]

    mova           xm0, [pd_47130256+ 16]
    mova            m1, [pd_47130256]
    phaddd          m9, m8
    phaddd          m5, m4
    phaddd          m9, m5
    vpermd          m0, m9                  ; cost[0-3]
    vpermd          m1, m9                  ; cost[4-7] | cost[0-3]

    ; now find the best cost
    pmaxsd         xm2, xm0, xm1
    pshufd         xm3, xm2, q1032
    pmaxsd         xm2, xm3
    pshufd         xm3, xm2, q2301
    pmaxsd         xm2, xm3 ; best cost

    ; find the idx using minpos
    ; make everything other than the best cost negative via subtraction
    ; find the min of unsigned 16-bit ints to sort out the negative values
    psubd          xm4, xm1, xm2
    psubd          xm3, xm0, xm2
    packssdw       xm3, xm4
    phminposuw     xm3, xm3

    ; convert idx to 32-bits
    psrld          xm3, 16
    movd           eax, xm3

    ; get idx^4 complement
    vpermd          m3, m1
    psubd          xm2, xm3
    psrld          xm2, 10
    movd        [varq], xm2
    RET

%endif ; ARCH_X86_64