xref: /aosp_15_r20/external/libdav1d/src/x86/refmvs.asm (revision c09093415860a1c2373dacd84c4fde00c507cdfd)

; Copyright © 2021, VideoLAN and dav1d authors
; Copyright © 2021, Two Orioles, LLC
; All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
;
; 1. Redistributions of source code must retain the above copyright notice, this
;    list of conditions and the following disclaimer.
;
; 2. Redistributions in binary form must reproduce the above copyright notice,
;    this list of conditions and the following disclaimer in the documentation
;    and/or other materials provided with the distribution.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

%include "config.asm"
%include "ext/x86/x86inc.asm"

SECTION_RODATA 64

%macro JMP_TABLE 2-*
    %xdefine %%prefix mangle(private_prefix %+ _%1)
    %1_table:
    %xdefine %%base %1_table
    %rep %0 - 1
        dd %%prefix %+ .w%2 - %%base
        %rotate 1
    %endrep
%endmacro

%macro SAVE_TMVS_TABLE 3 ; num_entries, w, suffix
    %rep %1
        db %2*3
        db mangle(private_prefix %+ _save_tmvs_%3).write%2 - \
           mangle(private_prefix %+ _save_tmvs_%3).write1
    %endrep
%endmacro

%if ARCH_X86_64
mv_proj:       dw    0, 16384, 8192, 5461, 4096, 3276, 2730, 2340
               dw 2048,  1820, 1638, 1489, 1365, 1260, 1170, 1092
               dw 1024,   963,  910,  862,  819,  780,  744,  712
               dw  682,   655,  630,  606,  585,  564,  546,  528
splat_mv_shuf: db  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11,  0,  1,  2,  3
               db  4,  5,  6,  7,  8,  9, 10, 11,  0,  1,  2,  3,  4,  5,  6,  7
               db  8,  9, 10, 11,  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11
               db  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11,  0,  1,  2,  3
%endif
save_pack0:    db  0,  1,  2,  3,  4,  0,  1,  2,  3,  4,  0,  1,  2,  3,  4,  0
               db  1,  2,  3,  4,  0,  1,  2,  3,  4,  0,  1,  2,  3,  4,  0,  1
save_pack1:    db  2,  3,  4,  0,  1,  2,  3,  4,  0,  1,  2,  3,  4,  0,  1,  2
               db  3,  4,  0,  1,  2,  3,  4,  0,  1,  2,  3,  4,  0,  1,  2,  3
save_ref_shuf: db  0, -1, -1, -1,  1, -1, -1, -1,  8, -1, -1, -1,  9, -1, -1, -1
cond_shuf512:  db  3,  3,  3,  3,  7,  7,  7,  7,  7,  7,  7,  7,  3,  3,  3,  3
save_cond0:    db  0x80, 0x81, 0x82, 0x83, 0x89, 0x84, 0x00, 0x00
save_cond1:    db  0x84, 0x85, 0x86, 0x87, 0x88, 0x80, 0x00, 0x00
pb_128:        times 16 db 128
pq_8192:       dq 8192

save_tmvs_ssse3_table: SAVE_TMVS_TABLE 2, 16, ssse3
                       SAVE_TMVS_TABLE 4,  8, ssse3
                       SAVE_TMVS_TABLE 4,  4, ssse3
                       SAVE_TMVS_TABLE 5,  2, ssse3
                       SAVE_TMVS_TABLE 7,  1, ssse3

%if ARCH_X86_64
save_tmvs_avx2_table: SAVE_TMVS_TABLE 2, 16, avx2
                      SAVE_TMVS_TABLE 4,  8, avx2
                      SAVE_TMVS_TABLE 4,  4, avx2
                      SAVE_TMVS_TABLE 5,  2, avx2
                      SAVE_TMVS_TABLE 7,  1, avx2

save_tmvs_avx512icl_table: SAVE_TMVS_TABLE 2, 16, avx512icl
                           SAVE_TMVS_TABLE 4,  8, avx512icl
                           SAVE_TMVS_TABLE 4,  4, avx512icl
                           SAVE_TMVS_TABLE 5,  2, avx512icl
                           SAVE_TMVS_TABLE 7,  1, avx512icl

JMP_TABLE splat_mv_avx512icl, 1, 2, 4, 8, 16, 32
JMP_TABLE splat_mv_avx2,      1, 2, 4, 8, 16, 32
%endif

JMP_TABLE splat_mv_sse2,      1, 2, 4, 8, 16, 32

struc rf
    .frm_hdr:         resq 1
    .iw4:             resd 1
    .ih4:             resd 1
    .iw8:             resd 1
    .ih8:             resd 1
    .sbsz:            resd 1
    .use_rf_mvs:      resd 1
    .sign_bias:       resb 7
    .mfmv_sign:       resb 7
    .pocdiff:         resb 7
    .mfmv_ref:        resb 3
    .mfmv_ref2cur:    resd 3
    .mfmv_ref2ref:    resd 3*7
    .n_mfmvs:         resd 1
    .n_blocks:        resd 1
    .rp:              resq 1
    .rp_ref:          resq 1
    .rp_proj:         resq 1
    .rp_stride:       resq 1
    .r:               resq 1
    .n_tile_threads:  resd 1
    .n_frame_threads: resd 1
endstruc

SECTION .text

%macro movif32 2
%if ARCH_X86_32
    mov             %1, %2
%endif
%endmacro

INIT_XMM ssse3
; refmvs_temporal_block *rp, ptrdiff_t stride,
; refmvs_block **rr, uint8_t *ref_sign,
; int col_end8, int row_end8, int col_start8, int row_start8
%if ARCH_X86_64
cglobal save_tmvs, 4, 13, 11, rp, stride, rr, ref_sign, \
                             xend, yend, xstart, ystart
%define base_reg r12
%else
cglobal save_tmvs, 6, 7, 8, rp, stride, rr, ref_sign, \
                            xend, yend, xstart, ystart
    movq            m5, [ref_signq]
    lea        strided, [strided*5]
    mov        stridem, strided
    mov             r3, xstartm
    mov             r1, ystartm
 DEFINE_ARGS b, ystart, rr, cand, xend, x
%define stridemp r1m
%define m8  [base+pb_128]
%define m9  [base+save_pack0+ 0]
%define m10 [base+save_pack0+16]
%define base_reg r6
%endif
%define base base_reg-.write1
    LEA       base_reg, .write1
%if ARCH_X86_64
    movifnidn    xendd, xendm
    movifnidn    yendd, yendm
    mov        xstartd, xstartm
    mov        ystartd, ystartm
    movq            m5, [ref_signq]
%endif
    movu            m4, [base+save_ref_shuf]
    movddup         m6, [base+save_cond0]
    movddup         m7, [base+save_cond1]
%if ARCH_X86_64
    mova            m8, [base+pb_128]
    mova            m9, [base+save_pack0+ 0]
    mova           m10, [base+save_pack0+16]
%endif
    psllq           m5, 8
%if ARCH_X86_64
    lea            r9d, [xendq*5]
    lea        xstartd, [xstartq*5]
    sub          yendd, ystartd
    add        ystartd, ystartd
    lea        strideq, [strideq*5]
    sub        xstartq, r9
    add          xendd, r9d
    add            rpq, r9
 DEFINE_ARGS rp, stride, rr, x, xend, h, xstart, ystart, b, cand
%else
    lea             r0, [xendd*5]   ; xend5
    lea             r3, [r3*5]      ; xstart5
    sub             r3, r0          ; -w5
    mov            r6m, r3
%define xstartq r6m
    add          xendd, r0          ; xend6
    add            r0m, r0          ; rp+xend5
    mov          xendm, xendd
    sub             r5, r1          ; h
    add             r1, r1
    mov            r7m, r1
    mov            r5m, r5
%define hd r5mp
    jmp .loop_y_noload
%endif
.loop_y:
    movif32    ystartd, r7m
    movif32      xendd, xendm
.loop_y_noload:
    and        ystartd, 30
    mov             xq, xstartq
    mov             bq, [rrq+ystartq*gprsize]
    add        ystartd, 2
    movif32        r7m, ystartd
    lea             bq, [bq+xendq*4]
.loop_x:
%if ARCH_X86_32
%define rpq  r3
%define r10  r1
%define r10d r1
%define r11  r4
%define r11d r4
%endif
    imul         candq, xq, 0x9999  ; x / 5 * 3
    sar          candq, 16
    movzx         r10d, byte [bq+candq*8+22] ; cand_b->bs
    movu            m0, [bq+candq*8+12]      ; cand_b
    movzx         r11d, byte [base+save_tmvs_ssse3_table+r10*2+0]
    movzx         r10d, byte [base+save_tmvs_ssse3_table+r10*2+1]
    add            r10, base_reg
    add          candq, r11
    jge .calc
    movu            m1, [bq+candq*8+12]
    movzx         r11d, byte [bq+candq*8+22]
    movzx         r11d, byte [base+save_tmvs_ssse3_table+r11*2+1]
    add            r11, base_reg
.calc:
    movif32        rpq, r0m
    ; ref check
    punpckhqdq      m2, m0, m1
    pshufb          m2, m4      ; b0.ref0 b0.ref1 b1.ref0 b1.ref1 | ...
    pshufb          m3, m5, m2  ; ref > 0 && res_sign[ref - 1]
    ; mv check
    punpcklqdq      m2, m0, m1  ; b0.mv0 b0.mv1 b1.mv0 b1.mv1 | ...
    pabsw           m2, m2
    psrlw           m2, 12      ; (abs(mv.x) | abs(mv.y)) < 4096
    ; res
    pcmpgtd         m3, m2
    pshufd          m2, m3, q2301
    pand            m3, m6      ; b0c0 b0c1 b1c0 b1c1 | ...
    pand            m2, m7      ; b0c1 b0c0 b1c1 b1c0 | ...
    por             m3, m2      ; b0.shuf b1.shuf | ...
    pxor            m3, m8      ; if cond0|cond1 == 0 => zero out
    pshufb          m0, m3
    pshufb          m1, m3
    call           r10
    jge .next_line
    pshufd          m0, m1, q3232
    call           r11
    jl .loop_x
.next_line:
    add            rpq, stridemp
    movif32        r0m, rpq
    dec             hd
    jg .loop_y
    RET
.write1:
    movd    [rpq+xq+0], m0
    psrlq           m0, 8
    movd    [rpq+xq+1], m0
    add             xq, 5*1
    ret
.write2:
    movq    [rpq+xq+0], m0
    psrlq           m0, 8
    movd    [rpq+xq+6], m0
    add             xq, 5*2
    ret
.write4:
    pshufb          m0, m9
    movu   [rpq+xq+ 0], m0
    psrlq           m0, 8
    movd   [rpq+xq+16], m0
    add             xq, 5*4
    ret
.write8:
    pshufb          m2, m0, m9
    movu   [rpq+xq+ 0], m2
    pshufb          m0, m10
    movu   [rpq+xq+16], m0
    psrldq          m2, 2
    movq   [rpq+xq+32], m2
    add             xq, 5*8
    ret
.write16:
    pshufb          m2, m0, m9
    movu   [rpq+xq+ 0], m2
    pshufb          m0, m10
    movu   [rpq+xq+16], m0
    shufps          m2, m0, q1032
    movu   [rpq+xq+48], m2
    shufps          m2, m0, q2121
    movu   [rpq+xq+32], m2
    shufps          m0, m2, q1032
    movu   [rpq+xq+64], m0
    add             xq, 5*16
    ret

INIT_XMM sse2
; refmvs_block **rr, refmvs_block *a, int bx4, int bw4, int bh4
cglobal splat_mv, 4, 5, 3, rr, a, bx4, bw4, bh4
    add           bx4d, bw4d
    tzcnt         bw4d, bw4d
    mova            m2, [aq]
    LEA             aq, splat_mv_sse2_table
    lea           bx4q, [bx4q*3-32]
    movsxd        bw4q, [aq+bw4q*4]
    movifnidn     bh4d, bh4m
    pshufd          m0, m2, q0210
    pshufd          m1, m2, q1021
    pshufd          m2, m2, q2102
    add           bw4q, aq
.loop:
    mov             aq, [rrq]
    add            rrq, gprsize
    lea             aq, [aq+bx4q*4]
    jmp           bw4q
.w32:
    mova    [aq-16*16], m0
    mova    [aq-16*15], m1
    mova    [aq-16*14], m2
    mova    [aq-16*13], m0
    mova    [aq-16*12], m1
    mova    [aq-16*11], m2
    mova    [aq-16*10], m0
    mova    [aq-16* 9], m1
    mova    [aq-16* 8], m2
    mova    [aq-16* 7], m0
    mova    [aq-16* 6], m1
    mova    [aq-16* 5], m2
.w16:
    mova    [aq-16* 4], m0
    mova    [aq-16* 3], m1
    mova    [aq-16* 2], m2
    mova    [aq-16* 1], m0
    mova    [aq+16* 0], m1
    mova    [aq+16* 1], m2
.w8:
    mova    [aq+16* 2], m0
    mova    [aq+16* 3], m1
    mova    [aq+16* 4], m2
.w4:
    mova    [aq+16* 5], m0
    mova    [aq+16* 6], m1
    mova    [aq+16* 7], m2
    dec           bh4d
    jg .loop
    RET
.w2:
    movu      [aq+104], m0
    movq      [aq+120], m1
    dec           bh4d
    jg .loop
    RET
.w1:
    movq      [aq+116], m0
    movd      [aq+124], m2
    dec           bh4d
    jg .loop
    RET

%if ARCH_X86_64
INIT_XMM sse4
; refmvs_frame *rf, int tile_row_idx,
; int col_start8, int col_end8, int row_start8, int row_end8
cglobal load_tmvs, 6, 15, 4, -0x50, rf, tridx, xstart, xend, ystart, yend, \
                                    stride, rp_proj, roff, troff, \
                                    xendi, xstarti, iw8, ih8, dst
    xor           r14d, r14d
    cmp dword [rfq+rf.n_tile_threads], 1
    mov           ih8d, [rfq+rf.ih8]
    mov           iw8d, [rfq+rf.iw8]
    mov        xstartd, xstartd
    mov          xendd, xendd
    cmove       tridxd, r14d
    lea       xstartid, [xstartq-8]
    lea         xendid, [xendq+8]
    mov        strideq, [rfq+rf.rp_stride]
    mov       rp_projq, [rfq+rf.rp_proj]
    cmp           ih8d, yendd
    mov     [rsp+0x30], strideq
    cmovs        yendd, ih8d
    test      xstartid, xstartid
    cmovs     xstartid, r14d
    cmp           iw8d, xendid
    cmovs       xendid, iw8d
    mov         troffq, strideq
    shl         troffq, 4
    imul        troffq, tridxq
    mov           dstd, ystartd
    and           dstd, 15
    imul          dstq, strideq
    add           dstq, troffq      ; (16 * tridx + (ystart & 15)) * stride
    lea           dstq, [dstq*5]
    add           dstq, rp_projq
    lea         troffq, [troffq*5]  ; 16 * tridx * stride * 5
    lea           r13d, [xendq*5]
    lea            r12, [strideq*5]
 DEFINE_ARGS rf, w5, xstart, xend, ystart, yend, h, x5, \
             _, troff, xendi, xstarti, stride5, _, dst
    lea            w5d, [xstartq*5]
    add             r7, troffq      ; rp_proj + tile_row_offset
    mov             hd, yendd
    mov     [rsp+0x28], r7
    add           dstq, r13
    sub            w5q, r13
    sub             hd, ystartd
.init_xloop_start:
    mov            x5q, w5q
    test           w5b, 1
    jz .init_2blk
    mov dword [dstq+x5q], 0x80008000
    add            x5q, 5
    jz .init_next_row
.init_2blk:
    mov dword [dstq+x5q+0], 0x80008000
    mov dword [dstq+x5q+5], 0x80008000
    add            x5q, 10
    jl .init_2blk
.init_next_row:
    add           dstq, stride5q
    dec             hd
    jg .init_xloop_start
 DEFINE_ARGS rf, _, xstart, xend, ystart, yend, n7, stride, \
             _, _, xendi, xstarti, stride5, _, n
    mov           r13d, [rfq+rf.n_mfmvs]
    test          r13d, r13d
    jz .ret
    mov     [rsp+0x0c], r13d
    mov        strideq, [rsp+0x30]
    movddup         m3, [pq_8192]
    mov            r9d, ystartd
    mov     [rsp+0x38], yendd
    mov     [rsp+0x20], xstartid
    xor             nd, nd
    xor            n7d, n7d
    imul            r9, strideq     ; ystart * stride
    mov     [rsp+0x48], rfq
    mov     [rsp+0x18], stride5q
    lea             r7, [r9*5]
    mov     [rsp+0x24], ystartd
    mov     [rsp+0x00], r7
.nloop:
 DEFINE_ARGS y, off, xstart, xend, ystart, rf, n7, refsign, \
             ref, rp_ref, xendi, xstarti, _, _, n
    mov            rfq, [rsp+0x48]
    mov           refd, [rfq+rf.mfmv_ref2cur+nq*4]
    cmp           refd, 0x80000000
    je .next_n
    mov     [rsp+0x40], refd
    mov           offq, [rsp+0x00]          ; ystart * stride * 5
    movzx         refd, byte [rfq+rf.mfmv_ref+nq]
    lea       refsignq, [refq-4]
    mov        rp_refq, [rfq+rf.rp_ref]
    movq            m2, refsignq
    add           offq, [rp_refq+refq*8]    ; r = rp_ref[ref] + row_offset
    mov     [rsp+0x14], nd
    mov             yd, ystartd
.yloop:
    mov           r11d, [rsp+0x24]          ; ystart
    mov           r12d, [rsp+0x38]          ; yend
    mov           r14d, yd
    and           r14d, ~7                  ; y_sb_align
    cmp           r11d, r14d
    cmovs         r11d, r14d                ; imax(y_sb_align, ystart)
    mov     [rsp+0x44], r11d                ; y_proj_start
    add           r14d, 8
    cmp           r12d, r14d
    cmovs         r14d, r12d                ; imin(y_sb_align + 8, yend)
    mov     [rsp+0x3c], r14d                ; y_proj_end
 DEFINE_ARGS y, src, xstart, xend, frac, rf, n7, mv, \
             ref, x, xendi, mvx, mvy, rb, ref2ref
    mov             xd, [rsp+0x20] ; xstarti
.xloop:
    lea            rbd, [xq*5]
    add            rbq, srcq
    movsx         refd, byte [rbq+4]
    test          refd, refd
    jz .next_x_bad_ref
    mov            rfq, [rsp+0x48]
    lea       ref2refd, [(rf.mfmv_ref2ref/4)+n7q+refq-1]
    mov       ref2refd, [rfq+ref2refq*4]    ; rf->mfmv_ref2ref[n][b_ref-1]
    test      ref2refd, ref2refd
    jz .next_x_bad_ref
    lea          fracq, [mv_proj]
    movzx        fracd, word [fracq+ref2refq*2]
    mov            mvd, [rbq]
    imul         fracd, [rsp+0x40] ; ref2cur
    pmovsxwq        m0, [rbq]
    movd            m1, fracd
    punpcklqdq      m1, m1
    pmuldq          m0, m1          ; mv * frac
    pshufd          m1, m0, q3311
    paddd           m0, m3
    paddd           m0, m1
    psrad           m0, 14          ; offset = (xy + (xy >> 31) + 8192) >> 14
    pabsd           m1, m0
    packssdw        m0, m0
    psrld           m1, 6
    packuswb        m1, m1
    pxor            m0, m2          ; offset ^ ref_sign
    psignd          m1, m0          ; apply_sign(abs(offset) >> 6, offset ^ refsign)
    movq          mvxq, m1
    lea           mvyd, [mvxq+yq]   ; ypos
    sar           mvxq, 32
 DEFINE_ARGS y, src, xstart, xend, _, _, n7, mv, \
             ref, x, xendi, mvx, ypos, rb, ref2ref
    cmp          yposd, [rsp+0x44] ; y_proj_start
    jl .next_x_bad_pos_y
    cmp          yposd, [rsp+0x3c] ; y_proj_end
    jge .next_x_bad_pos_y
    and          yposd, 15
    add           mvxq, xq          ; xpos
    imul         yposq, [rsp+0x30]  ; pos = (ypos & 15) * stride
 DEFINE_ARGS y, src, xstart, xend, dst, _, n7, mv, \
             ref, x, xendi, xpos, pos, rb, ref2ref
    mov           dstq, [rsp+0x28]  ; dst = rp_proj + tile_row_offset
    add           posq, xposq       ; pos += xpos
    lea           posq, [posq*5]
    add           dstq, posq        ; dst += pos5
    jmp .write_loop_entry
.write_loop:
    add            rbq, 5
    cmp           refb, byte [rbq+4]
    jne .xloop
    cmp            mvd, [rbq]
    jne .xloop
    add           dstq, 5
    inc          xposd
.write_loop_entry:
    mov           r12d, xd
    and           r12d, ~7
    lea            r5d, [r12-8]
    cmp            r5d, xstartd
    cmovs          r5d, xstartd     ; x_proj_start
    cmp          xposd, r5d
    jl .next_xpos
    add           r12d, 16
    cmp          xendd, r12d
    cmovs         r12d, xendd       ; x_proj_end
    cmp          xposd, r12d
    jge .next_xpos
    mov       [dstq+0], mvd
    mov  byte [dstq+4], ref2refb
.next_xpos:
    inc             xd
    cmp             xd, xendid
    jl .write_loop
.next_y:
 DEFINE_ARGS y, src, xstart, xend, ystart, _, n7, _, _, x, xendi, _, _, _, n
    add           srcq, [rsp+0x18] ; stride5
    inc             yd
    cmp             yd, [rsp+0x38] ; yend
    jne .yloop
    mov             nd, [rsp+0x14]
    mov        ystartd, [rsp+0x24]
.next_n:
    add            n7d, 7
    inc             nd
    cmp             nd, [rsp+0x0c] ; n_mfmvs
    jne .nloop
.ret:
    RET
.next_x:
 DEFINE_ARGS y, src, xstart, xend, _, _, n7, mv, ref, x, xendi, _, _, rb, _
    add            rbq, 5
    cmp           refb, byte [rbq+4]
    jne .xloop
    cmp            mvd, [rbq]
    jne .xloop
.next_x_bad_pos_y:
    inc             xd
    cmp             xd, xendid
    jl .next_x
    jmp .next_y
.next_x_bad_ref:
    inc             xd
    cmp             xd, xendid
    jl .xloop
    jmp .next_y

INIT_YMM avx2
; refmvs_temporal_block *rp, ptrdiff_t stride,
; refmvs_block **rr, uint8_t *ref_sign,
; int col_end8, int row_end8, int col_start8, int row_start8
cglobal save_tmvs, 4, 13, 10, rp, stride, rr, ref_sign, \
                              xend, yend, xstart, ystart
%define base r12-.write1
    lea            r12, [.write1]
    movifnidn    xendd, xendm
    movifnidn    yendd, yendm
    mov        xstartd, xstartm
    mov        ystartd, ystartm
    vpbroadcastq    m4, [ref_signq]
    vpbroadcastq    m3, [base+save_ref_shuf+8]
    vpbroadcastq    m5, [base+save_cond0]
    vpbroadcastq    m6, [base+save_cond1]
    vpbroadcastd    m7, [base+pb_128]
    mova            m8, [base+save_pack0]
    mova            m9, [base+save_pack1]
    psllq           m4, 8
    lea            r9d, [xendq*5]
    lea        xstartd, [xstartq*5]
    sub          yendd, ystartd
    add        ystartd, ystartd
    lea        strideq, [strideq*5]
    sub        xstartq, r9
    add          xendd, r9d
    add            rpq, r9
 DEFINE_ARGS rp, stride, rr, x, xend, h, xstart, ystart, b, cand
.loop_y:
    and        ystartd, 30
    mov             xq, xstartq
    mov             bq, [rrq+ystartq*8]
    add        ystartd, 2
    lea             bq, [bq+xendq*4]
.loop_x:
    imul         candq, xq, 0x9999
    sar          candq, 16                   ; x / 5 * 3
    movzx         r10d, byte [bq+candq*8+22] ; cand_b->bs
    movu           xm0, [bq+candq*8+12]      ; cand_b
    movzx         r11d, byte [base+save_tmvs_avx2_table+r10*2+0]
    movzx         r10d, byte [base+save_tmvs_avx2_table+r10*2+1]
    add            r10, r12
    add          candq, r11
    jge .calc
    vinserti128     m0, [bq+candq*8+12], 1
    movzx         r11d, byte [bq+candq*8+22]
    movzx         r11d, byte [base+save_tmvs_avx2_table+r11*2+1]
    add            r11, r12
.calc:
    pshufb          m1, m0, m3
    pabsw           m2, m0
    pshufb          m1, m4, m1  ; ref > 0 && res_sign[ref - 1]
    psrlw           m2, 12      ; (abs(mv.x) | abs(mv.y)) < 4096
    pcmpgtd         m1, m2
    pshufd          m2, m1, q2301
    pand            m1, m5      ; b0.cond0 b1.cond0
    pand            m2, m6      ; b0.cond1 b1.cond1
    por             m1, m2      ; b0.shuf b1.shuf
    pxor            m1, m7      ; if cond0|cond1 == 0 => zero out
    pshufb          m0, m1
    call           r10
    jge .next_line
    vextracti128   xm0, m0, 1
    call           r11
    jl .loop_x
.next_line:
    add            rpq, strideq
    dec             hd
    jg .loop_y
    RET
.write1:
    movd   [rpq+xq+ 0], xm0
    pextrb [rpq+xq+ 4], xm0, 4
    add             xq, 5*1
    ret
.write2:
    movq    [rpq+xq+0], xm0
    psrlq          xm1, xm0, 8
    movd    [rpq+xq+6], xm1
    add             xq, 5*2
    ret
.write4:
    pshufb         xm1, xm0, xm8
    movu   [rpq+xq+ 0], xm1
    psrlq          xm1, 8
    movd   [rpq+xq+16], xm1
    add             xq, 5*4
    ret
.write8:
    vinserti128     m1, m0, xm0, 1
    pshufb          m1, m8
    movu   [rpq+xq+ 0], m1
    psrldq         xm1, 2
    movq   [rpq+xq+32], xm1
    add             xq, 5*8
    ret
.write16:
    vinserti128     m1, m0, xm0, 1
    pshufb          m2, m1, m8
    movu   [rpq+xq+ 0], m2
    pshufb          m1, m9
    movu   [rpq+xq+32], m1
    shufps         xm2, xm1, q1021
    movu   [rpq+xq+64], xm2
    add             xq, 5*16
    ret

cglobal splat_mv, 4, 5, 3, rr, a, bx4, bw4, bh4
    add           bx4d, bw4d
    tzcnt         bw4d, bw4d
    vbroadcasti128  m0, [aq]
    lea             aq, [splat_mv_avx2_table]
    lea           bx4q, [bx4q*3-32]
    movsxd        bw4q, [aq+bw4q*4]
    pshufb          m0, [splat_mv_shuf]
    movifnidn     bh4d, bh4m
    pshufd          m1, m0, q2102
    pshufd          m2, m0, q1021
    add           bw4q, aq
.loop:
    mov             aq, [rrq]
    add            rrq, gprsize
    lea             aq, [aq+bx4q*4]
    jmp           bw4q
.w32:
    mova     [aq-32*8], m0
    mova     [aq-32*7], m1
    mova     [aq-32*6], m2
    mova     [aq-32*5], m0
    mova     [aq-32*4], m1
    mova     [aq-32*3], m2
.w16:
    mova     [aq-32*2], m0
    mova     [aq-32*1], m1
    mova     [aq+32*0], m2
.w8:
    mova     [aq+32*1], m0
    mova     [aq+32*2], m1
    mova     [aq+32*3], m2
    dec           bh4d
    jg .loop
    RET
.w4:
    movu      [aq+ 80], m0
    mova      [aq+112], xm1
    dec           bh4d
    jg .loop
    RET
.w2:
    movu      [aq+104], xm0
    movq      [aq+120], xm2
    dec           bh4d
    jg .loop
    RET
.w1:
    movq      [aq+116], xm0
    movd      [aq+124], xm1
    dec           bh4d
    jg .loop
    RET

INIT_ZMM avx512icl
; refmvs_temporal_block *rp, ptrdiff_t stride,
; refmvs_block **rr, uint8_t *ref_sign,
; int col_end8, int row_end8, int col_start8, int row_start8
cglobal save_tmvs, 4, 15, 10, rp, stride, rr, ref_sign, \
                              xend, yend, xstart, ystart
%define base r14-.write1
    lea            r14, [.write1]
    movifnidn    xendd, xendm
    movifnidn    yendd, yendm
    mov        xstartd, xstartm
    mov        ystartd, ystartm
    psllq           m4, [ref_signq]{bcstq}, 8
    vpbroadcastq    m3, [base+save_ref_shuf+8]
    vbroadcasti32x4 m5, [base+cond_shuf512]
    vbroadcasti32x4 m6, [base+save_cond0]
    vpbroadcastd    m7, [base+pb_128]
    mova            m8, [base+save_pack0]
    movu           xm9, [base+save_pack0+4]
    lea            r9d, [xendq*5]
    lea        xstartd, [xstartq*5]
    sub          yendd, ystartd
    add        ystartd, ystartd
    lea        strideq, [strideq*5]
    sub        xstartq, r9
    add          xendd, r9d
    add            rpq, r9
    mov           r10d, 0x1f
    kmovb           k2, r10d
 DEFINE_ARGS rp, stride, rr, x, xend, h, xstart, ystart, b, cand
.loop_y:
    and        ystartd, 30
    mov             xq, xstartq
    mov             bq, [rrq+ystartq*8]
    add        ystartd, 2
    lea             bq, [bq+xendq*4]
.loop_x:
    imul         candq, xq, 0x9999
    sar          candq, 16                   ; x / 5 * 3
    movzx         r10d, byte [bq+candq*8+22] ; cand_b->bs
    movu           xm0, [bq+candq*8+12]      ; cand_b
    movzx         r11d, byte [base+save_tmvs_avx512icl_table+r10*2+0]
    movzx         r10d, byte [base+save_tmvs_avx512icl_table+r10*2+1]
    add            r10, r14
    add          candq, r11
    jge .calc
    movzx         r11d, byte [bq+candq*8+22]
    vinserti32x4   ym0, [bq+candq*8+12], 1
    movzx         r12d, byte [base+save_tmvs_avx512icl_table+r11*2+0]
    movzx         r11d, byte [base+save_tmvs_avx512icl_table+r11*2+1]
    add            r11, r14
    add          candq, r12
    jge .calc
    movzx         r12d, byte [bq+candq*8+22]
    vinserti32x4    m0, [bq+candq*8+12], 2
    movzx         r13d, byte [base+save_tmvs_avx512icl_table+r12*2+0]
    movzx         r12d, byte [base+save_tmvs_avx512icl_table+r12*2+1]
    add            r12, r14
    add          candq, r13
    jge .calc
    vinserti32x4    m0, [bq+candq*8+12], 3
    movzx         r13d, byte [bq+candq*8+22]
    movzx         r13d, byte [base+save_tmvs_avx512icl_table+r13*2+1]
    add            r13, r14
.calc:
    pshufb          m1, m0, m3
    pabsw           m2, m0
    pshufb          m1, m4, m1      ; ref > 0 && res_sign[ref - 1]
    psrlw           m2, 12          ; (abs(mv.x) | abs(mv.y)) < 4096
    psubd           m2, m1
    pshufb          m2, m5           ; c0 c1 c1 c0
    pand            m2, m6
    punpckhqdq      m1, m2, m2
    vpternlogd      m1, m2, m7, 0x56 ; (c0shuf | c1shuf) ^ 0x80
    pshufb          m2, m0, m1
    mova           xm0, xm2
    call           r10
    jge .next_line
    vextracti32x4  xm0, m2, 1
    call           r11
    jge .next_line
    vextracti32x4  xm0, m2, 2
    call           r12
    jge .next_line
    vextracti32x4  xm0, m2, 3
    call           r13
    jl .loop_x
.next_line:
    add            rpq, strideq
    dec             hd
    jg .loop_y
    RET
.write1:
    vmovdqu8 [rpq+xq]{k2}, xm0
    add             xq, 5*1
    ret
.write2:
    pshufb         xm0, xm8
    vmovdqu16 [rpq+xq]{k2}, xm0
    add             xq, 5*2
    ret
.write4:
    vpermb         ym0, ym8, ym0
    vmovdqu32 [rpq+xq]{k2}, ym0
    add             xq, 5*4
    ret
.write8:
    vpermb          m0, m8, m0
    vmovdqu64 [rpq+xq]{k2}, m0
    add             xq, 5*8
    ret
.write16:
    vpermb          m1, m8, m0
    movu   [rpq+xq+ 0], m1
    pshufb         xm0, xm9
    movu   [rpq+xq+64], xm0
    add             xq, 5*16
    ret

INIT_ZMM avx512icl
cglobal splat_mv, 4, 7, 3, rr, a, bx4, bw4, bh4
    vbroadcasti32x4    m0, [aq]
    lea                r1, [splat_mv_avx512icl_table]
    tzcnt            bw4d, bw4d
    lea              bx4d, [bx4q*3]
    pshufb             m0, [splat_mv_shuf]
    movsxd           bw4q, [r1+bw4q*4]
    mov               r6d, bh4m
    add              bw4q, r1
    lea               rrq, [rrq+r6*8]
    mov               r1d, 0x3f
    neg                r6
    kmovb              k1, r1d
    jmp              bw4q
.w1:
    mov                r1, [rrq+r6*8]
    vmovdqu16 [r1+bx4q*4]{k1}, xm0
    inc                r6
    jl .w1
    RET
.w2:
    mov                r1, [rrq+r6*8]
    vmovdqu32 [r1+bx4q*4]{k1}, ym0
    inc                r6
    jl .w2
    RET
.w4:
    mov                r1, [rrq+r6*8]
    vmovdqu64 [r1+bx4q*4]{k1}, m0
    inc                r6
    jl .w4
    RET
.w8:
    pshufd            ym1, ym0, q1021
.w8_loop:
    mov                r1, [rrq+r6*8+0]
    mov                r3, [rrq+r6*8+8]
    movu   [r1+bx4q*4+ 0], m0
    mova   [r1+bx4q*4+64], ym1
    movu   [r3+bx4q*4+ 0], m0
    mova   [r3+bx4q*4+64], ym1
    add                r6, 2
    jl .w8_loop
    RET
.w16:
    pshufd             m1, m0, q1021
    pshufd             m2, m0, q2102
.w16_loop:
    mov                r1, [rrq+r6*8+0]
    mov                r3, [rrq+r6*8+8]
    mova [r1+bx4q*4+64*0], m0
    mova [r1+bx4q*4+64*1], m1
    mova [r1+bx4q*4+64*2], m2
    mova [r3+bx4q*4+64*0], m0
    mova [r3+bx4q*4+64*1], m1
    mova [r3+bx4q*4+64*2], m2
    add                r6, 2
    jl .w16_loop
    RET
.w32:
    pshufd             m1, m0, q1021
    pshufd             m2, m0, q2102
.w32_loop:
    mov                r1, [rrq+r6*8]
    lea                r1, [r1+bx4q*4]
    mova        [r1+64*0], m0
    mova        [r1+64*1], m1
    mova        [r1+64*2], m2
    mova        [r1+64*3], m0
    mova        [r1+64*4], m1
    mova        [r1+64*5], m2
    inc                r6
    jl .w32_loop
    RET
%endif ; ARCH_X86_64