xref: /aosp_15_r20/external/libdav1d/src/arm/64/looprestoration16.S (revision c09093415860a1c2373dacd84c4fde00c507cdfd)

/*
 * Copyright © 2018, VideoLAN and dav1d authors
 * Copyright © 2020, Martin Storsjo
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 *    list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "src/arm/asm.S"
#include "util.S"

const right_ext_mask_buf
        .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
        .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
        .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
        .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
        .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
        .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
right_ext_mask:
        .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
        .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
        .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
        .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
        .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
        .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
endconst

// void dav1d_wiener_filter7_16bpc_neon(pixel *p, const ptrdiff_t p_stride,
//                                      const pixel (*left)[4], const pixel *lpf,
//                                      const int w, int h,
//                                      const int16_t filter[2][8],
//                                      const enum LrEdgeFlags edges,
//                                      const int bitdepth_max);
function wiener_filter7_16bpc_neon, export=1
        ldr             w8,  [sp]
        AARCH64_SIGN_LINK_REGISTER
        stp             x29, x30, [sp, #-32]!
        stp             d8,  d9,  [sp, #16]
        mov             x29, sp
        ld1             {v0.8h, v1.8h},  [x6]
        tst             w7,  #4               // LR_HAVE_TOP
        sub_sp          384*2*6

        dup             v28.8h,  w8           // bitdepth_max
        clz             w8,  w8
        movi            v30.4s,  #1
        sub             w10, w8,  #38         // -(bitdepth + 6)
        sub             w11, w8,  #11         // round_bits_v
        sub             w8,  w8,  #25         // -round_bits_h
        neg             w10, w10              // bitdepth + 6
        neg             w11, w11              // -round_bits_v
        dup             v2.4s,   w10
        dup             v29.4s,  w8           // -round_bits_h
        dup             v27.4s,  w11          // -round_bits_v
        movi            v31.8h,  #0x20, lsl #8  // 1 << 13 = 8192
        ushl            v30.4s,  v30.4s,  v2.4s // 1 << (bitdepth + 6)

        zip1            v0.2d,   v0.2d,   v1.2d // move vertical coeffs to v0.h[4-7], freeing up v1

        // x9  - t6
        // x10 - t5
        // x11 - t4
        // x12 - t3
        // x13 - t2
        // x14 - t1
        // x15 - t0
        mov             x14, sp               // t1
        b.eq            L(no_top_7)

        mov             x16, x2               // backup left
        mov             x2,  #0
        bl              wiener_filter7_h_16bpc_neon
        add             x3,  x3,  x1          // lpf += stride
        mov             x9,  x14              // t6
        mov             x10, x14              // t5
        add             x14, x14, #384*2      // t1 += 384*2
        bl              wiener_filter7_h_16bpc_neon
        add             x3,  x3,  x1,  lsl #2
        add             x3,  x3,  x1          // lpf += stride*5
        mov             x11, x14              // t4
        add             x14, x14, #384*2      // t1 += 384*2
        mov             x2,  x16              // left
        mov             x16, x3               // backup lpf
        mov             x3,  x0               // lpf = p
        bl              wiener_filter7_h_16bpc_neon
        subs            w5,  w5,  #1          // h--
        mov             x12, x14              // t3
        mov             x13, x14              // t2
        b.eq            L(v1_7)
        add             x3,  x3,  x1          // src += stride
        add             x14, x14, #384*2      // t1 += 384*2
        bl              wiener_filter7_h_16bpc_neon
        mov             x13, x14              // t2
        subs            w5,  w5,  #1          // h--
        b.eq            L(v2_7)
        add             x3,  x3,  x1          // src += stride
        add             x14, x14, #384*2      // t1 += 384*2
        bl              wiener_filter7_h_16bpc_neon
        subs            w5,  w5,  #1          // h--
        b.eq            L(v3_7)
        add             x3,  x3,  x1          // src += stride

L(main_7):
        add             x15, x14, #384*2      // t0 = t1 + 384*2
L(main_loop_7):
        bl              wiener_filter7_hv_16bpc_neon
        subs            w5,  w5,  #1          // h--
        b.ne            L(main_loop_7)
        tst             w7,  #8 // LR_HAVE_BOTTOM
        b.eq            L(v3_7)

        mov             x3,  x16              // restore lpf
        mov             x2,  #0               // left = NULL
        bl              wiener_filter7_hv_16bpc_neon
        bl              wiener_filter7_hv_16bpc_neon
L(v1_7):
        bl              wiener_filter7_v_16bpc_neon

        mov             sp,  x29
        ldp             d8,  d9,  [sp, #16]
        ldp             x29, x30, [sp], #32
        AARCH64_VALIDATE_LINK_REGISTER
        ret

L(no_top_7):
        add             x3,  x3,  x1,  lsl #2
        add             x16, x3,  x1,  lsl #1 // lpf += stride*6, backup
        mov             x3,  x0               // lpf = p

        bl              wiener_filter7_h_16bpc_neon
        subs            w5,  w5,  #1          // h--
        mov             x9,  x14              // t6
        mov             x10, x14              // t5
        mov             x11, x14              // t4
        mov             x12, x14              // t3
        mov             x13, x14              // t2
        b.eq            L(v1_7)
        add             x3,  x3,  x1          // src += p_stride
        add             x14, x14, #384*2      // t1 += 384*2
        bl              wiener_filter7_h_16bpc_neon
        subs            w5,  w5,  #1          // h--
        mov             x13, x14              // t2
        b.eq            L(v2_7)
        add             x3,  x3,  x1          // src += p_stride
        add             x14, x14, #384*2      // t1 += 384*2
        bl              wiener_filter7_h_16bpc_neon
        subs            w5,  w5,  #1          // h--
        b.eq            L(v3_7)
        add             x3,  x3,  x1          // src += p_stride
        add             x15, x14, #384*2      // t0 = t1 + 384*2
        bl              wiener_filter7_hv_16bpc_neon
        subs            w5,  w5,  #1          // h--
        b.eq            L(v3_7)
        add             x15, x15, #384*2*4    // t0 += 384*2*4
        bl              wiener_filter7_hv_16bpc_neon
        subs            w5,  w5,  #1          // h--
        b.ne            L(main_7)
L(v3_7):
        bl              wiener_filter7_v_16bpc_neon
L(v2_7):
        bl              wiener_filter7_v_16bpc_neon
        b               L(v1_7)
endfunc


function wiener_filter7_h_16bpc_neon
        stp             x3,  x4,  [sp, #-32]!
        str             x14,      [sp, #16]

        // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
        tst             w7,  #1 // LR_HAVE_LEFT
        b.eq            1f
        // LR_HAVE_LEFT
        cbnz            x2,  0f
        // left == NULL
        sub             x3,  x3,  #6
        ld1             {v2.8h, v3.8h}, [x3], #32
        b               2f

0:
        // LR_HAVE_LEFT, left != NULL
        ld1             {v2.8h, v3.8h}, [x3], #32
        ld1             {v4.d}[1], [x2], #8
        // Move x3 back to account for the last 3 pixels we loaded earlier,
        // which we'll shift out.
        sub             x3,  x3,  #6
        ext             v3.16b,  v2.16b,  v3.16b,  #10
        ext             v2.16b,  v4.16b,  v2.16b,  #10
        b               2f

1:
        ld1             {v2.8h, v3.8h}, [x3], #32
        // !LR_HAVE_LEFT, fill v4 with the leftmost pixel
        // and shift v3 to have 3x the first pixel at the front.
        dup             v4.8h,  v2.h[0]
        // Move x3 back to account for the last 3 pixels we loaded before,
        // which we shifted out.
        sub             x3,  x3,  #6
        ext             v3.16b,  v2.16b,  v3.16b,  #10
        ext             v2.16b,  v4.16b,  v2.16b,  #10

2:
        ld1             {v4.8h}, [x3], #16

        tst             w7,  #2 // LR_HAVE_RIGHT
        b.ne            4f

3:      // !LR_HAVE_RIGHT

        // Check whether we need to pad the right edge
        cmp             w4,  #19
        b.ge            4f   // If w >= 19, all used input pixels are valid

        // 1 <= w < 19, w+3 pixels valid in v2-v4. For w>=9,
        // this ends up called again; it's not strictly needed in those
        // cases (we pad enough here), but keeping the code as simple as possible.

        // The padding pixel is v2/3/4.h[w+2]. x3 points at the next input, ie
        // v2/3/4.h[24]. Thus read from x3[w-22] to find the padding pixel.
        sub             w17, w4,  #22
        // Insert padding in v2/3/4.h[w+3] onwards; fuse the +3 (*2) into the
        // buffer pointer.
        movrel          x6,  right_ext_mask, -6
        ldr             h26, [x3,  w17, sxtw #1]
        sub             x6,  x6,  w4,  uxtw #1
        dup             v26.8h,  v26.h[0]
        ld1             {v23.16b, v24.16b, v25.16b}, [x6]

        bit             v2.16b,  v26.16b, v23.16b
        bit             v3.16b,  v26.16b, v24.16b
        bit             v4.16b,  v26.16b, v25.16b

4:      // Loop horizontally
        // Interleaving the mul/mla chains actually hurts performance
        // significantly on Cortex A53, thus keeping mul/mla tightly
        // chained like this.
        ext             v17.16b, v2.16b,  v3.16b, #4
        ext             v19.16b, v2.16b,  v3.16b, #8
        ext             v16.16b, v2.16b,  v3.16b, #2
        ext             v20.16b, v2.16b,  v3.16b, #10
        ext             v21.16b, v2.16b,  v3.16b, #12
        ext             v18.16b, v2.16b,  v3.16b, #6
        add             v19.8h,  v19.8h,  v17.8h
        add             v20.8h,  v20.8h,  v16.8h
        add             v21.8h,  v21.8h,  v2.8h
        smull           v6.4s,   v18.4h,  v0.h[3]
        smlal           v6.4s,   v19.4h,  v0.h[2]
        smlal           v6.4s,   v20.4h,  v0.h[1]
        smlal           v6.4s,   v21.4h,  v0.h[0]
        smull2          v7.4s,   v18.8h,  v0.h[3]
        smlal2          v7.4s,   v19.8h,  v0.h[2]
        smlal2          v7.4s,   v20.8h,  v0.h[1]
        smlal2          v7.4s,   v21.8h,  v0.h[0]

        ext             v17.16b, v3.16b,  v4.16b, #4
        ext             v19.16b, v3.16b,  v4.16b, #8
        ext             v16.16b, v3.16b,  v4.16b, #2
        ext             v20.16b, v3.16b,  v4.16b, #10
        ext             v21.16b, v3.16b,  v4.16b, #12
        ext             v18.16b, v3.16b,  v4.16b, #6

        add             v19.8h,  v19.8h,  v17.8h
        add             v20.8h,  v20.8h,  v16.8h
        add             v21.8h,  v21.8h,  v3.8h
        smull           v16.4s,  v18.4h,  v0.h[3]
        smlal           v16.4s,  v19.4h,  v0.h[2]
        smlal           v16.4s,  v20.4h,  v0.h[1]
        smlal           v16.4s,  v21.4h,  v0.h[0]
        smull2          v17.4s,  v18.8h,  v0.h[3]
        smlal2          v17.4s,  v19.8h,  v0.h[2]
        smlal2          v17.4s,  v20.8h,  v0.h[1]
        smlal2          v17.4s,  v21.8h,  v0.h[0]

        mvni            v24.8h,  #0x80, lsl #8 // 0x7fff = (1 << 15) - 1
        add             v6.4s,   v6.4s,   v30.4s
        add             v7.4s,   v7.4s,   v30.4s
        add             v16.4s,  v16.4s,  v30.4s
        add             v17.4s,  v17.4s,  v30.4s
        srshl           v6.4s,   v6.4s,   v29.4s
        srshl           v7.4s,   v7.4s,   v29.4s
        srshl           v16.4s,  v16.4s,  v29.4s
        srshl           v17.4s,  v17.4s,  v29.4s
        sqxtun          v6.4h,   v6.4s
        sqxtun2         v6.8h,   v7.4s
        sqxtun          v7.4h,   v16.4s
        sqxtun2         v7.8h,   v17.4s
        umin            v6.8h,   v6.8h,   v24.8h
        umin            v7.8h,   v7.8h,   v24.8h
        sub             v6.8h,   v6.8h,   v31.8h
        sub             v7.8h,   v7.8h,   v31.8h

        subs            w4,  w4,  #16

        st1             {v6.8h, v7.8h}, [x14], #32

        b.le            0f
        mov             v2.16b,  v4.16b
        tst             w7,  #2 // LR_HAVE_RIGHT
        ld1             {v3.8h, v4.8h}, [x3], #32
        b.ne            4b // If we don't need to pad, just keep filtering.
        b               3b // If we need to pad, check how many pixels we have left.

0:
        ldr             x14,      [sp, #16]
        ldp             x3,  x4,  [sp], #32
        ret
endfunc

function wiener_filter7_v_16bpc_neon
        // Backing up/restoring registers shifted, so that x9 gets the value
        // of x10, etc, afterwards.
        stp             x10, x11, [sp, #-64]!
        stp             x12, x13, [sp, #16]
        stp             x14, x14, [sp, #32]
        stp             x0,  x4,  [sp, #48]
1:
        ld1             {v16.8h, v17.8h}, [x9],  #32
        ld1             {v18.8h, v19.8h}, [x10], #32
        ld1             {v20.8h, v21.8h}, [x11], #32
        ld1             {v22.8h, v23.8h}, [x12], #32
        ld1             {v24.8h, v25.8h}, [x13], #32
        ld1             {v6.8h,  v7.8h},  [x14], #32

        smull           v2.4s,   v16.4h,  v0.h[4]
        smlal           v2.4s,   v18.4h,  v0.h[5]
        smlal           v2.4s,   v20.4h,  v0.h[6]
        smlal           v2.4s,   v22.4h,  v0.h[7]
        smlal           v2.4s,   v24.4h,  v0.h[6]
        smlal           v2.4s,   v6.4h,   v0.h[5]
        smlal           v2.4s,   v6.4h,   v0.h[4]
        smull2          v3.4s,   v16.8h,  v0.h[4]
        smlal2          v3.4s,   v18.8h,  v0.h[5]
        smlal2          v3.4s,   v20.8h,  v0.h[6]
        smlal2          v3.4s,   v22.8h,  v0.h[7]
        smlal2          v3.4s,   v24.8h,  v0.h[6]
        smlal2          v3.4s,   v6.8h,   v0.h[5]
        smlal2          v3.4s,   v6.8h,   v0.h[4]
        smull           v4.4s,   v17.4h,  v0.h[4]
        smlal           v4.4s,   v19.4h,  v0.h[5]
        smlal           v4.4s,   v21.4h,  v0.h[6]
        smlal           v4.4s,   v23.4h,  v0.h[7]
        smlal           v4.4s,   v25.4h,  v0.h[6]
        smlal           v4.4s,   v7.4h,   v0.h[5]
        smlal           v4.4s,   v7.4h,   v0.h[4]
        smull2          v5.4s,   v17.8h,  v0.h[4]
        smlal2          v5.4s,   v19.8h,  v0.h[5]
        smlal2          v5.4s,   v21.8h,  v0.h[6]
        smlal2          v5.4s,   v23.8h,  v0.h[7]
        smlal2          v5.4s,   v25.8h,  v0.h[6]
        smlal2          v5.4s,   v7.8h,   v0.h[5]
        smlal2          v5.4s,   v7.8h,   v0.h[4]
        srshl           v2.4s,   v2.4s,   v27.4s  // -round_bits_v
        srshl           v3.4s,   v3.4s,   v27.4s
        srshl           v4.4s,   v4.4s,   v27.4s
        srshl           v5.4s,   v5.4s,   v27.4s
        sqxtun          v2.4h,   v2.4s
        sqxtun2         v2.8h,   v3.4s
        sqxtun          v3.4h,   v4.4s
        sqxtun2         v3.8h,   v5.4s
        umin            v2.8h,   v2.8h,   v28.8h  // bitdepth_max
        umin            v3.8h,   v3.8h,   v28.8h
        subs            w4,  w4,  #16
        st1             {v2.8h, v3.8h}, [x0], #32
        b.gt            1b

        ldp             x0,  x4,  [sp, #48]
        ldp             x13, x14, [sp, #32]
        ldp             x11, x12, [sp, #16]
        ldp             x9,  x10, [sp], #64

        add             x0,  x0,  x1
        ret
endfunc

function wiener_filter7_hv_16bpc_neon
        // Backing up/restoring registers shifted, so that x9 gets the value
        // of x10, etc, and x15==x9, afterwards.
        stp             x10, x11, [sp, #-80]!
        stp             x12, x13, [sp, #16]
        stp             x14, x15, [sp, #32]
        stp             x10, x0,  [sp, #48]
        stp             x3,  x4,  [sp, #64]

        // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
        tst             w7,  #1 // LR_HAVE_LEFT
        b.eq            1f
        // LR_HAVE_LEFT
        cbnz            x2,  0f
        // left == NULL
        sub             x3,  x3,  #6
        ld1             {v2.8h, v3.8h}, [x3], #32
        b               2f

0:
        // LR_HAVE_LEFT, left != NULL
        ld1             {v2.8h, v3.8h}, [x3], #32
        ld1             {v4.d}[1], [x2], #8
        // Move x3 back to account for the last 3 pixels we loaded earlier,
        // which we'll shift out.
        sub             x3,  x3,  #6
        ext             v3.16b,  v2.16b,  v3.16b,  #10
        ext             v2.16b,  v4.16b,  v2.16b,  #10
        b               2f
1:
        ld1             {v2.8h, v3.8h}, [x3], #32
        // !LR_HAVE_LEFT, fill v4 with the leftmost pixel
        // and shift v3 to have 3x the first pixel at the front.
        dup             v4.8h,  v2.h[0]
        // Move x3 back to account for the last 3 pixels we loaded before,
        // which we shifted out.
        sub             x3,  x3,  #6
        ext             v3.16b,  v2.16b,  v3.16b,  #10
        ext             v2.16b,  v4.16b,  v2.16b,  #10

2:
        ld1             {v4.8h}, [x3], #16

        tst             w7,  #2 // LR_HAVE_RIGHT
        b.ne            4f

3:      // !LR_HAVE_RIGHT

        // Check whether we need to pad the right edge
        cmp             w4,  #19
        b.ge            4f   // If w >= 19, all used input pixels are valid

        // 1 <= w < 19, w+3 pixels valid in v2-v4. For w>=9,
        // this ends up called again; it's not strictly needed in those
        // cases (we pad enough here), but keeping the code as simple as possible.

        // The padding pixel is v2/3/4.h[w+2]. x3 points at the next input, ie
        // v2/3/4.h[24]. Thus read from x3[w-22] to find the padding pixel.
        sub             w17, w4,  #22
        // Insert padding in v2/3/4.h[w+3] onwards; fuse the +3 (*2) into the
        // buffer pointer.
        movrel          x6,  right_ext_mask, -6
        ldr             h26, [x3,  w17, sxtw #1]
        sub             x6,  x6,  w4,  uxtw #1
        dup             v26.8h,  v26.h[0]
        ld1             {v23.16b, v24.16b, v25.16b}, [x6]

        bit             v2.16b,  v26.16b, v23.16b
        bit             v3.16b,  v26.16b, v24.16b
        bit             v4.16b,  v26.16b, v25.16b

4:      // Loop horizontally
        ext             v17.16b, v2.16b,  v3.16b, #4
        ext             v19.16b, v2.16b,  v3.16b, #8
        ext             v16.16b, v2.16b,  v3.16b, #2
        ext             v20.16b, v2.16b,  v3.16b, #10
        ext             v21.16b, v2.16b,  v3.16b, #12
        ext             v18.16b, v2.16b,  v3.16b, #6
        add             v19.8h,  v19.8h,  v17.8h
        add             v20.8h,  v20.8h,  v16.8h
        add             v21.8h,  v21.8h,  v2.8h
        smull           v6.4s,   v18.4h,  v0.h[3]
        smlal           v6.4s,   v19.4h,  v0.h[2]
        smlal           v6.4s,   v20.4h,  v0.h[1]
        smlal           v6.4s,   v21.4h,  v0.h[0]
        smull2          v7.4s,   v18.8h,  v0.h[3]
        smlal2          v7.4s,   v19.8h,  v0.h[2]
        smlal2          v7.4s,   v20.8h,  v0.h[1]
        smlal2          v7.4s,   v21.8h,  v0.h[0]

        ext             v17.16b, v3.16b,  v4.16b, #4
        ext             v19.16b, v3.16b,  v4.16b, #8
        ext             v16.16b, v3.16b,  v4.16b, #2
        ext             v20.16b, v3.16b,  v4.16b, #10
        ext             v21.16b, v3.16b,  v4.16b, #12
        ext             v18.16b, v3.16b,  v4.16b, #6

        add             v19.8h,  v19.8h,  v17.8h
        add             v20.8h,  v20.8h,  v16.8h
        add             v21.8h,  v21.8h,  v3.8h
        smull           v24.4s,  v18.4h,  v0.h[3]
        smlal           v24.4s,  v19.4h,  v0.h[2]
        smlal           v24.4s,  v20.4h,  v0.h[1]
        smlal           v24.4s,  v21.4h,  v0.h[0]
        smull2          v25.4s,  v18.8h,  v0.h[3]
        smlal2          v25.4s,  v19.8h,  v0.h[2]
        smlal2          v25.4s,  v20.8h,  v0.h[1]
        smlal2          v25.4s,  v21.8h,  v0.h[0]

        ld1             {v16.8h, v17.8h}, [x9],  #32

        mvni            v26.8h,  #0x80, lsl #8 // 0x7fff = (1 << 15) - 1
        add             v6.4s,   v6.4s,   v30.4s
        add             v7.4s,   v7.4s,   v30.4s
        add             v24.4s,  v24.4s,  v30.4s
        add             v25.4s,  v25.4s,  v30.4s
        ld1             {v18.8h, v19.8h}, [x10], #32
        srshl           v6.4s,   v6.4s,   v29.4s
        srshl           v7.4s,   v7.4s,   v29.4s
        srshl           v24.4s,  v24.4s,  v29.4s
        srshl           v25.4s,  v25.4s,  v29.4s
        ld1             {v20.8h, v21.8h}, [x11], #32
        sqxtun          v6.4h,   v6.4s
        sqxtun2         v6.8h,   v7.4s
        sqxtun          v7.4h,   v24.4s
        sqxtun2         v7.8h,   v25.4s
        ld1             {v22.8h, v23.8h}, [x12], #32
        umin            v6.8h,   v6.8h,   v26.8h
        umin            v7.8h,   v7.8h,   v26.8h
        ld1             {v24.8h, v25.8h}, [x13], #32
        sub             v6.8h,   v6.8h,   v31.8h
        sub             v7.8h,   v7.8h,   v31.8h

        ld1             {v8.8h,  v9.8h},  [x14], #32

        smull           v1.4s,   v16.4h,  v0.h[4]
        smlal           v1.4s,   v18.4h,  v0.h[5]
        smlal           v1.4s,   v20.4h,  v0.h[6]
        smlal           v1.4s,   v22.4h,  v0.h[7]
        smlal           v1.4s,   v24.4h,  v0.h[6]
        smlal           v1.4s,   v8.4h,   v0.h[5]
        smlal           v1.4s,   v6.4h,   v0.h[4]
        smull2          v5.4s,   v16.8h,  v0.h[4]
        smlal2          v5.4s,   v18.8h,  v0.h[5]
        smlal2          v5.4s,   v20.8h,  v0.h[6]
        smlal2          v5.4s,   v22.8h,  v0.h[7]
        smlal2          v5.4s,   v24.8h,  v0.h[6]
        smlal2          v5.4s,   v8.8h,   v0.h[5]
        smlal2          v5.4s,   v6.8h,   v0.h[4]
        smull           v26.4s,  v17.4h,  v0.h[4]
        smlal           v26.4s,  v19.4h,  v0.h[5]
        smlal           v26.4s,  v21.4h,  v0.h[6]
        smlal           v26.4s,  v23.4h,  v0.h[7]
        smlal           v26.4s,  v25.4h,  v0.h[6]
        smlal           v26.4s,  v9.4h,   v0.h[5]
        smlal           v26.4s,  v7.4h,   v0.h[4]
        smull2          v16.4s,  v17.8h,  v0.h[4]
        smlal2          v16.4s,  v19.8h,  v0.h[5]
        smlal2          v16.4s,  v21.8h,  v0.h[6]
        smlal2          v16.4s,  v23.8h,  v0.h[7]
        smlal2          v16.4s,  v25.8h,  v0.h[6]
        smlal2          v16.4s,  v9.8h,   v0.h[5]
        smlal2          v16.4s,  v7.8h,   v0.h[4]
        srshl           v1.4s,   v1.4s,   v27.4s  // -round_bits_v
        srshl           v5.4s,   v5.4s,   v27.4s
        srshl           v26.4s,  v26.4s,  v27.4s
        srshl           v16.4s,  v16.4s,  v27.4s
        sqxtun          v18.4h,  v1.4s
        sqxtun2         v18.8h,  v5.4s
        sqxtun          v19.4h,  v26.4s
        sqxtun2         v19.8h,  v16.4s
        st1             {v6.8h, v7.8h}, [x15], #32
        umin            v18.8h,  v18.8h,  v28.8h  // bitdepth_max
        umin            v19.8h,  v19.8h,  v28.8h
        subs            w4,  w4,  #16

        st1             {v18.8h, v19.8h}, [x0], #32

        b.le            0f
        mov             v2.16b,  v4.16b
        tst             w7,  #2 // LR_HAVE_RIGHT
        ld1             {v3.8h, v4.8h}, [x3], #32
        b.ne            4b // If we don't need to pad, just keep filtering.
        b               3b // If we need to pad, check how many pixels we have left.

0:
        ldp             x3,  x4,  [sp, #64]
        ldp             x15, x0,  [sp, #48]
        ldp             x13, x14, [sp, #32]
        ldp             x11, x12, [sp, #16]
        ldp             x9,  x10, [sp], #80

        add             x3,  x3,  x1
        add             x0,  x0,  x1

        ret
endfunc

// void dav1d_wiener_filter5_16bpc_neon(pixel *p, const ptrdiff_t p_stride,
//                                      const pixel (*left)[4], const pixel *lpf,
//                                      const int w, int h,
//                                      const int16_t filter[2][8],
//                                      const enum LrEdgeFlags edges,
//                                      const int bitdepth_max);
function wiener_filter5_16bpc_neon, export=1
        ldr             w8,  [sp]
        AARCH64_SIGN_LINK_REGISTER
        stp             x29, x30, [sp, #-32]!
        stp             d8,  d9,  [sp, #16]
        mov             x29, sp
        ld1             {v0.8h, v1.8h},  [x6]
        tst             w7,  #4               // LR_HAVE_TOP
        sub_sp          384*2*4

        dup             v28.8h,  w8           // bitdepth_max
        clz             w8,  w8
        movi            v30.4s,  #1
        sub             w10, w8,  #38         // -(bitdepth + 6)
        sub             w11, w8,  #11         // round_bits_v
        sub             w8,  w8,  #25         // -round_bits_h
        neg             w10, w10              // bitdepth + 6
        neg             w11, w11              // -round_bits_v
        dup             v2.4s,   w10
        dup             v29.4s,  w8           // -round_bits_h
        dup             v27.4s,  w11          // -round_bits_v
        movi            v31.8h,  #0x20, lsl #8  // 1 << 13 = 8192
        ushl            v30.4s,  v30.4s,  v2.4s // 1 << (bitdepth + 6)

        zip1            v0.2d,   v0.2d,   v1.2d // move vertical coeffs to v0.h[4-7], freeing up v1

        // x11 - t4
        // x12 - t3
        // x13 - t2
        // x14 - t1
        // x15 - t0
        mov             x14, sp               // t1
        b.eq            L(no_top_5)

        mov             x16, x2               // backup left
        mov             x2,  #0
        bl              wiener_filter5_h_16bpc_neon
        add             x3,  x3,  x1          // lpf += stride
        mov             x11, x14              // t4
        add             x14, x14, #384*2      // t1 += 384*2
        bl              wiener_filter5_h_16bpc_neon
        add             x3,  x3,  x1,  lsl #2
        add             x3,  x3,  x1          // lpf += stride*5
        mov             x12, x14              // t3
        add             x14, x14, #384*2      // t1 += 384*2
        mov             x2,  x16              // left
        mov             x16, x3               // backup lpf
        mov             x3,  x0               // lpf = p
        bl              wiener_filter5_h_16bpc_neon
        subs            w5,  w5,  #1          // h--
        mov             x13, x14              // t2
        b.eq            L(v1_5)
        add             x3,  x3,  x1          // src += stride
        add             x14, x14, #384*2      // t1 += 384*2
        bl              wiener_filter5_h_16bpc_neon
        subs            w5,  w5,  #1          // h--
        b.eq            L(v2_5)
        add             x3,  x3,  x1          // src += stride

L(main_5):
        mov             x15, x11              // t0 = t4
L(main_loop_5):
        bl              wiener_filter5_hv_16bpc_neon
        subs            w5,  w5,  #1          // h--
        b.ne            L(main_loop_5)
        tst             w7,  #8 // LR_HAVE_BOTTOM
        b.eq            L(v2_5)

        mov             x3,  x16              // restore lpf
        mov             x2,  #0               // left = NULL
        bl              wiener_filter5_hv_16bpc_neon
        bl              wiener_filter5_hv_16bpc_neon
L(end_5):

        mov             sp,  x29
        ldp             d8,  d9,  [sp, #16]
        ldp             x29, x30, [sp], #32
        AARCH64_VALIDATE_LINK_REGISTER
        ret

L(no_top_5):
        add             x3,  x3,  x1,  lsl #2
        add             x16, x3,  x1,  lsl #1 // lpf += stride*6, backup
        mov             x3,  x0               // lpf = p

        bl              wiener_filter5_h_16bpc_neon
        subs            w5,  w5,  #1          // h--
        mov             x11, x14              // t4
        mov             x12, x14              // t3
        mov             x13, x14              // t2
        b.eq            L(v1_5)
        add             x3,  x3,  x1          // src += stride
        add             x14, x14, #384*2      // t1 += 384*2
        bl              wiener_filter5_h_16bpc_neon
        subs            w5,  w5,  #1          // h--
        b.eq            L(v2_5)
        add             x3,  x3,  x1          // src += stride
        add             x15, x14, #384*2      // t0 = t1 + 384*2
        bl              wiener_filter5_hv_16bpc_neon
        subs            w5,  w5,  #1          // h--
        b.eq            L(v2_5)
        add             x15, x15, #384*2*3    // t0 += 384*2*3
        bl              wiener_filter5_hv_16bpc_neon
        subs            w5,  w5,  #1          // h--
        b.ne            L(main_5)
L(v2_5):
        bl              wiener_filter5_v_16bpc_neon
        add             x0,  x0,  x1
        mov             x11, x12
        mov             x12, x13
        mov             x13, x14
L(v1_5):
        bl              wiener_filter5_v_16bpc_neon
        b               L(end_5)
endfunc


function wiener_filter5_h_16bpc_neon
        stp             x3,  x4,  [sp, #-32]!
        str             x14,      [sp, #16]

        // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
        tst             w7,  #1 // LR_HAVE_LEFT
        b.eq            1f
        // LR_HAVE_LEFT
        cbnz            x2,  0f
        // left == NULL
        sub             x3,  x3,  #4
        ld1             {v2.8h, v3.8h}, [x3], #32
        b               2f

0:
        // LR_HAVE_LEFT, left != NULL
        ld1             {v2.8h, v3.8h}, [x3], #32
        ld1             {v4.d}[1], [x2], #8
        // Move x3 back to account for the last 2 pixels we loaded earlier,
        // which we'll shift out.
        sub             x3,  x3,  #4
        ext             v3.16b,  v2.16b,  v3.16b,  #12
        ext             v2.16b,  v4.16b,  v2.16b,  #12
        b               2f

1:
        ld1             {v2.8h, v3.8h}, [x3], #32
        // !LR_HAVE_LEFT, fill v2 with the leftmost pixel
        // and shift v3 to have 3x the first pixel at the front.
        dup             v4.8h,  v2.h[0]
        // Move x3 back to account for the last 2 pixels we loaded before,
        // which we shifted out.
        sub             x3,  x3,  #4
        ext             v3.16b,  v2.16b,  v3.16b,  #12
        ext             v2.16b,  v4.16b,  v2.16b,  #12

2:
        ld1             {v4.8h}, [x3], #16

        tst             w7,  #2 // LR_HAVE_RIGHT
        b.ne            4f

3:      // !LR_HAVE_RIGHT

        // Check whether we need to pad the right edge
        cmp             w4,  #18
        b.ge            4f   // If w >= 18, all used input pixels are valid

        // 1 <= w < 18, w+2 pixels valid in v2-v4. For w>=9,
        // this ends up called again; it's not strictly needed in those
        // cases (we pad enough here), but keeping the code as simple as possible.

        // The padding pixel is v2/3/4.h[w+2]. x3 points at the next input, ie
        // v2/3/4.h[24]. Thus read from x3[w-23] to find the padding pixel.
        sub             w17, w4,  #23
        // Insert padding in v2/3/4.h[w+2] onwards; fuse the +2 (*2) into the
        // buffer pointer.
        movrel          x6,  right_ext_mask, -4
        ldr             h26, [x3,  w17, sxtw #1]
        sub             x6,  x6,  w4,  uxtw #1
        dup             v26.8h,  v26.h[0]
        ld1             {v23.16b, v24.16b, v25.16b}, [x6]

        bit             v2.16b,  v26.16b, v23.16b
        bit             v3.16b,  v26.16b, v24.16b
        bit             v4.16b,  v26.16b, v25.16b

4:      // Loop horizontally
        // Interleaving the mul/mla chains actually hurts performance
        // significantly on Cortex A53, thus keeping mul/mla tightly
        // chained like this.
        ext             v16.16b, v2.16b,  v3.16b, #2
        ext             v18.16b, v2.16b,  v3.16b, #6
        ext             v19.16b, v2.16b,  v3.16b, #8
        ext             v17.16b, v2.16b,  v3.16b, #4
        add             v18.8h,  v18.8h,  v16.8h
        add             v19.8h,  v19.8h,  v2.8h
        smull           v6.4s,   v17.4h,  v0.h[3]
        smlal           v6.4s,   v18.4h,  v0.h[2]
        smlal           v6.4s,   v19.4h,  v0.h[1]
        smull2          v7.4s,   v17.8h,  v0.h[3]
        smlal2          v7.4s,   v18.8h,  v0.h[2]
        smlal2          v7.4s,   v19.8h,  v0.h[1]

        ext             v16.16b, v3.16b,  v4.16b, #2
        ext             v18.16b, v3.16b,  v4.16b, #6
        ext             v19.16b, v3.16b,  v4.16b, #8
        ext             v17.16b, v3.16b,  v4.16b, #4
        add             v18.8h,  v18.8h,  v16.8h
        add             v19.8h,  v19.8h,  v3.8h
        smull           v16.4s,  v17.4h,  v0.h[3]
        smlal           v16.4s,  v18.4h,  v0.h[2]
        smlal           v16.4s,  v19.4h,  v0.h[1]
        smull2          v17.4s,  v17.8h,  v0.h[3]
        smlal2          v17.4s,  v18.8h,  v0.h[2]
        smlal2          v17.4s,  v19.8h,  v0.h[1]

        mvni            v24.8h,  #0x80, lsl #8 // 0x7fff = (1 << 15) - 1
        add             v6.4s,   v6.4s,   v30.4s
        add             v7.4s,   v7.4s,   v30.4s
        add             v16.4s,  v16.4s,  v30.4s
        add             v17.4s,  v17.4s,  v30.4s
        srshl           v6.4s,   v6.4s,   v29.4s
        srshl           v7.4s,   v7.4s,   v29.4s
        srshl           v16.4s,  v16.4s,  v29.4s
        srshl           v17.4s,  v17.4s,  v29.4s
        sqxtun          v6.4h,   v6.4s
        sqxtun2         v6.8h,   v7.4s
        sqxtun          v7.4h,   v16.4s
        sqxtun2         v7.8h,   v17.4s
        umin            v6.8h,   v6.8h,   v24.8h
        umin            v7.8h,   v7.8h,   v24.8h
        sub             v6.8h,   v6.8h,   v31.8h
        sub             v7.8h,   v7.8h,   v31.8h

        subs            w4,  w4,  #16

        st1             {v6.8h, v7.8h}, [x14], #32

        b.le            0f
        mov             v2.16b,  v4.16b
        tst             w7,  #2 // LR_HAVE_RIGHT
        ld1             {v3.8h, v4.8h}, [x3], #32
        b.ne            4b // If we don't need to pad, just keep filtering.
        b               3b // If we need to pad, check how many pixels we have left.

0:
        ldr             x14,      [sp, #16]
        ldp             x3,  x4,  [sp], #32
        ret
endfunc

function wiener_filter5_v_16bpc_neon
        stp             x11, x12, [sp, #-48]!
        stp             x13, x14, [sp, #16]
        stp             x0,  x4,  [sp, #32]
1:
        ld1             {v16.8h, v17.8h}, [x11], #32
        ld1             {v18.8h, v19.8h}, [x12], #32
        ld1             {v20.8h, v21.8h}, [x13], #32
        ld1             {v22.8h, v23.8h}, [x14], #32

        smull           v2.4s,   v16.4h,  v0.h[5]
        smlal           v2.4s,   v18.4h,  v0.h[6]
        smlal           v2.4s,   v20.4h,  v0.h[7]
        smlal           v2.4s,   v22.4h,  v0.h[6]
        smlal           v2.4s,   v22.4h,  v0.h[5]
        smull2          v3.4s,   v16.8h,  v0.h[5]
        smlal2          v3.4s,   v18.8h,  v0.h[6]
        smlal2          v3.4s,   v20.8h,  v0.h[7]
        smlal2          v3.4s,   v22.8h,  v0.h[6]
        smlal2          v3.4s,   v22.8h,  v0.h[5]
        smull           v4.4s,   v17.4h,  v0.h[5]
        smlal           v4.4s,   v19.4h,  v0.h[6]
        smlal           v4.4s,   v21.4h,  v0.h[7]
        smlal           v4.4s,   v23.4h,  v0.h[6]
        smlal           v4.4s,   v23.4h,  v0.h[5]
        smull2          v5.4s,   v17.8h,  v0.h[5]
        smlal2          v5.4s,   v19.8h,  v0.h[6]
        smlal2          v5.4s,   v21.8h,  v0.h[7]
        smlal2          v5.4s,   v23.8h,  v0.h[6]
        smlal2          v5.4s,   v23.8h,  v0.h[5]
        srshl           v2.4s,   v2.4s,   v27.4s  // -round_bits_v
        srshl           v3.4s,   v3.4s,   v27.4s
        srshl           v4.4s,   v4.4s,   v27.4s
        srshl           v5.4s,   v5.4s,   v27.4s
        sqxtun          v2.4h,   v2.4s
        sqxtun2         v2.8h,   v3.4s
        sqxtun          v3.4h,   v4.4s
        sqxtun2         v3.8h,   v5.4s
        umin            v2.8h,   v2.8h,   v28.8h  // bitdepth_max
        umin            v3.8h,   v3.8h,   v28.8h

        subs            w4,  w4,  #16
        st1             {v2.8h, v3.8h}, [x0], #32
        b.gt            1b

        ldp             x0,  x4,  [sp, #32]
        ldp             x13, x14, [sp, #16]
        ldp             x11, x12, [sp], #48

        ret
endfunc

function wiener_filter5_hv_16bpc_neon
        // Backing up/restoring registers shifted, so that x11 gets the value
        // of x12, etc, and x15==x11, afterwards.
        stp             x12, x13, [sp, #-64]!
        stp             x14, x15, [sp, #16]
        stp             x12, x0,  [sp, #32]
        stp             x3,  x4,  [sp, #48]

        // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
        tst             w7,  #1 // LR_HAVE_LEFT
        b.eq            1f
        // LR_HAVE_LEFT
        cbnz            x2,  0f
        // left == NULL
        sub             x3,  x3,  #4
        ld1             {v2.8h, v3.8h}, [x3], #32
        b               2f

0:
        // LR_HAVE_LEFT, left != NULL
        ld1             {v2.8h, v3.8h}, [x3], #32
        ld1             {v4.d}[1], [x2], #8
        // Move x3 back to account for the last 2 pixels we loaded earlier,
        // which we'll shift out.
        sub             x3,  x3,  #4
        ext             v3.16b,  v2.16b,  v3.16b,  #12
        ext             v2.16b,  v4.16b,  v2.16b,  #12
        b               2f
1:
        ld1             {v2.8h, v3.8h}, [x3], #32
        // !LR_HAVE_LEFT, fill v2 with the leftmost pixel
        // and shift v3 to have 2x the first pixel at the front.
        dup             v4.8h,   v2.h[0]
        // Move x3 back to account for the last 2 pixels we loaded before,
        // which we shifted out.
        sub             x3,  x3,  #4
        ext             v3.16b,  v2.16b,  v3.16b,  #12
        ext             v2.16b,  v4.16b,  v2.16b,  #12

2:
        ld1             {v4.8h}, [x3], #16

        tst             w7,  #2 // LR_HAVE_RIGHT
        b.ne            4f

3:      // !LR_HAVE_RIGHT

        // Check whether we need to pad the right edge
        cmp             w4,  #18
        b.ge            4f   // If w >= 18, all used input pixels are valid

        // 1 <= w < 18, w+2 pixels valid in v2-v4. For w>=9,
        // this ends up called again; it's not strictly needed in those
        // cases (we pad enough here), but keeping the code as simple as possible.

        // The padding pixel is v2/3/4.h[w+1]. x3 points at the next input, ie
        // v2/3/4.h[24]. Thus read from x3[w-23] to find the padding pixel.
        sub             w17, w4,  #23
        // Insert padding in v2/3/4.h[w+2] onwards; fuse the +2 (*2) into the
        // buffer pointer.
        movrel          x6,  right_ext_mask, -4
        ldr             h26, [x3,  w17, sxtw #1]
        sub             x6,  x6,  w4,  uxtw #1
        dup             v26.8h,  v26.h[0]
        ld1             {v23.16b, v24.16b, v25.16b}, [x6]

        bit             v2.16b,  v26.16b, v23.16b
        bit             v3.16b,  v26.16b, v24.16b
        bit             v4.16b,  v26.16b, v25.16b

4:      // Loop horizontally
        ext             v16.16b, v2.16b,  v3.16b, #2
        ext             v18.16b, v2.16b,  v3.16b, #6
        ext             v19.16b, v2.16b,  v3.16b, #8
        ext             v17.16b, v2.16b,  v3.16b, #4
        add             v18.8h,  v18.8h,  v16.8h
        add             v19.8h,  v19.8h,  v2.8h
        smull           v6.4s,   v17.4h,  v0.h[3]
        smlal           v6.4s,   v18.4h,  v0.h[2]
        smlal           v6.4s,   v19.4h,  v0.h[1]
        smull2          v7.4s,   v17.8h,  v0.h[3]
        smlal2          v7.4s,   v18.8h,  v0.h[2]
        smlal2          v7.4s,   v19.8h,  v0.h[1]

        ext             v16.16b, v3.16b,  v4.16b, #2
        ext             v18.16b, v3.16b,  v4.16b, #6
        ext             v19.16b, v3.16b,  v4.16b, #8
        ext             v17.16b, v3.16b,  v4.16b, #4
        add             v18.8h,  v18.8h,  v16.8h
        add             v19.8h,  v19.8h,  v3.8h
        smull           v24.4s,  v17.4h,  v0.h[3]
        smlal           v24.4s,  v18.4h,  v0.h[2]
        smlal           v24.4s,  v19.4h,  v0.h[1]
        smull2          v25.4s,  v17.8h,  v0.h[3]
        smlal2          v25.4s,  v18.8h,  v0.h[2]
        smlal2          v25.4s,  v19.8h,  v0.h[1]

        ld1             {v16.8h, v17.8h}, [x11], #32
        mvni            v26.8h,  #0x80, lsl #8 // 0x7fff = (1 << 15) - 1
        add             v6.4s,   v6.4s,   v30.4s
        add             v7.4s,   v7.4s,   v30.4s
        add             v24.4s,  v24.4s,  v30.4s
        add             v25.4s,  v25.4s,  v30.4s
        ld1             {v18.8h, v19.8h}, [x12], #32
        srshl           v6.4s,   v6.4s,   v29.4s
        srshl           v7.4s,   v7.4s,   v29.4s
        srshl           v24.4s,  v24.4s,  v29.4s
        srshl           v25.4s,  v25.4s,  v29.4s
        ld1             {v20.8h, v21.8h}, [x13], #32
        sqxtun          v6.4h,   v6.4s
        sqxtun2         v6.8h,   v7.4s
        sqxtun          v7.4h,   v24.4s
        sqxtun2         v7.8h,   v25.4s
        ld1             {v22.8h, v23.8h}, [x14], #32
        umin            v6.8h,   v6.8h,   v26.8h
        umin            v7.8h,   v7.8h,   v26.8h
        sub             v6.8h,   v6.8h,   v31.8h
        sub             v7.8h,   v7.8h,   v31.8h

        smull           v8.4s,   v16.4h,  v0.h[5]
        smlal           v8.4s,   v18.4h,  v0.h[6]
        smlal           v8.4s,   v20.4h,  v0.h[7]
        smlal           v8.4s,   v22.4h,  v0.h[6]
        smlal           v8.4s,   v6.4h,   v0.h[5]
        smull2          v9.4s,   v16.8h,  v0.h[5]
        smlal2          v9.4s,   v18.8h,  v0.h[6]
        smlal2          v9.4s,   v20.8h,  v0.h[7]
        smlal2          v9.4s,   v22.8h,  v0.h[6]
        smlal2          v9.4s,   v6.8h,   v0.h[5]
        smull           v1.4s,   v17.4h,  v0.h[5]
        smlal           v1.4s,   v19.4h,  v0.h[6]
        smlal           v1.4s,   v21.4h,  v0.h[7]
        smlal           v1.4s,   v23.4h,  v0.h[6]
        smlal           v1.4s,   v7.4h,   v0.h[5]
        smull2          v5.4s,   v17.8h,  v0.h[5]
        smlal2          v5.4s,   v19.8h,  v0.h[6]
        smlal2          v5.4s,   v21.8h,  v0.h[7]
        smlal2          v5.4s,   v23.8h,  v0.h[6]
        smlal2          v5.4s,   v7.8h,   v0.h[5]
        srshl           v8.4s,   v8.4s,   v27.4s  // -round_bits_v
        srshl           v9.4s,   v9.4s,   v27.4s
        srshl           v1.4s,   v1.4s,   v27.4s
        srshl           v5.4s,   v5.4s,   v27.4s
        sqxtun          v8.4h,   v8.4s
        sqxtun2         v8.8h,   v9.4s
        sqxtun          v9.4h,   v1.4s
        sqxtun2         v9.8h,   v5.4s
        st1             {v6.8h, v7.8h}, [x15], #32
        umin            v8.8h,   v8.8h,   v28.8h  // bitdepth_max
        umin            v9.8h,   v9.8h,   v28.8h

        subs            w4,  w4,  #16

        st1             {v8.8h, v9.8h}, [x0], #32

        b.le            0f
        mov             v2.16b,  v4.16b
        tst             w7,  #2 // LR_HAVE_RIGHT
        ld1             {v3.8h, v4.8h}, [x3], #32
        b.ne            4b // If we don't need to pad, just keep filtering.
        b               3b // If we need to pad, check how many pixels we have left.

0:
        ldp             x3,  x4,  [sp, #48]
        ldp             x15, x0,  [sp, #32]
        ldp             x13, x14, [sp, #16]
        ldp             x11, x12, [sp], #64

        add             x3,  x3,  x1
        add             x0,  x0,  x1

        ret
endfunc

#include "looprestoration_tmpl.S"

// void dav1d_sgr_box3_row_h_16bpc_neon(int32_t *sumsq, int16_t *sum,
//                                      const pixel (*left)[4],
//                                      const pixel *src, const int w,
//                                      const enum LrEdgeFlags edges);
function sgr_box3_row_h_16bpc_neon, export=1
        add             w4,  w4,  #2 // w += 2

        tst             w5,  #1 // LR_HAVE_LEFT
        b.eq            1f
        cbnz            x2,  0f

        // LR_HAVE_LEFT && left == NULL
        sub             x3,  x3,  #4
        ld1             {v0.8h, v1.8h}, [x3], #32
        b               2f

0:
        // LR_HAVE_LEFT, left != NULL
        ld1             {v0.8h, v1.8h}, [x3], #32
        ld1             {v2.d}[1], [x2]
        // Move x3 back to account for the last 2 pixels we loaded earlier,
        // which we'll shift out.
        sub             x3,  x3,  #4
        ext             v1.16b, v0.16b, v1.16b, #12
        ext             v0.16b, v2.16b, v0.16b, #12
        b               2f

1:
        ld1             {v0.8h, v1.8h}, [x3], #32
        // !LR_HAVE_LEFT, fill v2 with the leftmost pixel
        // and shift v0/v1 to have 2x the first pixel at the front.
        dup             v2.8h, v0.h[0]
        // Move x3 back to account for the last 2 pixels we loaded before,
        // which we shifted out.
        sub             x3,  x3,  #4
        ext             v1.16b, v0.16b, v1.16b, #12
        ext             v0.16b, v2.16b, v0.16b, #12

2:
        tst             w5,  #2 // LR_HAVE_RIGHT
        b.ne            4f
        // If we'll need to pad the right edge, load that pixel to pad with
        // here since we can find it pretty easily from here.
        sub             w13, w4, #(2 + 16 - 2 + 1)
        ldr             h30, [x3,  w13, sxtw #1]
        // Fill v30 with the right padding pixel
        dup             v30.8h,  v30.h[0]
3:      // !LR_HAVE_RIGHT

        // Check whether we need to pad the right edge
        cmp             w4,  #10
        b.ge            4f   // If w >= 10, all used input pixels are valid

        // 1 <= w < 10, w pixels valid in v0. For w=9, this ends up called
        // again; it's not strictly needed in those cases (we pad enough here),
        // but keeping the code as simple as possible.

        // Insert padding in v0.b[w] onwards
        movrel          x13, right_ext_mask
        sub             x13, x13, w4,  uxtw #1
        ld1             {v28.16b, v29.16b}, [x13]

        bit             v0.16b,  v30.16b, v28.16b
        bit             v1.16b,  v30.16b, v29.16b

4:      // Loop horizontally
        ext             v26.16b, v0.16b,  v1.16b,  #2
        ext             v27.16b, v0.16b,  v1.16b,  #4

        add             v6.8h,   v0.8h,   v26.8h
        umull           v22.4s,  v0.4h,   v0.4h
        umlal           v22.4s,  v26.4h,  v26.4h
        umlal           v22.4s,  v27.4h,  v27.4h
        add             v6.8h,   v6.8h,   v27.8h
        umull2          v23.4s,  v0.8h,   v0.8h
        umlal2          v23.4s,  v26.8h,  v26.8h
        umlal2          v23.4s,  v27.8h,  v27.8h

        subs            w4,  w4,  #8

        st1             {v6.8h},         [x1],  #16
        st1             {v22.4s,v23.4s}, [x0],  #32

        b.le            9f
        tst             w5,  #2 // LR_HAVE_RIGHT
        mov             v0.16b,  v1.16b
        ld1             {v1.8h},  [x3],  #16

        b.ne            4b // If we don't need to pad, just keep summing.
        b               3b // If we need to pad, check how many pixels we have left.

9:
        ret
endfunc

// void dav1d_sgr_box5_row_h_16bpc_neon(int32_t *sumsq, int16_t *sum,
//                                      const pixel (*left)[4],
//                                      const pixel *src, const int w,
//                                      const enum LrEdgeFlags edges);
function sgr_box5_row_h_16bpc_neon, export=1
        add             w4,  w4,  #2 // w += 2

        tst             w5,  #1 // LR_HAVE_LEFT
        b.eq            1f
        cbnz            x2,  0f

        // LR_HAVE_LEFT && left == NULL
        sub             x3,  x3,  #6
        ld1             {v0.8h, v1.8h}, [x3], #32
        b               2f

0:
        // LR_HAVE_LEFT, left != NULL
        ld1             {v0.8h, v1.8h}, [x3], #32
        ld1             {v2.d}[1], [x2], #8
        // Move x3 back to account for the last 3 pixels we loaded earlier,
        // which we'll shift out.
        sub             x3,  x3,  #6
        ext             v1.16b,  v0.16b,  v1.16b,  #10
        ext             v0.16b,  v2.16b,  v0.16b,  #10
        b               2f

1:
        ld1             {v0.8h, v1.8h}, [x3], #32
        // !LR_HAVE_LEFT, fill v2 with the leftmost pixel
        // and shift v0/v1 to have 3x the first pixel at the front.
        dup             v2.8h,  v0.h[0]
        // Move x3 back to account for the last 3 pixels we loaded before,
        // which we shifted out.
        sub             x3,  x3,  #6
        ext             v1.16b,  v0.16b,  v1.16b,  #10
        ext             v0.16b,  v2.16b,  v0.16b,  #10

2:
        tst             w5,  #2 // LR_HAVE_RIGHT
        b.ne            4f
        // If we'll need to pad the right edge, load that pixel to pad with
        // here since we can find it pretty easily from here.
        sub             w13, w4, #(2 + 16 - 3 + 1)
        ldr             h30, [x3,  w13, sxtw #1]
        // Fill v30 with the right padding pixel
        dup             v30.8h,  v30.h[0]
3:      // !LR_HAVE_RIGHT

        // Check whether we need to pad the right edge
        cmp             w4,  #11
        b.ge            4f   // If w >= 11, all used input pixels are valid

        // 1 <= w < 11, w+1 pixels valid in v0. For w=9 or w=10,
        // this ends up called again; it's not strictly needed in those
        // cases (we pad enough here), but keeping the code as simple as possible.

        // Insert padding in v0.b[w+1] onwards; fuse the +1 into the
        // buffer pointer.
        movrel          x13, right_ext_mask, -1
        sub             x13, x13, w4,  uxtw #1
        ld1             {v28.16b, v29.16b}, [x13]

        bit             v0.16b,  v30.16b, v28.16b
        bit             v1.16b,  v30.16b, v29.16b

4:      // Loop horizontally
        ext             v26.16b, v0.16b,  v1.16b,  #2
        ext             v27.16b, v0.16b,  v1.16b,  #4

        add             v6.8h,   v0.8h,   v26.8h
        umull           v22.4s,  v0.4h,   v0.4h
        umlal           v22.4s,  v26.4h,  v26.4h
        umlal           v22.4s,  v27.4h,  v27.4h
        add             v6.8h,   v6.8h,   v27.8h
        umull2          v23.4s,  v0.8h,   v0.8h
        umlal2          v23.4s,  v26.8h,  v26.8h
        umlal2          v23.4s,  v27.8h,  v27.8h

        ext             v26.16b, v0.16b,  v1.16b,  #6
        ext             v27.16b, v0.16b,  v1.16b,  #8

        add             v6.8h,   v6.8h,   v26.8h
        umlal           v22.4s,  v26.4h,  v26.4h
        umlal           v22.4s,  v27.4h,  v27.4h
        add             v6.8h,   v6.8h,   v27.8h
        umlal2          v23.4s,  v26.8h,  v26.8h
        umlal2          v23.4s,  v27.8h,  v27.8h

        subs            w4,  w4,  #8

        st1             {v6.8h},         [x1],  #16
        st1             {v22.4s,v23.4s}, [x0],  #32

        b.le            9f
        tst             w5,  #2 // LR_HAVE_RIGHT
        mov             v0.16b,  v1.16b
        ld1             {v1.8h}, [x3], #16

        b.ne            4b // If we don't need to pad, just keep summing.
        b               3b // If we need to pad, check how many pixels we have left.

9:
        ret
endfunc

// void dav1d_sgr_box35_row_h_16bpc_neon(int32_t *sumsq3, int16_t *sum3,
//                                       int32_t *sumsq5, int16_t *sum5,
//                                       const pixel (*left)[4],
//                                       const pixel *src, const int w,
//                                       const enum LrEdgeFlags edges);
function sgr_box35_row_h_16bpc_neon, export=1
        add             w6,  w6,  #2 // w += 2

        tst             w7,  #1 // LR_HAVE_LEFT
        b.eq            1f
        cbnz            x4,  0f

        // LR_HAVE_LEFT && left == NULL
        sub             x5,  x5,  #6
        ld1             {v0.8h, v1.8h}, [x5], #32
        b               2f

0:
        // LR_HAVE_LEFT, left != NULL
        ld1             {v0.8h, v1.8h}, [x5], #32
        ld1             {v2.d}[1], [x4], #8
        // Move x3 back to account for the last 3 pixels we loaded earlier,
        // which we'll shift out.
        sub             x5,  x5,  #6
        ext             v1.16b,  v0.16b,  v1.16b,  #10
        ext             v0.16b,  v2.16b,  v0.16b,  #10
        b               2f

1:
        ld1             {v0.8h, v1.8h}, [x5], #32
        // !LR_HAVE_LEFT, fill v2 with the leftmost pixel
        // and shift v0/v1 to have 3x the first pixel at the front.
        dup             v2.8h,  v0.h[0]
        // Move x5 back to account for the last 3 pixels we loaded before,
        // which we shifted out.
        sub             x5,  x5,  #6
        ext             v1.16b,  v0.16b,  v1.16b,  #10
        ext             v0.16b,  v2.16b,  v0.16b,  #10

2:
        tst             w7,  #2 // LR_HAVE_RIGHT
        b.ne            4f
        // If we'll need to pad the right edge, load that pixel to pad with
        // here since we can find it pretty easily from here.
        sub             w13, w6, #(2 + 16 - 3 + 1)
        ldr             h30, [x5,  w13, sxtw #1]
        // Fill v30 with the right padding pixel
        dup             v30.8h,  v30.h[0]
3:      // !LR_HAVE_RIGHT

        // Check whether we need to pad the right edge
        cmp             w6,  #11
        b.ge            4f   // If w >= 11, all used input pixels are valid

        // 1 <= w < 11, w+1 pixels valid in v0. For w=9 or w=10,
        // this ends up called again; it's not strictly needed in those
        // cases (we pad enough here), but keeping the code as simple as possible.

        // Insert padding in v0.b[w+1] onwards; fuse the +1 into the
        // buffer pointer.
        movrel          x13, right_ext_mask, -1
        sub             x13, x13, w6,  uxtw #1
        ld1             {v28.16b, v29.16b}, [x13]

        bit             v0.16b,  v30.16b, v28.16b
        bit             v1.16b,  v30.16b, v29.16b

4:      // Loop horizontally
        ext             v16.16b, v0.16b,  v1.16b,  #2
        ext             v17.16b, v0.16b,  v1.16b,  #4
        ext             v19.16b, v0.16b,  v1.16b,  #8
        ext             v18.16b, v0.16b,  v1.16b,  #6

        add             v20.8h,  v16.8h,  v17.8h
        add             v21.8h,  v0.8h,   v19.8h
        add             v20.8h,  v20.8h,  v18.8h

        umull           v22.4s,  v16.4h,  v16.4h
        umlal           v22.4s,  v17.4h,  v17.4h
        umlal           v22.4s,  v18.4h,  v18.4h

        umull2          v23.4s,  v16.8h,  v16.8h
        umlal2          v23.4s,  v17.8h,  v17.8h
        umlal2          v23.4s,  v18.8h,  v18.8h

        add             v21.8h,  v21.8h,  v20.8h
        st1             {v20.8h},        [x1], #16
        st1             {v22.4s,v23.4s}, [x0], #32

        umlal           v22.4s,  v0.4h,   v0.4h
        umlal           v22.4s,  v19.4h,  v19.4h

        umlal2          v23.4s,  v0.8h,   v0.8h
        umlal2          v23.4s,  v19.8h,  v19.8h

        subs            w6,  w6,  #8

        st1             {v21.8h},        [x3], #16
        st1             {v22.4s,v23.4s}, [x2], #32

        b.le            9f
        tst             w7,  #2 // LR_HAVE_RIGHT
        mov             v0.16b,  v1.16b
        ld1             {v1.8h}, [x5], #16

        b.ne            4b // If we don't need to pad, just keep summing.
        b               3b // If we need to pad, check how many pixels we have left.

9:
        ret
endfunc

sgr_funcs 16