xref: /aosp_15_r20/external/libdav1d/src/arm/32/looprestoration16.S (revision c09093415860a1c2373dacd84c4fde00c507cdfd)

/*
 * Copyright © 2018, VideoLAN and dav1d authors
 * Copyright © 2020, Martin Storsjo
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 *    list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "src/arm/asm.S"
#include "util.S"

const right_ext_mask_buf
        .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
        .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
        .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
        .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
right_ext_mask:
        .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
        .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
        .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
        .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
endconst

// void dav1d_wiener_filter_h_16bpc_neon(int16_t *dst, const pixel (*left)[4],
//                                       const pixel *src, ptrdiff_t stride,
//                                       const int16_t fh[7], const intptr_t w,
//                                       int h, enum LrEdgeFlags edges,
//                                       const int bitdepth_max);
function wiener_filter_h_16bpc_neon, export=1
        push            {r4-r11,lr}
        vpush           {q4-q7}
        ldrd            r4,  r5,  [sp, #100]
        ldrd            r6,  r7,  [sp, #108]
        ldr             r8,       [sp, #116] // bitdepth_max
        vld1.16         {q0}, [r4, :128]
        clz             r8,  r8
        vmov.i32        q14, #1
        sub             r9,  r8,  #38  // -(bitdepth + 6)
        sub             r8,  r8,  #25  // -round_bits_h
        neg             r9,  r9        // bitdepth + 6
        vdup.32         q1,  r9
        vdup.32         q13, r8        // -round_bits_h
        vmov.i16        q15, #8192
        vshl.u32        q14, q14, q1   // 1 << (bitdepth + 6)
        mov             r8,  r5
        // Calculate mid_stride
        add             r10, r5,  #7
        bic             r10, r10, #7
        lsl             r10, r10, #1

        // Set up pointers for reading/writing alternate rows
        add             r12, r0,  r10
        lsl             r10, r10, #1
        add             lr,  r2,  r3
        lsl             r3,  r3,  #1

        // Subtract the aligned width from mid_stride
        add             r11, r5,  #7
        bic             r11, r11, #7
        sub             r10, r10, r11, lsl #1

        // Subtract the number of pixels read from the source stride
        add             r11, r11, #8
        sub             r3,  r3,  r11, lsl #1

        // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
        tst             r7,  #1 // LR_HAVE_LEFT
        beq             2f
        // LR_HAVE_LEFT
        cmp             r1,  #0
        bne             0f
        // left == NULL
        sub             r2,  r2,  #6
        sub             lr,  lr,  #6
        b               1f
0:      // LR_HAVE_LEFT, left != NULL
2:      // !LR_HAVE_LEFT, increase the stride.
        // For this case we don't read the left 3 pixels from the src pointer,
        // but shift it as if we had done that.
        add             r3,  r3,  #6


1:      // Loop vertically
        vld1.16         {q2, q3}, [r2]!
        vld1.16         {q4, q5}, [lr]!

        tst             r7,  #1 // LR_HAVE_LEFT
        beq             0f
        cmp             r1,  #0
        beq             2f
        // LR_HAVE_LEFT, left != NULL
        vld1.16         {d3},  [r1]!
        // Move r2/lr back to account for the last 3 pixels we loaded earlier,
        // which we'll shift out.
        sub             r2,  r2,  #6
        sub             lr,  lr,  #6
        vld1.16         {d13}, [r1]!
        vext.8          q3,  q2,  q3,  #10
        vext.8          q2,  q1,  q2,  #10
        vext.8          q5,  q4,  q5,  #10
        vext.8          q4,  q6,  q4,  #10
        b               2f
0:
        // !LR_HAVE_LEFT, fill q1 with the leftmost pixel
        // and shift q2/q3 to have 3x the first pixel at the front.
        vdup.16         q1,  d4[0]
        vdup.16         q6,  d8[0]
        // Move r2 back to account for the last 3 pixels we loaded before,
        // which we shifted out.
        sub             r2,  r2,  #6
        sub             lr,  lr,  #6
        vext.8          q3,  q2,  q3,  #10
        vext.8          q2,  q1,  q2,  #10
        vext.8          q5,  q4,  q5,  #10
        vext.8          q4,  q6,  q4,  #10

2:

        tst             r7,  #2 // LR_HAVE_RIGHT
        bne             4f
        // If we'll need to pad the right edge, load that pixel to pad with
        // here since we can find it pretty easily from here.
        sub             r9,  r5,  #14
        lsl             r9,  r9,  #1
        ldrh            r11, [r2, r9]
        ldrh            r9,  [lr, r9]
        // Fill q11/q12 with the right padding pixel
        vdup.16         q11, r11
        vdup.16         q12, r9
3:      // !LR_HAVE_RIGHT

        // Check whether we need to pad the right edge
        cmp             r5,  #11
        bge             4f   // If w >= 11, all used input pixels are valid

        // 1 <= w < 11, w+3 pixels valid in q2-q3. For w=9 or w=10,
        // this ends up called again; it's not strictly needed in those
        // cases (we pad enough here), but keeping the code as simple as possible.

        // Insert padding in q2/3.h[w+3] onwards; fuse the +3 (*2) into the
        // buffer pointer.
        movrel_local    r4,  right_ext_mask, -6
        sub             r4,  r4,  r5,  lsl #1
        vld1.8          {q9, q10}, [r4]

        vbit            q2,  q11, q9
        vbit            q3,  q11, q10
        vbit            q4,  q12, q9
        vbit            q5,  q12, q10

4:      // Loop horizontally
        vext.8          q7,  q2,  q3,  #4
        vext.8          q8,  q2,  q3,  #8
        vext.8          q6,  q2,  q3,  #2
        vext.8          q9,  q2,  q3,  #10
        vadd.i16        q8,  q8,  q7
        vadd.i16        q9,  q9,  q6
        vext.8          q6,  q2,  q3,  #12
        vext.8          q7,  q2,  q3,  #6
        vadd.i16        q2,  q2,  q6
        vmull.s16       q6,  d14, d0[3]
        vmlal.s16       q6,  d16, d1[0]
        vmlal.s16       q6,  d18, d1[1]
        vmlal.s16       q6,  d4,  d1[2]
        vmull.s16       q7,  d15, d0[3]
        vmlal.s16       q7,  d17, d1[0]
        vmlal.s16       q7,  d19, d1[1]
        vmlal.s16       q7,  d5,  d1[2]

        vext.8          q8,  q4,  q5,  #4
        vext.8          q10, q4,  q5,  #8
        vext.8          q9,  q4,  q5,  #2
        vext.8          q2,  q4,  q5,  #10
        vadd.i16        q10, q10, q8
        vadd.i16        q2,  q2,  q9
        vext.8          q8,  q4,  q5,  #12
        vext.8          q9,  q4,  q5,  #6
        vadd.i16        q4,  q4,  q8
        vmull.s16       q8,  d18, d0[3]
        vmlal.s16       q8,  d20, d1[0]
        vmlal.s16       q8,  d4,  d1[1]
        vmlal.s16       q8,  d8,  d1[2]
        vmull.s16       q9,  d19, d0[3]
        vmlal.s16       q9,  d21, d1[0]
        vmlal.s16       q9,  d5,  d1[1]
        vmlal.s16       q9,  d9,  d1[2]

        vmvn.i16        q10, #0x8000 // 0x7fff = (1 << 15) - 1
        vadd.i32        q6,  q6,  q14
        vadd.i32        q7,  q7,  q14
        vadd.i32        q8,  q8,  q14
        vadd.i32        q9,  q9,  q14
        vrshl.s32       q6,  q6,  q13
        vrshl.s32       q7,  q7,  q13
        vrshl.s32       q8,  q8,  q13
        vrshl.s32       q9,  q9,  q13
        vqmovun.s32     d12, q6
        vqmovun.s32     d13, q7
        vqmovun.s32     d14, q8
        vqmovun.s32     d15, q9
        vmin.u16        q6,  q6,  q10
        vmin.u16        q7,  q7,  q10
        vsub.i16        q6,  q6,  q15
        vsub.i16        q7,  q7,  q15
        subs            r5,  r5,  #8
        vst1.16         {q6}, [r0,  :128]!
        vst1.16         {q7}, [r12, :128]!

        ble             9f
        tst             r7,  #2 // LR_HAVE_RIGHT
        vmov            q2,  q3
        vmov            q4,  q5
        vld1.16         {q3}, [r2]!
        vld1.16         {q5}, [lr]!
        bne             4b // If we don't need to pad, just keep filtering.
        b               3b // If we need to pad, check how many pixels we have left.

9:
        subs            r6,  r6,  #2
        ble             0f
        // Jump to the next row and loop horizontally
        add             r0,  r0,  r10
        add             r12, r12, r10
        add             r2,  r2,  r3
        add             lr,  lr,  r3
        mov             r5,  r8
        b               1b
0:
        vpop            {q4-q7}
        pop             {r4-r11,pc}
endfunc

// void dav1d_wiener_filter_v_16bpc_neon(pixel *dst, ptrdiff_t stride,
//                                       const int16_t *mid, int w, int h,
//                                       const int16_t fv[7], enum LrEdgeFlags edges,
//                                       ptrdiff_t mid_stride, const int bitdepth_max);
function wiener_filter_v_16bpc_neon, export=1
        push            {r4-r7,lr}
        vpush           {q4-q5}
        ldrd            r4,  r5,  [sp, #52]
        ldrd            r6,  r7,  [sp, #60]
        ldr             lr,       [sp, #68] // bitdepth_max
        vld1.16         {q0},  [r5, :128]
        vdup.16         q5,  lr
        clz             lr,  lr
        sub             lr,  lr,  #11   // round_bits_v
        vdup.32         q4,  lr
        mov             lr,  r4
        vneg.s32        q4,  q4         // -round_bits_v

        // Calculate the number of rows to move back when looping vertically
        mov             r12, r4
        tst             r6,  #4 // LR_HAVE_TOP
        beq             0f
        sub             r2,  r2,  r7, lsl #1
        add             r12, r12, #2
0:
        tst             r6,  #8 // LR_HAVE_BOTTOM
        beq             1f
        add             r12, r12, #2

1:      // Start of horizontal loop; start one vertical filter slice.
        // Load rows into q8-q11 and pad properly.
        tst             r6,  #4 // LR_HAVE_TOP
        vld1.16         {q8},  [r2, :128], r7
        beq             2f
        // LR_HAVE_TOP
        vld1.16         {q10}, [r2, :128], r7
        vmov            q9,  q8
        vld1.16         {q11}, [r2, :128], r7
        b               3f
2:      // !LR_HAVE_TOP
        vmov            q9,  q8
        vmov            q10, q8
        vmov            q11, q8

3:
        cmp             r4,  #4
        blt             5f
        // Start filtering normally; fill in q12-q14 with unique rows.
        vld1.16         {q12}, [r2, :128], r7
        vld1.16         {q13}, [r2, :128], r7
        vld1.16         {q14}, [r2, :128], r7

4:
.macro filter compare
        subs            r4,  r4,  #1
        // Interleaving the mul/mla chains actually hurts performance
        // significantly on Cortex A53, thus keeping mul/mla tightly
        // chained like this.
        vmull.s16       q2,  d16, d0[0]
        vmlal.s16       q2,  d18, d0[1]
        vmlal.s16       q2,  d20, d0[2]
        vmlal.s16       q2,  d22, d0[3]
        vmlal.s16       q2,  d24, d1[0]
        vmlal.s16       q2,  d26, d1[1]
        vmlal.s16       q2,  d28, d1[2]
        vmull.s16       q3,  d17, d0[0]
        vmlal.s16       q3,  d19, d0[1]
        vmlal.s16       q3,  d21, d0[2]
        vmlal.s16       q3,  d23, d0[3]
        vmlal.s16       q3,  d25, d1[0]
        vmlal.s16       q3,  d27, d1[1]
        vmlal.s16       q3,  d29, d1[2]
        vrshl.s32       q2,  q2,  q4    // round_bits_v
        vrshl.s32       q3,  q3,  q4
        vqmovun.s32     d4,  q2
        vqmovun.s32     d5,  q3
        vmin.u16        q2,  q2,  q5    // bitdepth_max
        vst1.16         {q2}, [r0, :128], r1
.if \compare
        cmp             r4,  #4
.else
        ble             9f
.endif
        vmov            q8,  q9
        vmov            q9,  q10
        vmov            q10, q11
        vmov            q11, q12
        vmov            q12, q13
        vmov            q13, q14
.endm
        filter          1
        blt             7f
        vld1.16         {q14}, [r2, :128], r7
        b               4b

5:      // Less than 4 rows in total; not all of q12-q13 are filled yet.
        tst             r6,  #8 // LR_HAVE_BOTTOM
        beq             6f
        // LR_HAVE_BOTTOM
        cmp             r4,  #2
        // We load at least 2 rows in all cases.
        vld1.16         {q12}, [r2, :128], r7
        vld1.16         {q13}, [r2, :128], r7
        bgt             53f // 3 rows in total
        beq             52f // 2 rows in total
51:     // 1 row in total, q11 already loaded, load edge into q12-q14.
        vmov            q13, q12
        b               8f
52:     // 2 rows in total, q11 already loaded, load q12 with content data
        // and 2 rows of edge.
        vld1.16         {q14}, [r2, :128], r7
        vmov            q15, q14
        b               8f
53:
        // 3 rows in total, q11 already loaded, load q12 and q13 with content
        // and 2 rows of edge.
        vld1.16         {q14}, [r2, :128], r7
        vld1.16         {q15}, [r2, :128], r7
        vmov            q1,  q15
        b               8f

6:
        // !LR_HAVE_BOTTOM
        cmp             r4,  #2
        bgt             63f // 3 rows in total
        beq             62f // 2 rows in total
61:     // 1 row in total, q11 already loaded, pad that into q12-q14.
        vmov            q12, q11
        vmov            q13, q11
        vmov            q14, q11
        b               8f
62:     // 2 rows in total, q11 already loaded, load q12 and pad that into q12-q15.
        vld1.16         {q12}, [r2, :128], r7
        vmov            q13, q12
        vmov            q14, q12
        vmov            q15, q12
        b               8f
63:
        // 3 rows in total, q11 already loaded, load q12 and q13 and pad q13 into q14-q15,q1.
        vld1.16         {q12}, [r2, :128], r7
        vld1.16         {q13}, [r2, :128], r7
        vmov            q14, q13
        vmov            q15, q13
        vmov            q1,  q13
        b               8f

7:
        // All registers up to q13 are filled already, 3 valid rows left.
        // < 4 valid rows left; fill in padding and filter the last
        // few rows.
        tst             r6,  #8 // LR_HAVE_BOTTOM
        beq             71f
        // LR_HAVE_BOTTOM; load 2 rows of edge.
        vld1.16         {q14}, [r2, :128], r7
        vld1.16         {q15}, [r2, :128], r7
        vmov            q1,  q15
        b               8f
71:
        // !LR_HAVE_BOTTOM, pad 3 rows
        vmov            q14, q13
        vmov            q15, q13
        vmov            q1,  q13

8:      // At this point, all registers up to q14-q15,q1 are loaded with
        // edge/padding (depending on how many rows are left).
        filter          0 // This branches to 9f when done
        vmov            q14, q15
        vmov            q15, q1
        b               8b

9:      // End of one vertical slice.
        subs            r3,  r3,  #8
        ble             0f
        // Move pointers back up to the top and loop horizontally.
        mls             r0,  r1,  lr,  r0
        mls             r2,  r7,  r12, r2
        add             r0,  r0,  #16
        add             r2,  r2,  #16
        mov             r4,  lr
        b               1b

0:
        vpop            {q4-q5}
        pop             {r4-r7,pc}
.purgem filter
endfunc

#define SUM_STRIDE (384+16)

#include "looprestoration_tmpl.S"

// void dav1d_sgr_box3_h_16bpc_neon(int32_t *sumsq, int16_t *sum,
//                                  const pixel (*left)[4],
//                                  const pixel *src, const ptrdiff_t stride,
//                                  const int w, const int h,
//                                  const enum LrEdgeFlags edges);
function sgr_box3_h_16bpc_neon, export=1
        push            {r4-r11,lr}
        vpush           {q4-q7}
        ldrd            r4,  r5,  [sp, #100]
        ldrd            r6,  r7,  [sp, #108]
        add             r5,  r5,  #2 // w += 2

        // Set up pointers for reading/writing alternate rows
        add             r10, r0,  #(4*SUM_STRIDE)   // sumsq
        add             r11, r1,  #(2*SUM_STRIDE)   // sum
        add             r12, r3,  r4                // src
        lsl             r4,  r4,  #1
        mov             r9,       #(2*2*SUM_STRIDE) // double sum stride

        // Subtract the aligned width from the output stride.
        add             lr,  r5,  #7
        bic             lr,  lr,  #7
        sub             r9,  r9,  lr, lsl #1

        // Store the width for the vertical loop
        mov             r8,  r5

        // Subtract the number of pixels read from the input from the stride
        add             lr,  lr,  #8
        sub             r4,  r4,  lr, lsl #1

        // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
        tst             r7,  #1 // LR_HAVE_LEFT
        beq             2f
        // LR_HAVE_LEFT
        cmp             r2,  #0
        bne             0f
        // left == NULL
        sub             r3,  r3,  #4
        sub             r12, r12, #4
        b               1f
0:      // LR_HAVE_LEFT, left != NULL
2:      // !LR_HAVE_LEFT, increase the stride.
        // For this case we don't read the left 2 pixels from the src pointer,
        // but shift it as if we had done that.
        add             r4,  r4,  #4


1:      // Loop vertically
        vld1.16         {q0, q1}, [r3]!
        vld1.16         {q4, q5}, [r12]!

        tst             r7,  #1 // LR_HAVE_LEFT
        beq             0f
        cmp             r2,  #0
        beq             2f
        // LR_HAVE_LEFT, left != NULL
        vld1.16         {d5}, [r2]!
        // Move r3/r12 back to account for the last 2 pixels we loaded earlier,
        // which we'll shift out.
        sub             r3,  r3,  #4
        sub             r12, r12, #4
        vld1.16         {d13}, [r2]!
        vext.8          q1,  q0,  q1,  #12
        vext.8          q0,  q2,  q0,  #12
        vext.8          q5,  q4,  q5,  #12
        vext.8          q4,  q6,  q4,  #12
        b               2f
0:
        // !LR_HAVE_LEFT, fill q2 with the leftmost pixel
        // and shift q0 to have 2x the first byte at the front.
        vdup.16         q2,  d0[0]
        vdup.16         q6,  d8[0]
        // Move r3 back to account for the last 2 pixels we loaded before,
        // which we shifted out.
        sub             r3,  r3,  #4
        sub             r12, r12, #4
        vext.8          q1,  q0,  q1,  #12
        vext.8          q0,  q2,  q0,  #12
        vext.8          q5,  q4,  q5,  #12
        vext.8          q4,  q6,  q4,  #12

2:
        tst             r7,  #2 // LR_HAVE_RIGHT
        bne             4f
        // If we'll need to pad the right edge, load that pixel to pad with
        // here since we can find it pretty easily from here.
        sub             lr,  r5,  #(2 + 16 - 2 + 1)
        lsl             lr,  lr,  #1
        ldrh            r11, [r3,  lr]
        ldrh            lr,  [r12, lr]
        // Fill q14/q15 with the right padding pixel
        vdup.16         q14, r11
        vdup.16         q15, lr
        // Restore r11 after using it for a temporary value
        add             r11, r1,  #(2*SUM_STRIDE)
3:      // !LR_HAVE_RIGHT

        // Check whether we need to pad the right edge
        cmp             r5,  #10
        bge             4f   // If w >= 10, all used input pixels are valid

        // 1 <= w < 10, w pixels valid in q0-q1. For w=9, this ends up called
        // again; it's not strictly needed in those cases (we pad enough here),
        // but keeping the code as simple as possible.

        // Insert padding in q0/1.h[w] onwards
        movrel_local    lr,  right_ext_mask
        sub             lr,  lr,  r5,  lsl #1
        vld1.8          {q12, q13}, [lr]

        vbit            q0,  q14, q12
        vbit            q1,  q14, q13
        vbit            q4,  q15, q12
        vbit            q5,  q15, q13

4:      // Loop horizontally
        vext.8          q8,  q0,  q1,  #2
        vext.8          q10, q4,  q5,  #2
        vext.8          q9,  q0,  q1,  #4
        vext.8          q11, q4,  q5,  #4
        vadd.i16        q2,  q0,  q8
        vadd.i16        q3,  q4,  q10
        vadd.i16        q2,  q2,  q9
        vadd.i16        q3,  q3,  q11

        vmull.u16       q6,  d0,  d0
        vmlal.u16       q6,  d16, d16
        vmlal.u16       q6,  d18, d18
        vmull.u16       q12, d8,  d8
        vmlal.u16       q12, d20, d20
        vmlal.u16       q12, d22, d22
        vmull.u16       q7,  d1,  d1
        vmlal.u16       q7,  d17, d17
        vmlal.u16       q7,  d19, d19
        vmull.u16       q13, d9,  d9
        vmlal.u16       q13, d21, d21
        vmlal.u16       q13, d23, d23
        subs            r5,  r5,  #8
        vst1.16         {q2},       [r1,  :128]!
        vst1.16         {q3},       [r11, :128]!
        vst1.32         {q6,  q7},  [r0,  :128]!
        vst1.32         {q12, q13}, [r10, :128]!

        ble             9f
        tst             r7,  #2 // LR_HAVE_RIGHT
        vmov            q0,  q1
        vmov            q4,  q5
        vld1.16         {q1}, [r3]!
        vld1.16         {q5}, [r12]!

        bne             4b // If we don't need to pad, just keep summing.
        b               3b // If we need to pad, check how many pixels we have left.

9:
        subs            r6,  r6,  #2
        ble             0f
        // Jump to the next row and loop horizontally
        add             r0,  r0,  r9, lsl #1
        add             r10, r10, r9, lsl #1
        add             r1,  r1,  r9
        add             r11, r11, r9
        add             r3,  r3,  r4
        add             r12, r12, r4
        mov             r5,  r8
        b               1b
0:
        vpop            {q4-q7}
        pop             {r4-r11,pc}
endfunc

// void dav1d_sgr_box5_h_16bpc_neon(int32_t *sumsq, int16_t *sum,
//                                  const pixel (*left)[4],
//                                  const pixel *src, const ptrdiff_t stride,
//                                  const int w, const int h,
//                                  const enum LrEdgeFlags edges);
function sgr_box5_h_16bpc_neon, export=1
        push            {r4-r11,lr}
        vpush           {q4-q7}
        ldrd            r4,  r5,  [sp, #100]
        ldrd            r6,  r7,  [sp, #108]
        add             r5,  r5,  #2 // w += 2

        // Set up pointers for reading/writing alternate rows
        add             r10, r0,  #(4*SUM_STRIDE)   // sumsq
        add             r11, r1,  #(2*SUM_STRIDE)   // sum
        add             r12, r3,  r4                // src
        lsl             r4,  r4,  #1
        mov             r9,       #(2*2*SUM_STRIDE) // double sum stride

        // Subtract the aligned width from the output stride.
        add             lr,  r5,  #7
        bic             lr,  lr,  #7
        sub             r9,  r9,  lr, lsl #1
        add             lr,  lr,  #8
        sub             r4,  r4,  lr, lsl #1

        // Store the width for the vertical loop
        mov             r8,  r5

        // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
        tst             r7,  #1 // LR_HAVE_LEFT
        beq             2f
        // LR_HAVE_LEFT
        cmp             r2,  #0
        bne             0f
        // left == NULL
        sub             r3,  r3,  #6
        sub             r12, r12, #6
        b               1f
0:      // LR_HAVE_LEFT, left != NULL
2:      // !LR_HAVE_LEFT, increase the stride.
        // For this case we don't read the left 3 pixels from the src pointer,
        // but shift it as if we had done that.
        add             r4,  r4,  #6

1:      // Loop vertically
        vld1.16         {q0, q1}, [r3]!
        vld1.16         {q4, q5}, [r12]!

        tst             r7,  #1 // LR_HAVE_LEFT
        beq             0f
        cmp             r2,  #0
        beq             2f
        // LR_HAVE_LEFT, left != NULL
        vld1.16         {d5}, [r2]!
        // Move r3/r12 back to account for the last 3 pixels we loaded earlier,
        // which we'll shift out.
        sub             r3,  r3,  #6
        sub             r12, r12, #6
        vld1.16         {d13}, [r2]!
        vext.8          q1,  q0,  q1,  #10
        vext.8          q0,  q2,  q0,  #10
        vext.8          q5,  q4,  q5,  #10
        vext.8          q4,  q6,  q4,  #10
        b               2f
0:
        // !LR_HAVE_LEFT, fill q2 with the leftmost pixel
        // and shift q0 to have 3x the first pixel at the front.
        vdup.16         q2,  d0[0]
        vdup.16         q6,  d8[0]
        // Move r3 back to account for the last 3 pixels we loaded before,
        // which we shifted out.
        sub             r3,  r3,  #6
        sub             r12, r12, #6
        vext.8          q1,  q0,  q1,  #10
        vext.8          q0,  q2,  q0,  #10
        vext.8          q5,  q4,  q5,  #10
        vext.8          q4,  q6,  q4,  #10

2:
        tst             r7,  #2 // LR_HAVE_RIGHT
        bne             4f
        // If we'll need to pad the right edge, load that pixel to pad with
        // here since we can find it pretty easily from here.
        sub             lr,  r5,  #(2 + 16 - 3 + 1)
        lsl             lr,  lr,  #1
        ldrh            r11, [r3,  lr]
        ldrh            lr,  [r12, lr]
        // Fill q14/q15 with the right padding pixel
        vdup.16         q14, r11
        vdup.16         q15, lr
        // Restore r11 after using it for a temporary value
        add             r11, r1,  #(2*SUM_STRIDE)
3:      // !LR_HAVE_RIGHT

        // Check whether we need to pad the right edge
        cmp             r5,  #11
        bge             4f   // If w >= 11, all used input pixels are valid

        // 1 <= w < 11, w+1 pixels valid in q0-q1. For w=9 or w=10,
        // this ends up called again; it's not strictly needed in those
        // cases (we pad enough here), but keeping the code as simple as possible.

        // Insert padding in q0/1.h[w+1] onwards; fuse the +1 into the
        // buffer pointer.
        movrel_local    lr,  right_ext_mask, -2
        sub             lr,  lr,  r5,  lsl #1
        vld1.8          {q12, q13}, [lr]

        vbit            q0,  q14, q12
        vbit            q1,  q14, q13
        vbit            q4,  q15, q12
        vbit            q5,  q15, q13

4:      // Loop horizontally
        vext.8          q8,  q0,  q1,  #2
        vext.8          q10, q4,  q5,  #2
        vext.8          q9,  q0,  q1,  #4
        vext.8          q11, q4,  q5,  #4
        vadd.i16        q2,  q0,  q8
        vadd.i16        q3,  q4,  q10
        vadd.i16        q2,  q2,  q9
        vadd.i16        q3,  q3,  q11

        vmull.u16       q6,  d0,  d0
        vmlal.u16       q6,  d16, d16
        vmlal.u16       q6,  d18, d18
        vmull.u16       q12, d8,  d8
        vmlal.u16       q12, d20, d20
        vmlal.u16       q12, d22, d22
        vmull.u16       q7,  d1,  d1
        vmlal.u16       q7,  d17, d17
        vmlal.u16       q7,  d19, d19
        vmull.u16       q13, d9,  d9
        vmlal.u16       q13, d21, d21
        vmlal.u16       q13, d23, d23

        vext.8          q8,  q0,  q1,  #6
        vext.8          q10, q4,  q5,  #6
        vext.8          q9,  q0,  q1,  #8
        vext.8          q11, q4,  q5,  #8
        vadd.i16        q2,  q2,  q8
        vadd.i16        q3,  q3,  q10
        vadd.i16        q2,  q2,  q9
        vadd.i16        q3,  q3,  q11

        vmlal.u16       q6,  d16, d16
        vmlal.u16       q6,  d1,  d1
        vmlal.u16       q12, d20, d20
        vmlal.u16       q12, d9,  d9
        vmlal.u16       q7,  d17, d17
        vmlal.u16       q7,  d19, d19
        vmlal.u16       q13, d21, d21
        vmlal.u16       q13, d23, d23

        subs            r5,  r5,  #8
        vst1.16         {q2},       [r1,  :128]!
        vst1.16         {q3},       [r11, :128]!
        vst1.32         {q6,  q7},  [r0,  :128]!
        vst1.32         {q12, q13}, [r10, :128]!

        ble             9f
        tst             r7,  #2 // LR_HAVE_RIGHT
        vmov            q0,  q1
        vmov            q4,  q5
        vld1.16         {q1}, [r3]!
        vld1.16         {q5}, [r12]!
        bne             4b // If we don't need to pad, just keep summing.
        b               3b // If we need to pad, check how many pixels we have left.

9:
        subs            r6,  r6,  #2
        ble             0f
        // Jump to the next row and loop horizontally
        add             r0,  r0,  r9, lsl #1
        add             r10, r10, r9, lsl #1
        add             r1,  r1,  r9
        add             r11, r11, r9
        add             r3,  r3,  r4
        add             r12, r12, r4
        mov             r5,  r8
        b               1b
0:
        vpop            {q4-q7}
        pop             {r4-r11,pc}
endfunc

sgr_funcs 16