xref: /aosp_15_r20/external/libdav1d/src/arm/32/filmgrain.S (revision c09093415860a1c2373dacd84c4fde00c507cdfd)

/*
 * Copyright © 2021, VideoLAN and dav1d authors
 * Copyright © 2021, Martin Storsjo
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 *    list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "src/arm/asm.S"
#include "util.S"
#include "src/arm/asm-offsets.h"

#define GRAIN_WIDTH 82
#define GRAIN_HEIGHT 73

#define SUB_GRAIN_WIDTH 44
#define SUB_GRAIN_HEIGHT 38

.macro increment_seed steps, shift=1
        lsr             r11, r2,  #3
        lsr             r12, r2,  #12
        lsr             lr,  r2,  #1
        eor             r11, r2,  r11                     // (r >> 0) ^ (r >> 3)
        eor             r12, r12, lr                      // (r >> 12) ^ (r >> 1)
        eor             r11, r11, r12                     // (r >> 0) ^ (r >> 3) ^ (r >> 12) ^ (r >> 1)
.if \shift
        lsr             r2,  r2,  #\steps
.endif
        and             r11, r11, #((1 << \steps) - 1)    // bit
.if \shift
        orr             r2,  r2,  r11, lsl #(16 - \steps) // *state
.else
        orr             r2,  r2,  r11, lsl #16            // *state
.endif
.endm

.macro read_rand dest, bits, age
        ubfx            \dest,  r2,   #16 - \bits - \age, #\bits
.endm

.macro read_shift_rand dest, bits
        ubfx            \dest,  r2,   #17 - \bits, #\bits
        lsr             r2,  r2,  #1
.endm

// special calling convention:
// r2 holds seed
// r3 holds dav1d_gaussian_sequence
// clobbers r11-r12
// returns in d0-d1
function get_gaussian_neon
        push            {r5-r6,lr}
        increment_seed  4
        read_rand       r5,  11,  3
        read_rand       r6,  11,  2
        add             r5,  r3,  r5,  lsl #1
        add             r6,  r3,  r6,  lsl #1
        vld1.16         {d0[0]}, [r5]
        read_rand       r5,  11,  1
        vld1.16         {d0[1]}, [r6]
        add             r5,  r3,  r5,  lsl #1
        read_rand       r6, 11,  0
        increment_seed  4
        add             r6,  r3,  r6,  lsl #1
        vld1.16         {d0[2]}, [r5]
        read_rand       r5,  11,  3
        vld1.16         {d0[3]}, [r6]
        add             r5,  r3,  r5,  lsl #1
        read_rand       r6,  11,  2
        vld1.16         {d1[0]}, [r5]
        add             r6,  r3,  r6,  lsl #1
        read_rand       r5,  11,  1
        vld1.16         {d1[1]}, [r6]
        read_rand       r6,  11,  0
        add             r5,  r3,  r5,  lsl #1
        add             r6,  r3,  r6,  lsl #1
        vld1.16         {d1[2]}, [r5]
        vld1.16         {d1[3]}, [r6]
        pop             {r5-r6,pc}
endfunc

.macro get_grain_row r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10
        bl              get_gaussian_neon
        vrshl.s16       q0,  q0,  q15
        vmovn.i16       \r0, q0
        bl              get_gaussian_neon
        vrshl.s16       q0,  q0,  q15
        vmovn.i16       \r1, q0
        bl              get_gaussian_neon
        vrshl.s16       q0,  q0,  q15
        vmovn.i16       \r2, q0
        bl              get_gaussian_neon
        vrshl.s16       q0,  q0,  q15
        vmovn.i16       \r3, q0
        bl              get_gaussian_neon
        vrshl.s16       q0,  q0,  q15
        vmovn.i16       \r4, q0
        bl              get_gaussian_neon
        vrshl.s16       q0,  q0,  q15
        vmovn.i16       \r5, q0
        bl              get_gaussian_neon
        vrshl.s16       q0,  q0,  q15
        vmovn.i16       \r6, q0
        bl              get_gaussian_neon
        vrshl.s16       q0,  q0,  q15
        vmovn.i16       \r7, q0
        bl              get_gaussian_neon
        vrshl.s16       q0,  q0,  q15
        vmovn.i16       \r8, q0
        bl              get_gaussian_neon
        vrshl.s16       q0,  q0,  q15
        vmovn.i16       \r9, q0
        increment_seed  2
        read_rand       r11, 11,  1
        read_rand       r12, 11,  0
        add             r11, r3,  r11, lsl #1
        add             r12, r3,  r12, lsl #1
        vld1.16         {d0[0]}, [r11]
        vld1.16         {d0[1]}, [r12]
        vrshl.s16       d0,  d0,  d30
        vmovn.i16       \r10, q0
.endm

.macro store_grain_row r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10
        vst1.16         {\r0, \r1, \r2, \r3}, [r0]!
        vst1.16         {\r4, \r5, \r6, \r7}, [r0]!
        vst1.16         {\r8, \r9},           [r0]!
        vst1.16         {\r10[0]},            [r0]!
.endm

.macro get_grain_row_44 r0, r1, r2, r3, r4, r5
        bl              get_gaussian_neon
        vrshl.s16       q0,  q0,  q15
        vmovn.i16       \r0, q0
        bl              get_gaussian_neon
        vrshl.s16       q0,  q0,  q15
        vmovn.i16       \r1, q0
        bl              get_gaussian_neon
        vrshl.s16       q0,  q0,  q15
        vmovn.i16       \r2, q0
        bl              get_gaussian_neon
        vrshl.s16       q0,  q0,  q15
        vmovn.i16       \r3, q0
        bl              get_gaussian_neon
        vrshl.s16       q0,  q0,  q15
        vmovn.i16       \r4, q0
        increment_seed  4
        read_rand       r11, 11,  3
        read_rand       r12, 11,  2
        add             r11, r3,  r11, lsl #1
        add             r12, r3,  r12, lsl #1
        vld1.16         {d0[]}, [r11]
        read_rand       r11, 11,  1
        vld1.16         {d0[1]}, [r12]
        add             r11, r3,  r11, lsl #1
        read_rand       r12, 11,  0
        vld1.16         {d0[2]}, [r11]
        add             r12, r3,  r12, lsl #1
        vld1.16         {d0[3]}, [r12]
        vrshl.s16       d0,  d0,  d30
        vmovn.i16       \r5, q0
.endm

.macro store_grain_row_44 r0, r1, r2, r3, r4, r5
        vst1.16         {\r0, \r1, \r2, \r3}, [r0]!
        vst1.16         {\r4, \r5},           [r0]
        add             r0,  r0,  #GRAIN_WIDTH-32
.endm

function get_grain_2_neon
        push            {r11,lr}
        increment_seed  2
        read_rand       r11, 11,  1
        read_rand       r12, 11,  0
        add             r11, r3,  r11, lsl #1
        add             r12, r3,  r12, lsl #1
        vld1.16         {d0[0]}, [r11]
        vld1.16         {d0[1]}, [r12]
        vrshl.s16       d0,  d0,  d30
        vmovn.i16       d0,  q0
        pop             {r11,pc}
endfunc

.macro get_grain_2 dst
        bl              get_grain_2_neon
.ifnc \dst, d0
        vmov            \dst, d0
.endif
.endm

// r1 holds the number of entries to produce
// r6, r8 and r10 hold the previous output entries
// q0 holds the vector of produced entries
// q1 holds the input vector of sums from above
.macro output_lag n
function output_lag\n\()_neon
        push            {r0, lr}
.if \n == 1
        mov             lr,  #-128
.else
        mov             r0,  #1
        mov             lr,  #1
        sub             r7,  r7,  #1
        sub             r9,  r9,  #1
        lsl             r0,  r0,  r7
        lsl             lr,  lr,  r9
        add             r7,  r7,  #1
        add             r9,  r9,  #1
.endif
1:
        read_shift_rand r12, 11
        vmov.32         r11, d2[0]
        lsl             r12, r12, #1
        vext.8          q0,  q0,  q0,  #1
        ldrsh           r12, [r3, r12]
.if \n == 1
        mla             r11, r6,  r4,  r11        // sum (above) + *coeff * prev output
        add             r6,  r11, r8              // 1 << (ar_coeff_shift - 1)
        add             r12, r12, r10
        asr             r6,  r6,  r7              // >> ar_coeff_shift
        asr             r12, r12, r9              // >> (4 + grain_scale_shift)
        add             r6,  r6,  r12
        cmp             r6,  r5
.elseif \n == 2
        mla             r11, r8,  r4,  r11        // sum (above) + *coeff * prev output 1
        mla             r11, r6,  r10, r11        // += *coeff * prev output 2
        mov             r8,  r6
        add             r6,  r11, r0              // 1 << (ar_coeff_shift - 1)
        add             r12, r12, lr              // 1 << (4 + grain_scale_shift - 1)
        asr             r6,  r6,  r7              // >> ar_coeff_shift
        asr             r12, r12, r9              // >> (4 + grain_scale_shift)
        add             r6,  r6,  r12
        push            {lr}
        cmp             r6,  r5
        mov             lr,  #-128
.else
        push            {r1-r3}
        sbfx            r1,  r4,  #0,  #8
        sbfx            r2,  r4,  #8,  #8
        sbfx            r3,  r4,  #16, #8
        mla             r11, r10, r1,  r11        // sum (above) + *coeff * prev output 1
        mla             r11, r8,  r2,  r11        // sum (above) + *coeff * prev output 2
        mla             r11, r6,  r3,  r11        // += *coeff * prev output 3
        pop             {r1-r3}
        mov             r10, r8
        mov             r8,  r6

        add             r6,  r11, r0              // 1 << (ar_coeff_shift - 1)
        add             r12, r12, lr              // 1 << (4 + grain_scale_shift - 1)
        asr             r6,  r6,  r7              // >> ar_coeff_shift
        asr             r12, r12, r9              // >> (4 + grain_scale_shift)
        add             r6,  r6,  r12
        push            {lr}
        cmp             r6,  r5
        mov             lr,  #-128
.endif
        it              gt
        movgt           r6,  r5
        cmp             r6,  lr
        it              lt
        movlt           r6,  lr
.if \n >= 2
        pop             {lr}
.endif
        subs            r1,  r1,  #1
        vext.8          q1,  q1,  q1,  #4
        vmov.8          d1[7], r6
        bgt             1b
        pop             {r0, pc}
endfunc
.endm

output_lag 1
output_lag 2
output_lag 3


function sum_lag1_above_neon
        vmull.s8        q2,  d6,  d28
        vmull.s8        q3,  d7,  d28
        vmull.s8        q4,  d0,  d27
        vmull.s8        q5,  d1,  d27

        vaddl.s16       q0,  d4,  d8
        vaddl.s16       q2,  d5,  d9
        vaddl.s16       q4,  d6,  d10
        vaddl.s16       q5,  d7,  d11

        vmull.s8        q3,  d3,  d29
        vmull.s8        q1,  d2,  d29

        vaddw.s16       q4,  q4,  d6
        vaddw.s16       q5,  q5,  d7
        vaddw.s16       q3,  q2,  d3
        vaddw.s16       q2,  q0,  d2
        bx              lr
endfunc

.macro sum_lag_n_body lag, type, uv_layout, edge, elems, store, uv_coeff
.ifc \lag\()_\edge, lag3_left
        bl              sum_lag3_left_above_neon
.else
        bl              sum_\lag\()_above_neon
.endif
.ifc \type, uv_420
        vpush           {q6-q7}
        add             r12, r11, #GRAIN_WIDTH
        vld1.16         {q0, q1}, [r11]!
        vld1.16         {q6, q7}, [r12]!
        vpaddl.s8       q0,  q0
        vpaddl.s8       q1,  q1
        vpaddl.s8       q6,  q6
        vpaddl.s8       q7,  q7
        vadd.i16        q0,  q0,  q6
        vadd.i16        q1,  q1,  q7
        vpop            {q6-q7}
        vrshrn.s16      d0,  q0,  #2
        vrshrn.s16      d1,  q1,  #2
.endif
.ifc \type, uv_422
        vld1.8          {q0, q1}, [r11]!
        vpaddl.s8       q0,  q0
        vpaddl.s8       q1,  q1
        vrshrn.s16      d0,  q0,  #1
        vrshrn.s16      d1,  q1,  #1
.endif
.ifc \type, uv_444
        vld1.8          {q0}, [r11]!
.endif
.if \uv_layout
.ifnb \uv_coeff
        vdup.8          d13, \uv_coeff
.endif
        vmull.s8        q1,  d0,  d13
        vmull.s8        q0,  d1,  d13
        vaddw.s16       q2,  q2,  d2
        vaddw.s16       q3,  q3,  d3
        vaddw.s16       q4,  q4,  d0
        vaddw.s16       q5,  q5,  d1
.endif
.if \uv_layout && \elems == 16
        b               sum_\lag\()_y_\edge\()_start
.elseif \uv_layout == 444 && \elems == 15
        b               sum_\lag\()_y_\edge\()_start
.elseif \uv_layout == 422 && \elems == 9
        b               sum_\lag\()_uv_420_\edge\()_start
.else
sum_\lag\()_\type\()_\edge\()_start:
        push            {r11}
.ifc \edge, left
        increment_seed  4
        read_rand       r11, 11,  3
        read_rand       r12, 11,  2
        add             r11, r3,  r11, lsl #1
        add             r12, r3,  r12, lsl #1
        vld1.16         {d1[1]}, [r11]
        read_rand       r11, 11,  1
        vld1.16         {d1[2]}, [r12]
        add             r11, r3,  r11, lsl #1
        vld1.16         {d1[3]}, [r11]
        lsl             r2,  r2,  #1             // shift back the state as if we'd done increment_seed with shift=0
        vrshl.s16       d1,  d1,  d30
        vmovn.i16       d1,  q0
        vext.8          q2,  q2,  q2,  #12
.ifc \lag, lag3
        vmov.s8         r10, d1[5]
.endif
.ifnc \lag, lag1
        vmov.s8         r8,  d1[6]
.endif
        vmov.s8         r6,  d1[7]

        vmov            q1,  q2
        mov             r1,  #1
        bl              output_\lag\()_neon
.else
        increment_seed  4, shift=0
        vmov            q1,  q2
        mov             r1,  #4
        bl              output_\lag\()_neon
.endif

        increment_seed  4, shift=0
        vmov            q1,  q3
        mov             r1,  #4
        bl              output_\lag\()_neon

        increment_seed  4, shift=0
        vmov            q1,  q4
.if \elems == 9
        mov             r1,  #1
        bl              output_\lag\()_neon
        lsr             r2,  r2,  #3

        read_rand       r11, 11,  2
        read_rand       r12, 11,  1
        add             r11, r3,  r11, lsl #1
        add             r12, r3,  r12, lsl #1
        vld1.16         {d2[0]}, [r11]
        read_rand       r11, 11,  0
        vld1.16         {d2[1]}, [r12]
        add             r11, r3,  r11, lsl #1
        vld1.16         {d2[2]}, [r11]
        vrshl.s16       d2,  d2,  d30
        vmovn.i16       d2,  q1
        vext.8          q0,  q0,  q1,  #7
.else
        mov             r1,  #4
        bl              output_\lag\()_neon

        increment_seed  4, shift=0
        vmov            q1,  q5

.ifc \edge, right
        mov             r1,  #3
        bl              output_\lag\()_neon
        read_shift_rand r11, 11
        add             r11, r3,  r11, lsl #1
        vld1.16         {d2[0]}, [r11]
        vrshl.s16       d2,  d2,  d30
        vext.8          q0,  q0,  q1,  #1
.else
        mov             r1,  #4
        bl              output_\lag\()_neon
.endif
.endif
.if \store
        vst1.8          {q0}, [r0]!
.endif
        pop             {r11}
        pop             {r1, pc}
.endif
.endm

.macro sum_lag1_func type, uv_layout, edge, elems=16
function sum_\type\()_lag1_\edge\()_neon
        push            {r1, lr}
        sum_lag_n_body  lag1, \type, \uv_layout, \edge, \elems, store=0
endfunc
.endm

sum_lag1_func y,      0,   left
sum_lag1_func y,      0,   mid
sum_lag1_func y,      0,   right, 15
sum_lag1_func uv_444, 444, left
sum_lag1_func uv_444, 444, mid
sum_lag1_func uv_444, 444, right, 15
sum_lag1_func uv_422, 422, left
sum_lag1_func uv_422, 422, mid
sum_lag1_func uv_422, 422, right, 9
sum_lag1_func uv_420, 420, left
sum_lag1_func uv_420, 420, mid
sum_lag1_func uv_420, 420, right, 9

.macro sum_lag1 type, dst, left, mid, right, edge=mid
        vmov            q3,  \mid
        vext.8          q0,  \left, \mid,   #15
        vext.8          q1,  \mid,  \right, #1
        bl              sum_\type\()_lag1_\edge\()_neon
        vmov            \dst, q0
.endm

.macro sum_y_lag1 dst, left, mid, right, edge=mid
        sum_lag1        y, \dst, \left, \mid, \right, \edge
.endm

.macro sum_uv_444_lag1 dst, left, mid, right, edge=mid
        sum_lag1        uv_444, \dst, \left, \mid, \right, \edge
.endm

.macro sum_uv_422_lag1 dst, left, mid, right, edge=mid
        sum_lag1        uv_422, \dst, \left, \mid, \right, \edge
.endm

.macro sum_uv_420_lag1 dst, left, mid, right, edge=mid
        sum_lag1        uv_420, \dst, \left, \mid, \right, \edge
.endm


function sum_lag2_above_neon
        push            {lr}
        sub             r12, r0,  #2*GRAIN_WIDTH - 16
        sub             lr,  r0,  #1*GRAIN_WIDTH - 16
        vld1.8          {q10}, [r12] // load top right
        vld1.8          {q13}, [lr]

        vext.8          q6,  q8,  q9,  #14 // top left, top mid
        vdup.8          d14, d28[0]
        vext.8          q8,  q8,  q9,  #15
        vdup.8          d15, d28[1]

        vmull.s8        q0,  d12, d14
        vmull.s8        q1,  d13, d14
        vmull.s8        q6,  d16, d15
        vmull.s8        q8,  d17, d15

        vaddl.s16       q2,  d0,  d12
        vaddl.s16       q3,  d1,  d13
        vaddl.s16       q4,  d2,  d16
        vaddl.s16       q5,  d3,  d17

        vext.8          q6,  q9,  q10, #1  // top mid, top right
        vdup.8          d14, d28[3]
        vext.8          q8,  q9,  q10, #2
        vdup.8          d15, d28[4]

        vmull.s8        q0,  d12, d14
        vmull.s8        q1,  d13, d14
        vmull.s8        q6,  d16, d15
        vmull.s8        q8,  d17, d15

        vaddl.s16       q7,  d0,  d12
        vaddl.s16       q0,  d1,  d13
        vaddl.s16       q6,  d2,  d16
        vaddl.s16       q1,  d3,  d17

        vadd.i32        q2,  q2,  q7
        vadd.i32        q3,  q3,  q0
        vadd.i32        q4,  q4,  q6
        vadd.i32        q5,  q5,  q1

        vext.8          q6,  q11, q12, #14 // top left, top mid
        vdup.8          d14, d28[5]
        vext.8          q8,  q11, q12, #15
        vdup.8          d15, d28[6]

        vmull.s8        q0,  d12, d14
        vmull.s8        q1,  d13, d14
        vmull.s8        q6,  d16, d15
        vmull.s8        q8,  d17, d15

        vaddl.s16       q7,  d0,  d12
        vaddl.s16       q0,  d1,  d13
        vaddl.s16       q6,  d2,  d16
        vaddl.s16       q1,  d3,  d17

        vadd.i32        q2,  q2,  q7
        vadd.i32        q3,  q3,  q0
        vadd.i32        q4,  q4,  q6
        vadd.i32        q5,  q5,  q1

        vext.8          q6,  q12, q13, #1  // top mid, top right
        vdup.8          d14, d29[0]
        vext.8          q8,  q12, q13, #2
        vdup.8          d15, d29[1]

        vmull.s8        q0,  d12, d14
        vmull.s8        q1,  d13, d14
        vmull.s8        q6,  d16, d15
        vmull.s8        q8,  d17, d15

        vaddl.s16       q7,  d0,  d12
        vaddl.s16       q0,  d1,  d13
        vaddl.s16       q6,  d2,  d16
        vaddl.s16       q1,  d3,  d17

        vadd.i32        q2,  q2,  q7
        vadd.i32        q3,  q3,  q0
        vadd.i32        q4,  q4,  q6
        vadd.i32        q5,  q5,  q1

        vdup.8          d14, d28[2]
        vdup.8          d15, d28[7]

        vmull.s8        q0,  d18, d14
        vmull.s8        q1,  d19, d14
        vmull.s8        q6,  d24, d15
        vmull.s8        q8,  d25, d15

        vaddl.s16       q7,  d0,  d12
        vaddl.s16       q0,  d1,  d13
        vaddl.s16       q6,  d2,  d16
        vaddl.s16       q1,  d3,  d17

        vmov            q8,  q9
        vmov            q9,  q10

        vadd.i32        q2,  q2,  q7
        vadd.i32        q3,  q3,  q0
        vadd.i32        q4,  q4,  q6
        vadd.i32        q5,  q5,  q1

        vmov            q11, q12
        vmov            q12, q13

        pop             {pc}
endfunc

.macro sum_lag2_func type, uv_layout, edge, elems=16
function sum_\type\()_lag2_\edge\()_neon
        push            {r1, lr}
.ifc \edge, left
        sub             r12, r0,  #2*GRAIN_WIDTH
        sub             lr,  r0,  #1*GRAIN_WIDTH
        vld1.8          {q9},  [r12] // load the previous block right above
        vld1.8          {q12}, [lr]
.endif
        sum_lag_n_body  lag2, \type, \uv_layout, \edge, \elems, store=1, uv_coeff=d29[4]
endfunc
.endm

sum_lag2_func y,      0,   left
sum_lag2_func y,      0,   mid
sum_lag2_func y,      0,   right, 15
sum_lag2_func uv_444, 444, left
sum_lag2_func uv_444, 444, mid
sum_lag2_func uv_444, 444, right, 15
sum_lag2_func uv_422, 422, left
sum_lag2_func uv_422, 422, mid
sum_lag2_func uv_422, 422, right, 9
sum_lag2_func uv_420, 420, left
sum_lag2_func uv_420, 420, mid
sum_lag2_func uv_420, 420, right, 9


function sum_lag3_left_above_neon
        // A separate codepath for the left edge, to avoid reading outside
        // of the edge of the buffer.
        sub             r12, r0,  #3*GRAIN_WIDTH
        vld1.8          {q11, q12}, [r12]
        vext.8          q12, q11, q12, #13
        vext.8          q11, q11, q11, #13
        b               sum_lag3_above_start
endfunc

function sum_lag3_above_neon
        sub             r12, r0,  #3*GRAIN_WIDTH + 3
        vld1.8          {q11, q12}, [r12]

sum_lag3_above_start:
        vdup.8          d20, d26[0]
        vext.8          q9,  q11, q12, #1
        vdup.8          d21, d26[1]

        vmull.s8        q0,  d22, d20
        vmull.s8        q1,  d23, d20
        vmull.s8        q6,  d18, d21
        vmull.s8        q7,  d19, d21

        vext.8          q8,  q11, q12, #2
        vdup.8          d20, d26[2]
        vext.8          q9,  q11, q12, #3
        vdup.8          d21, d26[3]

        vaddl.s16       q2,  d0,  d12
        vaddl.s16       q3,  d1,  d13
        vaddl.s16       q4,  d2,  d14
        vaddl.s16       q5,  d3,  d15

        vmull.s8        q0,  d16, d20
        vmull.s8        q1,  d17, d20
        vmull.s8        q6,  d18, d21
        vmull.s8        q7,  d19, d21

        vaddl.s16       q8,  d0,  d12
        vaddl.s16       q9,  d1,  d13
        vaddl.s16       q0,  d2,  d14
        vaddl.s16       q1,  d3,  d15

        vext.8          q6,  q11, q12, #4
        vdup.8          d20, d26[4]
        vext.8          q7,  q11, q12, #5
        vdup.8          d21, d26[5]

        vadd.i32        q2,  q2,  q8
        vadd.i32        q3,  q3,  q9
        vadd.i32        q4,  q4,  q0
        vadd.i32        q5,  q5,  q1

        vmull.s8        q0,  d12, d20
        vmull.s8        q1,  d13, d20
        vmull.s8        q8,  d14, d21
        vmull.s8        q9,  d15, d21

        sub             r12, r0,  #2*GRAIN_WIDTH + 3

        vaddl.s16       q6,  d0,  d16
        vaddl.s16       q7,  d1,  d17
        vaddl.s16       q0,  d2,  d18
        vaddl.s16       q1,  d3,  d19

        vext.8          q8,  q11, q12, #6
        vld1.8          {q11, q12}, [r12]
        vdup.8          d20, d26[6]
        vdup.8          d21, d26[7]

        vadd.i32        q2,  q2,  q6
        vadd.i32        q3,  q3,  q7
        vadd.i32        q4,  q4,  q0
        vadd.i32        q5,  q5,  q1

        vmull.s8        q0,  d16, d20
        vmull.s8        q1,  d17, d20
        vmull.s8        q6,  d22, d21
        vmull.s8        q7,  d23, d21

        vaddl.s16       q8,  d0,  d12
        vaddl.s16       q9,  d1,  d13
        vaddl.s16       q0,  d2,  d14
        vaddl.s16       q1,  d3,  d15

        vext.8          q6,  q11, q12, #1
        vdup.8          d20, d27[0]
        vext.8          q7,  q11, q12, #2
        vdup.8          d21, d27[1]

        vadd.i32        q2,  q2,  q8
        vadd.i32        q3,  q3,  q9
        vadd.i32        q4,  q4,  q0
        vadd.i32        q5,  q5,  q1

        vmull.s8        q0,  d12, d20
        vmull.s8        q1,  d13, d20
        vmull.s8        q8,  d14, d21
        vmull.s8        q9,  d15, d21

        vaddl.s16       q6,  d0,  d16
        vaddl.s16       q7,  d1,  d17
        vaddl.s16       q0,  d2,  d18
        vaddl.s16       q1,  d3,  d19

        vext.8          q8,  q11, q12, #3
        vdup.8          d20, d27[2]
        vext.8          q9,  q11, q12, #4
        vdup.8          d21, d27[3]

        vadd.i32        q2,  q2,  q6
        vadd.i32        q3,  q3,  q7
        vadd.i32        q4,  q4,  q0
        vadd.i32        q5,  q5,  q1

        vmull.s8        q0,  d16, d20
        vmull.s8        q1,  d17, d20
        vmull.s8        q6,  d18, d21
        vmull.s8        q7,  d19, d21

        sub             r12, r0,  #1*GRAIN_WIDTH + 3

        vaddl.s16       q8,  d0,  d12
        vaddl.s16       q9,  d1,  d13
        vaddl.s16       q0,  d2,  d14
        vaddl.s16       q1,  d3,  d15

        vext.8          q6,  q11, q12, #5
        vdup.8          d20, d27[4]
        vext.8          q7,  q11, q12, #6
        vdup.8          d21, d27[5]

        vld1.8          {q11, q12}, [r12]

        vadd.i32        q2,  q2,  q8
        vadd.i32        q3,  q3,  q9
        vadd.i32        q4,  q4,  q0
        vadd.i32        q5,  q5,  q1

        vmull.s8        q0,  d12, d20
        vmull.s8        q1,  d13, d20
        vmull.s8        q8,  d14, d21
        vmull.s8        q9,  d15, d21

        vaddl.s16       q6,  d0,  d16
        vaddl.s16       q7,  d1,  d17
        vaddl.s16       q0,  d2,  d18
        vaddl.s16       q1,  d3,  d19

        vdup.8          d20, d27[6]
        vext.8          q9,  q11, q12, #1
        vdup.8          d21, d27[7]

        vadd.i32        q2,  q2,  q6
        vadd.i32        q3,  q3,  q7
        vadd.i32        q4,  q4,  q0
        vadd.i32        q5,  q5,  q1

        vmull.s8        q0,  d22, d20
        vmull.s8        q1,  d23, d20
        vmull.s8        q6,  d18, d21
        vmull.s8        q7,  d19, d21

        vaddl.s16       q8,  d0,  d12
        vaddl.s16       q9,  d1,  d13
        vaddl.s16       q0,  d2,  d14
        vaddl.s16       q1,  d3,  d15

        vext.8          q6,  q11, q12, #2
        vdup.8          d20, d28[0]
        vext.8          q7,  q11, q12, #3
        vdup.8          d21, d28[1]

        vadd.i32        q2,  q2,  q8
        vadd.i32        q3,  q3,  q9
        vadd.i32        q4,  q4,  q0
        vadd.i32        q5,  q5,  q1

        vmull.s8        q0,  d12, d20
        vmull.s8        q1,  d13, d20
        vmull.s8        q8,  d14, d21
        vmull.s8        q9,  d15, d21

        vaddl.s16       q6,  d0,  d16
        vaddl.s16       q7,  d1,  d17
        vaddl.s16       q0,  d2,  d18
        vaddl.s16       q1,  d3,  d19

        vext.8          q8,  q11, q12, #4
        vdup.8          d20, d28[2]
        vext.8          q9,  q11, q12, #5
        vdup.8          d21, d28[3]

        vadd.i32        q2,  q2,  q6
        vadd.i32        q3,  q3,  q7
        vadd.i32        q4,  q4,  q0
        vadd.i32        q5,  q5,  q1

        vmull.s8        q0,  d16, d20
        vmull.s8        q1,  d17, d20
        vmull.s8        q6,  d18, d21
        vmull.s8        q7,  d19, d21

        vaddl.s16       q8,  d0,  d12
        vaddl.s16       q9,  d1,  d13
        vaddl.s16       q0,  d2,  d14
        vaddl.s16       q1,  d3,  d15

        vext.8          q6,  q11, q12, #6
        vdup.8          d20, d28[4]

        vadd.i32        q2,  q2,  q8
        vadd.i32        q3,  q3,  q9
        vadd.i32        q4,  q4,  q0
        vadd.i32        q5,  q5,  q1

        vmull.s8        q0,  d12, d20
        vmull.s8        q1,  d13, d20

        vaddw.s16       q2,  q2,  d0
        vaddw.s16       q3,  q3,  d1
        vaddw.s16       q4,  q4,  d2
        vaddw.s16       q5,  q5,  d3

        bx              lr
endfunc

.macro sum_lag3_func type, uv_layout, edge, elems=16
function sum_\type\()_lag3_\edge\()_neon
        push            {r1, lr}
        sum_lag_n_body  lag3, \type, \uv_layout, \edge, \elems, store=1, uv_coeff=d29[0]
endfunc
.endm

sum_lag3_func y,      0,   left
sum_lag3_func y,      0,   mid
sum_lag3_func y,      0,   right, 15
sum_lag3_func uv_444, 444, left
sum_lag3_func uv_444, 444, mid
sum_lag3_func uv_444, 444, right, 15
sum_lag3_func uv_422, 422, left
sum_lag3_func uv_422, 422, mid
sum_lag3_func uv_422, 422, right, 9
sum_lag3_func uv_420, 420, left
sum_lag3_func uv_420, 420, mid
sum_lag3_func uv_420, 420, right, 9

function generate_grain_rows_neon
        push            {r11,lr}
1:
        get_grain_row   d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26
        subs            r1,  r1,  #1
        store_grain_row d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26
        bgt             1b
        pop             {r11,pc}
endfunc

function generate_grain_rows_44_neon
        push            {r11,lr}
1:
        get_grain_row_44 d16, d17, d18, d19, d20, d21
        subs            r1,  r1,  #1
        store_grain_row_44 d16, d17, d18, d19, d20, d21
        bgt             1b
        pop             {r11,pc}
endfunc

function gen_grain_uv_444_lag0_neon
        vld1.8          {q3}, [r11]!
        push            {r11,lr}
        bl              get_gaussian_neon
        vrshl.s16       q8,  q0,  q15
        bl              get_gaussian_neon
        vrshl.s16       q9,  q0,  q15
        vqmovn.s16      d0,  q8
        vqmovn.s16      d1,  q9

        vand            q3,  q3,  q1
        vmull.s8        q2,  d6,  d22
        vmull.s8        q3,  d7,  d22
        vrshl.s16       q2,  q2,  q12
        vrshl.s16       q3,  q3,  q12
        vaddw.s8        q2,  q2,  d0
        vaddw.s8        q3,  q3,  d1
        vqmovn.s16      d4,  q2
        vqmovn.s16      d5,  q3
        vst1.8          {q2}, [r0]!
        pop             {r11,pc}
endfunc

function get_grain_row_44_neon
        push            {r11,lr}
        get_grain_row_44 d16, d17, d18, d19, d20, d21
        pop             {r11,pc}
endfunc

function add_uv_420_coeff_lag0_neon
        vld1.16         {q2, q3}, [r11]!
        vld1.16         {q4, q5}, [r12]!
        vpaddl.s8       q2,  q2
        vpaddl.s8       q3,  q3
        vpaddl.s8       q4,  q4
        vpaddl.s8       q5,  q5
        vadd.i16        q2,  q2,  q4
        vadd.i16        q3,  q3,  q5
        vrshrn.s16      d4,  q2,  #2
        vrshrn.s16      d5,  q3,  #2
        b               add_coeff_lag0_start
endfunc

function add_uv_422_coeff_lag0_neon
        vld1.16         {q2, q3}, [r11]!
        vpaddl.s8       q2,  q2
        vpaddl.s8       q3,  q3
        vrshrn.s16      d4,  q2,  #1
        vrshrn.s16      d5,  q3,  #1

add_coeff_lag0_start:
        vand            q3,  q2,  q1
        vmull.s8        q2,  d6,  d22
        vmull.s8        q3,  d7,  d22
        vrshl.s16       q2,  q2,  q12
        vrshl.s16       q3,  q3,  q12
        vaddw.s8        q2,  q2,  d0
        vaddw.s8        q3,  q3,  d1
        vqmovn.s16      d4,  q2
        vqmovn.s16      d5,  q3
        bx              lr
endfunc

.macro gen_grain_82 type
function generate_grain_\type\()_8bpc_neon, export=1
        push            {r4-r11,lr}

.ifc \type, uv_444
        mov             r12, r3
        mov             lr,  #28
        add             r11, r1,  #3*GRAIN_WIDTH
        mov             r1,  r2
        mul             r12, r12, lr
.endif
        movrel          r3,  X(gaussian_sequence)
        ldr             r2,  [r1, #FGD_SEED]
        ldr             r9,  [r1, #FGD_GRAIN_SCALE_SHIFT]
.ifc \type, y
        add             r4,  r1,  #FGD_AR_COEFFS_Y
.else
        add             r4,  r1,  #FGD_AR_COEFFS_UV
.endif
        adr             r5,  L(gen_grain_\type\()_tbl)
        ldr             r6,  [r1, #FGD_AR_COEFF_LAG]
        add             r9,  r9,  #4
        ldr             r6,  [r5, r6, lsl #2]
        vdup.16         q15, r9    // 4 + data->grain_scale_shift
        add             r5,  r5,  r6
        vneg.s16        q15, q15

.ifc \type, uv_444
        cmp             r12, #0
        movw            r10, #0x49d8
        movw            lr,  #0xb524
        // Intentionally using a separate register instead of moveq with an
        // immediate constant, to avoid armv8 deprecated it instruction forms.
        it              eq
        moveq           r10, lr
        add             r4,  r4,  r12       // Add offset to ar_coeffs_uv[1]
        eor             r2,  r2,  r10
.endif

        ldr             r7,  [r1, #FGD_AR_COEFF_SHIFT]
        mov             r8,  #1
        mov             r10, #1
        lsl             r8,  r8,  r7        // 1 << ar_coeff_shift
        lsl             r10, r10, r9        // 1 << (4 + data->grain_scale_shift)
        lsr             r8,  r8,  #1        // 1 << (ar_coeff_shift - 1)
        lsr             r10, r10, #1        // 1 << (4 + data->grain_scale_shift - 1)

        bx              r5

        .align 2
L(gen_grain_\type\()_tbl):
        .word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB
        .word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB
        .word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB
        .word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB

L(generate_grain_\type\()_lag0):
.ifc \type, y
        mov             r1,  #GRAIN_HEIGHT
        bl              generate_grain_rows_neon
.else

        mov             r1,  #3
        bl              generate_grain_rows_neon
        mov             r1,  #GRAIN_HEIGHT-3

        vdup.16         q12, r7
        vld1.8          {d22[]}, [r4]       // ar_coeffs_uv[0]
        vmov.i8         q0,  #0
        vmov.i8         q1,  #255
        vext.8          q13, q0,  q1,  #13
        vext.8          q14, q1,  q0,  #1
        vneg.s16        q12, q12

1:
        vmov            q1,  q13
        bl              gen_grain_uv_444_lag0_neon // 16
        vmov.i8         q1,  #255
        bl              gen_grain_uv_444_lag0_neon // 32
        bl              gen_grain_uv_444_lag0_neon // 48
        bl              gen_grain_uv_444_lag0_neon // 64
        vmov            q1,  q14
        bl              gen_grain_uv_444_lag0_neon // 80
        get_grain_2     d16
        subs            r1,  r1,  #1
        add             r11, r11, #2
        vst1.16         {d16[0]}, [r0]!
        bgt             1b
.endif
        pop             {r4-r11,pc}

L(generate_grain_\type\()_lag1):
        vpush           {q4-q7}
        mov             r5,  #127
        vld1.8          {d27[]}, [r4]!      // ar_coeffs_y[0]
        vld1.8          {d28[]}, [r4]!      // ar_coeffs_y[1]
        vld1.8          {d29[]}, [r4]       // ar_coeffs_y[2]
.ifc \type, y
        ldrsb           r4,  [r4, #1]       // ar_coeffs_y[3]
.else
        add             r4,  r4,  #2
.endif

        mov             r1,  #3
.ifc \type, uv_444
        vld1.8          {d13[]}, [r4]       // ar_coeffs_uv[4]
        ldrsb           r4,  [r4, #-1]      // ar_coeffs_uv[3]
.endif
        bl              generate_grain_rows_neon

        mov             r1,  #GRAIN_HEIGHT - 3
1:
        sum_\type\()_lag1 q7,  q8,  q8,  q9,  left
        sum_\type\()_lag1 q8,  q8,  q9,  q10
        sum_\type\()_lag1 q9,  q9,  q10, q11
        sum_\type\()_lag1 q10, q10, q11, q12
        sum_\type\()_lag1 q12, q11, q12, q13, right
        get_grain_2     d26
        subs            r1,  r1,  #1
.ifc \type, uv_444
        add             r11, r11, #2
.endif
        store_grain_row d14, d15, d16, d17, d18, d19, d20, d21, d24, d25, d26
        vmov            q11, q10
        vmov            q10, q9
        vmov            q9,  q8
        vmov            q8,  q7
        bgt             1b

        vpop            {q4-q7}
        pop             {r4-r11,pc}

L(generate_grain_\type\()_lag2):
        vpush           {q4-q7}
        mov             r5,  #127
        vld1.8          {d28,d29}, [r4]     // ar_coeffs_y[0-11], ar_coeffs_uv[0-12]

        vmov.s8         r4,  d29[2]
        vmov.s8         r10, d29[3]

        mov             r1,  #3
        bl              generate_grain_rows_neon

        mov             r1,  #GRAIN_HEIGHT - 3
1:
        bl              sum_\type\()_lag2_left_neon
        bl              sum_\type\()_lag2_mid_neon
        bl              sum_\type\()_lag2_mid_neon
        bl              sum_\type\()_lag2_mid_neon
        bl              sum_\type\()_lag2_right_neon
        get_grain_2     d16
        subs            r1,  r1,  #1
.ifc \type, uv_444
        add             r11, r11, #2
.endif
        vst1.16         {d16[0]}, [r0]!
        bgt             1b

        vpop            {q4-q7}
        pop             {r4-r11,pc}

L(generate_grain_\type\()_lag3):
        vpush           {q4-q7}
        mov             r5,  #127
        vld1.8          {q13, q14}, [r4]    // ar_coeffs_y[0-23], ar_coeffs_uv[0-24]

        vmov.u8         r4,  d28[5]
        vmov.u8         r10, d28[6]
        vmov.u8         r12, d28[7]

        orr             r4,  r4,  r10, lsl #8
        orr             r4,  r4,  r12, lsl #16

        mov             r1,  #3
        vpush           {d26}
        bl              generate_grain_rows_neon
        vpop            {d26}

        mov             r1,  #GRAIN_HEIGHT - 3
1:
        bl              sum_\type\()_lag3_left_neon
        bl              sum_\type\()_lag3_mid_neon
        bl              sum_\type\()_lag3_mid_neon
        bl              sum_\type\()_lag3_mid_neon
        bl              sum_\type\()_lag3_right_neon
        get_grain_2     d16
        subs            r1,  r1,  #1
.ifc \type, uv_444
        add             r11, r11, #2
.endif
        vst1.16         {d16[0]}, [r0]!
        bgt             1b

        vpop            {q4-q7}
        pop             {r4-r11,pc}
endfunc
.endm

gen_grain_82 y
gen_grain_82 uv_444

.macro set_height dst, type
.ifc \type, uv_420
        mov             \dst,  #SUB_GRAIN_HEIGHT-3
.else
        mov             \dst,  #GRAIN_HEIGHT-3
.endif
.endm

.macro increment_y_ptr reg, type
.ifc \type, uv_420
        add             \reg, \reg, #2*GRAIN_WIDTH-(3*32)
.else
        sub             \reg, \reg, #3*32-GRAIN_WIDTH
.endif
.endm

.macro gen_grain_44 type
function generate_grain_\type\()_8bpc_neon, export=1
        push            {r4-r11,lr}

        mov             r12, r3
        mov             lr,  #28
        add             r11, r1,  #3*GRAIN_WIDTH-3
        mov             r1,  r2
        mul             r12, r12, lr

        movrel          r3,  X(gaussian_sequence)
        ldr             r2,  [r1, #FGD_SEED]
        ldr             r9,  [r1, #FGD_GRAIN_SCALE_SHIFT]
        add             r4,  r1,  #FGD_AR_COEFFS_UV
        adr             r5,  L(gen_grain_\type\()_tbl)
        ldr             r6,  [r1, #FGD_AR_COEFF_LAG]
        add             r9,  r9,  #4
        ldr             r6,  [r5, r6, lsl #2]
        vdup.16         q15, r9    // 4 + data->grain_scale_shift
        add             r5,  r5,  r6
        vneg.s16        q15, q15

        cmp             r12, #0
        movw            r10, #0x49d8
        movw            lr,  #0xb524
        // Intentionally using a separate register instead of moveq with an
        // immediate constant, to avoid armv8 deprecated it instruction forms.
        it              eq
        moveq           r10, lr
        add             r4,  r4,  r12       // Add offset to ar_coeffs_uv[1]
        eor             r2,  r2,  r10

        ldr             r7,  [r1, #FGD_AR_COEFF_SHIFT]
        mov             r8,  #1
        mov             r10, #1
        lsl             r8,  r8,  r7        // 1 << ar_coeff_shift
        lsl             r10, r10, r9        // 1 << (4 + data->grain_scale_shift)
        lsr             r8,  r8,  #1        // 1 << (ar_coeff_shift - 1)
        lsr             r10, r10, #1        // 1 << (4 + data->grain_scale_shift - 1)
        bx              r5

        .align 2
L(gen_grain_\type\()_tbl):
        .word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB
        .word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB
        .word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB
        .word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB

L(generate_grain_\type\()_lag0):
.ifc \type, uv_420
        vpush           {q4-q5}
.endif
        mov             r1,  #3
        bl              generate_grain_rows_44_neon
        set_height      r1,  \type

        vdup.16         q12, r7
        vld1.8          {d22[]}, [r4]       // ar_coeffs_uv[0]
        vmov.i8         q0,  #0
        vmov.i8         q1,  #255
        vext.8          q13, q0,  q1,  #13
        vext.8          q14, q1,  q0,  #7
        vneg.s16        q12, q12

1:
        bl              get_grain_row_44_neon
.ifc \type, uv_420
        add             r12, r11, #GRAIN_WIDTH
.endif
        vmov            q1,  q13
        vmov            q0,  q8
        bl              add_\type\()_coeff_lag0_neon
        vmov.i8         q1,  #255
        vmov            q0,  q9
        vmov            q8,  q2
        bl              add_\type\()_coeff_lag0_neon
        vmov.i8         q1,  q14
        vmov            q0,  q10
        vmov            q9,  q2
        bl              add_\type\()_coeff_lag0_neon
        vmov            q10, q2
        subs            r1,  r1,  #1
        increment_y_ptr r11, \type
        store_grain_row_44 d16, d17, d18, d19, d20, d21
        bgt             1b

.ifc \type, uv_420
        vpop            {q4-q5}
.endif
        pop             {r4-r11,pc}

L(generate_grain_\type\()_lag1):
        vpush           {q4-q7}
        mov             r5,  #127
        vld1.8          {d27[]}, [r4]!      // ar_coeffs_uv[0]
        vld1.8          {d28[]}, [r4]!      // ar_coeffs_uv[1]
        vld1.8          {d29[]}, [r4]       // ar_coeffs_uv[2]
        add             r4,  r4,  #2

        mov             r1,  #3
        vld1.8          {d13[]}, [r4]       // ar_coeffs_uv[4]
        ldrsb           r4,  [r4, #-1]      // ar_coeffs_uv[3]
        bl              generate_grain_rows_44_neon

        set_height      r1,  \type
1:
        sum_\type\()_lag1 q7,  q8,  q8,  q9,  left
        sum_\type\()_lag1 q8,  q8,  q9,  q10
        sum_\type\()_lag1 q10, q9,  q10, q11, right
        subs            r1,  r1,  #1
        increment_y_ptr r11, \type
        store_grain_row_44 d14, d15, d16, d17, d20, d21
        vmov            q9,  q8
        vmov            q8,  q7
        bgt             1b

        vpop            {q4-q7}
        pop             {r4-r11,pc}

L(generate_grain_\type\()_lag2):
        vpush           {q4-q7}
        mov             r5,  #127
        vld1.8          {d28,d29}, [r4]     // ar_coeffs_uv[0-12]

        vmov.s8         r4,  d29[2]
        vmov.s8         r10, d29[3]

        mov             r1,  #3
        bl              generate_grain_rows_44_neon

        set_height      r1,  \type
1:
        bl              sum_\type\()_lag2_left_neon
        bl              sum_\type\()_lag2_mid_neon
        bl              sum_\type\()_lag2_right_neon
        subs            r1,  r1,  #1
        increment_y_ptr r11, \type
        add             r0,  r0,  #GRAIN_WIDTH-48
        bgt             1b

        vpop            {q4-q7}
        pop             {r4-r11,pc}

L(generate_grain_\type\()_lag3):
        vpush           {q4-q7}
        mov             r5,  #127
        vld1.8          {q13, q14}, [r4]    // ar_coeffs_y[0-23], ar_coeffs_uv[0-24]

        vmov.u8         r4,  d28[5]
        vmov.u8         r10, d28[6]
        vmov.u8         r12, d28[7]

        orr             r4,  r4,  r10, lsl #8
        orr             r4,  r4,  r12, lsl #16

        mov             r1,  #3
        bl              generate_grain_rows_44_neon

        set_height      r1,  \type
1:
        bl              sum_\type\()_lag3_left_neon
        bl              sum_\type\()_lag3_mid_neon
        bl              sum_\type\()_lag3_right_neon
        subs            r1,  r1,  #1
        increment_y_ptr r11, \type
        add             r0,  r0,  #GRAIN_WIDTH-48
        bgt             1b

        vpop            {q4-q7}
        pop             {r4-r11,pc}
endfunc
.endm

gen_grain_44 uv_420
gen_grain_44 uv_422

.macro gather_interleaved dst1, dst2, src1, src2, off
        vmov.u8         r11, \src1[0+\off]
        vmov.u8         r12, \src2[0+\off]
        add             r11, r11, r3
        vmov.u8         lr,  \src1[2+\off]
        add             r12, r12, r3
        vld1.8          {\dst1[0+\off]}, [r11]
        vmov.u8         r11, \src2[2+\off]
        add             lr,  lr,  r3
        vld1.8          {\dst2[0+\off]}, [r12]
        vmov.u8         r12, \src1[4+\off]
        add             r11, r11, r3
        vld1.8          {\dst1[2+\off]}, [lr]
        vmov.u8         lr,  \src2[4+\off]
        add             r12, r12, r3
        vld1.8          {\dst2[2+\off]}, [r11]
        vmov.u8         r11, \src1[6+\off]
        add             lr,  lr,  r3
        vld1.8          {\dst1[4+\off]}, [r12]
        vmov.u8         r12, \src2[6+\off]
        add             r11, r11, r3
        vld1.8          {\dst2[4+\off]}, [lr]
        add             r12, r12, r3
        vld1.8          {\dst1[6+\off]}, [r11]
        vld1.8          {\dst2[6+\off]}, [r12]
.endm

.macro gather dst1, dst2, dst3, dst4, src1, src2, src3, src4
        gather_interleaved \dst1, \dst3, \src1, \src3, 0
        gather_interleaved \dst1, \dst3, \src1, \src3, 1
        gather_interleaved \dst2, \dst4, \src2, \src4, 0
        gather_interleaved \dst2, \dst4, \src2, \src4, 1
.endm

function gather32_neon
        push            {r11-r12,lr}
        gather          d8,  d9,  d10, d11, d0,  d1,  d2,  d3
        pop             {r11-r12,pc}
endfunc

function gather16_neon
        push            {r11-r12,lr}
        gather_interleaved d8,  d9,  d0,  d1,  0
        gather_interleaved d8,  d9,  d0,  d1,  1
        pop             {r11-r12,pc}
endfunc

const overlap_coeffs_0, align=4
        .byte 27, 17, 0,  0,  0,  0,  0,  0
        .byte 17, 27, 32, 32, 32, 32, 32, 32
endconst

const overlap_coeffs_1, align=4
        .byte 23, 0,  0,  0,  0,  0,  0,  0
        .byte 22, 32, 32, 32, 32, 32, 32, 32
endconst

.macro calc_offset offx, offy, src, sx, sy
        and             \offy, \src,  #0xF     // randval & 0xF
        lsr             \offx, \src,  #4       // randval >> 4
.if \sy == 0
        add             \offy, \offy, \offy    // 2 * (randval & 0xF)
.endif
.if \sx == 0
        add             \offx, \offx, \offx    // 2 * (randval >> 4)
.endif
.endm

.macro add_offset dst, offx, offy, src, stride
        mla             \dst, \stride, \offy, \src // grain_lut += grain_stride * offy
        add             \dst, \dst, \offx          // grain_lut += offx
.endm

// void dav1d_fgy_32x32_8bpc_neon(pixel *const dst, const pixel *const src,
//                                const ptrdiff_t stride,
//                                const uint8_t scaling[SCALING_SIZE],
//                                const int scaling_shift,
//                                const entry grain_lut[][GRAIN_WIDTH],
//                                const int offsets[][2],
//                                const int h, const ptrdiff_t clip,
//                                const ptrdiff_t type);
function fgy_32x32_8bpc_neon, export=1
        push            {r4-r11,lr}
        vpush           {q4-q7}
        ldrd            r4,  r5,  [sp, #100]   // scaling_shift, grain_lut
        ldrd            r6,  r7,  [sp, #108]   // offsets, h
        ldr             r8,       [sp, #116]   // clip
        mov             r9,  #GRAIN_WIDTH      // grain_lut stride

        neg             r4,  r4
        vdup.16         q13, r4                // -scaling_shift
        cmp             r8,  #0

        movrel_local    r12, overlap_coeffs_0

        beq             1f
        // clip
        vmov.i8         q14, #16
        vmov.i8         q15, #235
        b               2f
1:
        // no clip
        vmov.i8         q14, #0
        vmov.i8         q15, #255
2:

        vld1.8          {d24, d25}, [r12, :128] // overlap_coeffs

        add             r5,  r5,  #9           // grain_lut += 9
        add             r5,  r5,  r9,  lsl #3  // grain_lut += 8 * grain_stride
        add             r5,  r5,  r9           // grain_lut += grain_stride

        ldr             r10, [r6, #8]          // offsets[1][0]
        calc_offset     r10, r4,  r10, 0,   0
        add_offset      r4,  r10, r4,  r5,  r9
        ldr             r10, [r6, #4]          // offsets[0][1]
        calc_offset     r10, r11, r10, 0,   0
        add_offset      r11, r10, r11, r5,  r9
        ldr             r10, [r6, #12]         // offsets[1][1]
        calc_offset     r10, r8,  r10, 0,   0
        add_offset      r8,  r10, r8,  r5,  r9
        ldr             r6,  [r6]              // offsets[0][0]
        calc_offset     r6,  lr,  r6,  0,   0
        add_offset      r5,  r6,  lr,  r5,  r9

        add             r4,  r4,  #32          // grain_lut += FG_BLOCK_SIZE * bx
        add             r6,  r11, r9,  lsl #5  // grain_lut += grain_stride * FG_BLOCK_SIZE * by

        ldr             r10, [sp, #120]        // type
        adr             r11, L(fgy_loop_tbl)

        tst             r10, #1
        ldr             r10, [r11, r10, lsl #2]

        add             r8,  r8,  r9,  lsl #5  // grain_lut += grain_stride * FG_BLOCK_SIZE * by
        add             r8,  r8,  #32          // grain_lut += FG_BLOCK_SIZE * bx

        add             r11, r11, r10

        beq             1f
        // y overlap
        vdup.8          d14, d24[0]
        vdup.8          d15, d24[1]
        mov             r10, r7                // backup actual h
        mov             r7,  #2
1:
        bx              r11
endfunc

function fgy_loop_neon
L(fgy_loop_tbl):
        .word L(loop_00) - L(fgy_loop_tbl) + CONFIG_THUMB
        .word L(loop_01) - L(fgy_loop_tbl) + CONFIG_THUMB
        .word L(loop_10) - L(fgy_loop_tbl) + CONFIG_THUMB
        .word L(loop_11) - L(fgy_loop_tbl) + CONFIG_THUMB

.macro fgy ox, oy
L(loop_\ox\oy):
1:
.if \ox
        vld1.8          {d8},       [r4],       r9 // grain_lut old
.endif
.if \oy
        vld1.8          {q2, q3},   [r6],       r9 // grain_lut top
.endif
.if \ox && \oy
        vld1.8          {d10},      [r8],       r9 // grain_lut top old
.endif
        vld1.8          {q0,  q1},  [r1, :128], r2 // src
        vld1.8          {q10, q11}, [r5],       r9 // grain_lut

.if \ox
        vmull.s8        q4,  d8,  d24
        vmlal.s8        q4,  d20, d25
.endif

.if \oy
.if \ox
        vmull.s8        q5,  d10, d24
        vmlal.s8        q5,  d4,  d25
        vqrshrn.s16     d20, q4,  #5
        vqrshrn.s16     d4,  q5,  #5
.endif

        vmull.s8        q4,  d20, d15
        vmull.s8        q5,  d21, d15
        vmull.s8        q8,  d22, d15
        vmull.s8        q9,  d23, d15
        vmlal.s8        q4,  d4,  d14
        vmlal.s8        q5,  d5,  d14
        vmlal.s8        q8,  d6,  d14
        vmlal.s8        q9,  d7,  d14
        vqrshrn.s16     d20, q4,  #5
        vqrshrn.s16     d21, q5,  #5
        vqrshrn.s16     d22, q8,  #5
        vqrshrn.s16     d23, q9,  #5
.elseif \ox
        vqrshrn.s16     d20, q4,  #5
.endif

        bl              gather32_neon

        vmovl.s8        q8,  d20       // grain
        vmovl.s8        q9,  d21
        vmovl.s8        q10, d22
        vmovl.s8        q11, d23

        vmovl.u8        q2,  d8        // scaling
        vmovl.u8        q3,  d9
        vmovl.u8        q4,  d10
        vmovl.u8        q5,  d11

        vmul.i16        q8,  q8,  q2   // scaling * grain
        vmul.i16        q9,  q9,  q3
        vmul.i16        q10, q10, q4
        vmul.i16        q11, q11, q5

        vrshl.s16       q8,  q8,  q13  // round2(scaling * grain, scaling_shift)
        vrshl.s16       q9,  q9,  q13
        vrshl.s16       q10, q10, q13
        vrshl.s16       q11, q11, q13

        vaddw.u8        q8,  q8,  d0   // *src + noise
        vaddw.u8        q9,  q9,  d1
        vaddw.u8        q10, q10, d2
        vaddw.u8        q11, q11, d3

        vqmovun.s16     d0,  q8
        vqmovun.s16     d1,  q9
        vqmovun.s16     d2,  q10
        vqmovun.s16     d3,  q11

        vmax.u8         q0,  q0,  q14
        vmax.u8         q1,  q1,  q14
        vmin.u8         q0,  q0,  q15
        vmin.u8         q1,  q1,  q15

        subs            r7,  r7,  #1
.if \oy
        vdup.8          d14, d25[0]
        vdup.8          d15, d25[1]
.endif
        vst1.8          {q0, q1}, [r0, :128], r2 // dst
        bgt             1b

.if \oy
        cmp             r10, #2
        sub             r7,  r10, #2           // restore actual remaining h
        bgt             L(loop_\ox\()0)
.endif
        vpop            {q4-q7}
        pop             {r4-r11,pc}
.endm

        fgy             0, 0
        fgy             0, 1
        fgy             1, 0
        fgy             1, 1
endfunc

// void dav1d_fguv_32x32_420_8bpc_neon(pixel *const dst,
//                                     const pixel *const src,
//                                     const ptrdiff_t stride,
//                                     const uint8_t scaling[SCALING_SIZE],
//                                     const Dav1dFilmGrainData *const data,
//                                     const entry grain_lut[][GRAIN_WIDTH],
//                                     const pixel *const luma_row,
//                                     const ptrdiff_t luma_stride,
//                                     const int offsets[][2],
//                                     const ptrdiff_t h, const ptrdiff_t uv,
//                                     const ptrdiff_t is_id,
//                                     const ptrdiff_t type);
.macro fguv layout, sx, sy
function fguv_32x32_\layout\()_8bpc_neon, export=1
        push            {r4-r11,lr}
        vpush           {q4-q7}
        ldrd            r4,  r5,  [sp, #100]   // data, grain_lut
        ldrd            r6,  r7,  [sp, #108]   // luma_row, luma_stride
        ldrd            r8,  r9,  [sp, #116]   // offsets, h
        ldrd            r10, r11, [sp, #124]   // uv, is_id

        // !csfl
        add             r10, r4,  r10, lsl #2  // + 4*uv
        add             r12, r10, #FGD_UV_LUMA_MULT
        add             lr,  r10, #FGD_UV_MULT
        add             r10, r10, #FGD_UV_OFFSET
        vld1.16         {d4[]},  [r12]         // uv_luma_mult
        vld1.16         {d4[2]}, [r10]         // uv_offset
        vld1.16         {d4[1]}, [lr]          // uv_mult

        ldr             lr,  [r4, #FGD_SCALING_SHIFT]
        ldr             r12, [r4, #FGD_CLIP_TO_RESTRICTED_RANGE]
        neg             lr,  lr                // -scaling_shift

        cmp             r12, #0
        vdup.16         q13, lr                // -scaling_shift

        beq             1f
        // clip
        cmp             r11, #0
        vmov.i8         q14, #16
        vmov.i8         q15, #240
        beq             2f
        // is_id
        vmov.i8         q15, #235
        b               2f
1:
        // no clip
        vmov.i8         q14, #0
        vmov.i8         q15, #255
2:

        mov             r10, #GRAIN_WIDTH      // grain_lut stride

        add             r5,  r5,  #(3 + (2 >> \sx)*3) // grain_lut += 9 or 6
.if \sy
        add             r5,  r5,  r10, lsl #2  // grain_lut += 4 * grain_stride
        add             r5,  r5,  r10, lsl #1  // grain_lut += 2 * grain_stride
.else
        add             r5,  r5,  r10, lsl #3  // grain_lut += 8 * grain_stride
        add             r5,  r5,  r10          // grain_lut += grain_stride
.endif

        ldr             r12, [r8, #8]          // offsets[1][0]
        calc_offset     r12, r4,  r12, \sx, \sy
        add_offset      r4,  r12, r4,  r5,  r10

        ldr             r12, [r8, #4]          // offsets[0][1]
        calc_offset     r12, lr,  r12, \sx, \sy
        add_offset      lr,  r12, lr,  r5,  r10

        ldr             r12, [r8, #12]         // offsets[1][1]
        calc_offset     r12, r11, r12, \sx, \sy
        add_offset      r11, r12, r11, r5,  r10

        ldr             r8,  [r8]              // offsets[0][0]
        calc_offset     r8,  r12, r8,  \sx, \sy
        add_offset      r5,  r8,  r12, r5,  r10

        add             r4,  r4,  #(32 >> \sx) // grain_lut += FG_BLOCK_SIZE * bx
        add             r8,  lr,  r10, lsl #(5 - \sy) // grain_lut += grain_stride * FG_BLOCK_SIZE * by
        add             r11, r11, r10, lsl #(5 - \sy) // grain_lut += grain_stride * FG_BLOCK_SIZE * by
        add             r11, r11, #(32 >> \sx) // grain_lut += FG_BLOCK_SIZE * bx

        movrel_local    r12, overlap_coeffs_\sx
        ldr             lr,  [sp, #132]        // type

        vld1.8          {d24, d25}, [r12, :128] // overlap_coeffs

        movrel_local    r12, L(fguv_loop_sx\sx\()_tbl)
#if CONFIG_THUMB
        // This uses movrel_local instead of adr above, because the target
        // can be out of range for adr. But movrel_local leaves the thumb bit
        // set on COFF (but probably wouldn't if building for thumb on ELF),
        // thus try to clear the bit for robustness.
        bic             r12, r12, #1
#endif

        tst             lr,  #1
        ldr             lr,  [r12, lr,  lsl #2]

        add             r12, r12, lr

        beq             1f
        // y overlap
        sub             lr,  r9,  #(2 >> \sy)  // backup remaining h
        mov             r9,  #(2 >> \sy)

1:

.if \sy
        vmov.i8         d6,  #23
        vmov.i8         d7,  #22
.else
        vmov.i8         d6,  #27
        vmov.i8         d7,  #17
.endif

.if \sy
        add             r7,  r7,  r7           // luma_stride *= 2
.endif

        bx              r12
endfunc
.endm

fguv 420, 1, 1
fguv 422, 1, 0
fguv 444, 0, 0

function fguv_loop_sx0_neon
L(fguv_loop_sx0_tbl):
        .word L(fguv_loop_sx0_csfl0_00) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB
        .word L(fguv_loop_sx0_csfl0_01) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB
        .word L(fguv_loop_sx0_csfl0_10) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB
        .word L(fguv_loop_sx0_csfl0_11) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB
        .word L(fguv_loop_sx0_csfl1_00) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB
        .word L(fguv_loop_sx0_csfl1_01) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB
        .word L(fguv_loop_sx0_csfl1_10) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB
        .word L(fguv_loop_sx0_csfl1_11) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB

.macro fguv_loop_sx0 csfl, ox, oy
L(fguv_loop_sx0_csfl\csfl\()_\ox\oy):
.if \oy
        mov             r12, lr
.endif
1:
.if \ox
        vld1.8          {d8},       [r4],        r10 // grain_lut old
.endif
.if \oy
        vld1.8          {q8, q9},   [r8],        r10 // grain_lut top
.endif
.if \ox && \oy
        vld1.8          {d10},      [r11],       r10 // grain_lut top old
.endif
        vld1.8          {q0,  q1},  [r6, :128],  r7  // luma
        vld1.8          {q10, q11}, [r5],        r10 // grain_lut

.if \ox
        vmull.s8        q4,  d8,  d24
        vmlal.s8        q4,  d20, d25
.endif

.if \oy
.if \ox
        vmull.s8        q5,  d10, d24
        vmlal.s8        q5,  d16, d25
        vqrshrn.s16     d20, q4,  #5
        vqrshrn.s16     d16, q5,  #5
.endif

        vmull.s8        q4,  d20, d7
        vmull.s8        q5,  d21, d7
        vmull.s8        q6,  d22, d7
        vmull.s8        q7,  d23, d7
        vmlal.s8        q4,  d16, d6
        vmlal.s8        q5,  d17, d6
        vmlal.s8        q6,  d18, d6
        vmlal.s8        q7,  d19, d6
        vqrshrn.s16     d20, q4,  #5
        vqrshrn.s16     d21, q5,  #5
        vqrshrn.s16     d22, q6,  #5
        vqrshrn.s16     d23, q7,  #5
.elseif \ox
        vqrshrn.s16     d20, q4,  #5
.endif
.if !\csfl
        vld1.8          {q8,  q9},  [r1, :128] // src
        vmovl.u8        q4,  d0
        vmovl.u8        q5,  d1
        vmovl.u8        q6,  d2
        vmovl.u8        q7,  d3
        vmovl.u8        q0,  d16
        vmovl.u8        q1,  d17
        vmovl.u8        q8,  d18
        vmovl.u8        q9,  d19
        vmul.i16        q4,  q4,  d4[0]
        vmul.i16        q5,  q5,  d4[0]
        vmul.i16        q6,  q6,  d4[0]
        vmul.i16        q7,  q7,  d4[0]
        vmul.i16        q0,  q0,  d4[1]
        vmul.i16        q1,  q1,  d4[1]
        vmul.i16        q8,  q8,  d4[1]
        vmul.i16        q9,  q9,  d4[1]
        vqadd.s16       q4,  q4,  q0
        vqadd.s16       q5,  q5,  q1
        vqadd.s16       q6,  q6,  q8
        vqadd.s16       q7,  q7,  q9
        vdup.16         q0,  d4[2]
        vshr.s16        q4,  q4,  #6
        vshr.s16        q5,  q5,  #6
        vshr.s16        q6,  q6,  #6
        vshr.s16        q7,  q7,  #6
        vadd.i16        q4,  q4,  q0
        vadd.i16        q5,  q5,  q0
        vadd.i16        q6,  q6,  q0
        vadd.i16        q7,  q7,  q0
        vqmovun.s16     d0,  q4
        vqmovun.s16     d1,  q5
        vqmovun.s16     d2,  q6
        vqmovun.s16     d3,  q7
.endif

        bl              gather32_neon

        vld1.8          {q0,  q1},  [r1, :128], r2 // src

        vmovl.s8        q8,  d20       // grain
        vmovl.s8        q9,  d21
        vmovl.s8        q10, d22
        vmovl.s8        q11, d23

        vmovl.u8        q6,  d8        // scaling
        vmovl.u8        q7,  d9
        vmovl.u8        q4,  d10
        vmovl.u8        q5,  d11

        vmul.i16        q8,  q8,  q6   // scaling * grain
        vmul.i16        q9,  q9,  q7
        vmul.i16        q10, q10, q4
        vmul.i16        q11, q11, q5

        vrshl.s16       q8,  q8,  q13  // round2(scaling * grain, scaling_shift)
        vrshl.s16       q9,  q9,  q13
        vrshl.s16       q10, q10, q13
        vrshl.s16       q11, q11, q13

        vaddw.u8        q8,  q8,  d0   // *src + noise
        vaddw.u8        q9,  q9,  d1
        vaddw.u8        q10, q10, d2
        vaddw.u8        q11, q11, d3

        vqmovun.s16     d0,  q8
        vqmovun.s16     d1,  q9
        vqmovun.s16     d2,  q10
        vqmovun.s16     d3,  q11

        vmax.u8         q0,  q0,  q14
        vmax.u8         q1,  q1,  q14
        vmin.u8         q0,  q0,  q15
        vmin.u8         q1,  q1,  q15

        subs            r9,  r9,  #1
.if \oy
        vdup.8          d6,  d25[0]
        vdup.8          d7,  d25[1]
.endif

        vst1.8          {q0, q1}, [r0, :128], r2 // dst
        bgt             1b

.if \oy
        cmp             r12, #0
        mov             r9,  r12               // restore actual remaining h
        bgt             L(fguv_loop_sx0_csfl\csfl\()_\ox\()0)
.endif
        b               9f
.endm
        fguv_loop_sx0   0, 0, 0
        fguv_loop_sx0   0, 0, 1
        fguv_loop_sx0   0, 1, 0
        fguv_loop_sx0   0, 1, 1
        fguv_loop_sx0   1, 0, 0
        fguv_loop_sx0   1, 0, 1
        fguv_loop_sx0   1, 1, 0
        fguv_loop_sx0   1, 1, 1

9:
        vpop            {q4-q7}
        pop             {r4-r11,pc}
endfunc

function fguv_loop_sx1_neon
L(fguv_loop_sx1_tbl):
        .word L(fguv_loop_sx1_csfl0_00) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB
        .word L(fguv_loop_sx1_csfl0_01) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB
        .word L(fguv_loop_sx1_csfl0_10) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB
        .word L(fguv_loop_sx1_csfl0_11) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB
        .word L(fguv_loop_sx1_csfl1_00) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB
        .word L(fguv_loop_sx1_csfl1_01) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB
        .word L(fguv_loop_sx1_csfl1_10) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB
        .word L(fguv_loop_sx1_csfl1_11) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB

.macro fguv_loop_sx1 csfl, ox, oy
L(fguv_loop_sx1_csfl\csfl\()_\ox\oy):
.if \oy
        mov             r12, lr
.endif
1:
.if \ox
        vld1.8          {d8},       [r4],        r10 // grain_lut old
.endif
.if \oy
        vld1.8          {q8},       [r8],        r10 // grain_lut top
.endif
.if \ox && \oy
        vld1.8          {d10},      [r11],       r10 // grain_lut top old
.endif
        vld1.8          {q0,  q1},  [r6, :128],  r7  // luma
        vld1.8          {q10},      [r5],        r10 // grain_lut
        vld1.8          {q11},      [r1, :128],  r2  // src

.if \ox
        vmull.s8        q4,  d8,  d24
        vmlal.s8        q4,  d20, d25
.endif

        vpaddl.u8       q0,  q0
        vpaddl.u8       q1,  q1
.if \oy
.if \ox
        vmull.s8        q5,  d10, d24
        vmlal.s8        q5,  d16, d25
        vqrshrn.s16     d20, q4,  #5
        vqrshrn.s16     d16, q5,  #5
.endif

        vmull.s8        q4,  d20, d7
        vmull.s8        q5,  d21, d7
        vmlal.s8        q4,  d16, d6
        vmlal.s8        q5,  d17, d6
        vqrshrn.s16     d20, q4,  #5
        vqrshrn.s16     d21, q5,  #5
.elseif \ox
        vqrshrn.s16     d20, q4,  #5
.endif
.if \csfl
        vrshrn.u16      d0,  q0,  #1
        vrshrn.u16      d1,  q1,  #1
.else
        vrshr.u16       q4,  q0,  #1
        vrshr.u16       q5,  q1,  #1
        vmovl.u8        q0,  d22
        vmovl.u8        q1,  d23
        vmul.i16        q4,  q4,  d4[0]
        vmul.i16        q5,  q5,  d4[0]
        vmul.i16        q0,  q0,  d4[1]
        vmul.i16        q1,  q1,  d4[1]
        vqadd.s16       q4,  q4,  q0
        vqadd.s16       q5,  q5,  q1
        vdup.16         q0,  d4[2]
        vshr.s16        q4,  q4,  #6
        vshr.s16        q5,  q5,  #6
        vadd.i16        q4,  q4,  q0
        vadd.i16        q5,  q5,  q0
        vqmovun.s16     d0,  q4
        vqmovun.s16     d1,  q5
.endif

        bl              gather16_neon

        vmovl.s8        q8,  d20       // grain
        vmovl.s8        q9,  d21

        vmovl.u8        q6,  d8        // scaling
        vmovl.u8        q7,  d9

        vmul.i16        q8,  q8,  q6   // scaling * grain
        vmul.i16        q9,  q9,  q7

        vrshl.s16       q8,  q8,  q13  // round2(scaling * grain, scaling_shift)
        vrshl.s16       q9,  q9,  q13

        vaddw.u8        q8,  q8,  d22  // *src + noise
        vaddw.u8        q9,  q9,  d23

        vqmovun.s16     d0,  q8
        vqmovun.s16     d1,  q9

        vmax.u8         q0,  q0,  q14
        vmin.u8         q0,  q0,  q15

        subs            r9,  r9,  #1
.if \oy
        vswp            d6,  d7
.endif
        vst1.8          {q0}, [r0, :128], r2 // dst
        bgt             1b

.if \oy
        cmp             r12, #0
        mov             r9,  r12               // restore actual remaining h
        bgt             L(fguv_loop_sx1_csfl\csfl\()_\ox\()0)
.endif

        b               9f
.endm
        fguv_loop_sx1   0, 0, 0
        fguv_loop_sx1   0, 0, 1
        fguv_loop_sx1   0, 1, 0
        fguv_loop_sx1   0, 1, 1
        fguv_loop_sx1   1, 0, 0
        fguv_loop_sx1   1, 0, 1
        fguv_loop_sx1   1, 1, 0
        fguv_loop_sx1   1, 1, 1

9:
        vpop            {q4-q7}
        pop             {r4-r11,pc}
endfunc