xref: /aosp_15_r20/external/libdav1d/src/arm/32/msac.S (revision c09093415860a1c2373dacd84c4fde00c507cdfd)

/*
 * Copyright © 2019, VideoLAN and dav1d authors
 * Copyright © 2020, Martin Storsjo
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 *    list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "src/arm/asm.S"
#include "util.S"

#define BUF_POS 0
#define BUF_END 4
#define DIF 8
#define RNG 12
#define CNT 16
#define ALLOW_UPDATE_CDF 20

const coeffs
        .short 60, 56, 52, 48, 44, 40, 36, 32, 28, 24, 20, 16, 12, 8, 4, 0
        .short 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0, 0, 0
endconst

const bits, align=4
        .short   0x1,   0x2,   0x4,   0x8,   0x10,   0x20,   0x40,   0x80
        .short 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000
endconst

.macro vld1_align_n d0, q0, q1, src, n
.if \n == 4
        vld1.16         {\d0},  [\src, :64]
.elseif \n == 8
        vld1.16         {\q0},  [\src, :128]
.else
        vld1.16         {\q0, \q1},  [\src, :128]
.endif
.endm

.macro vld1_n d0, q0, q1, src, n
.if \n == 4
        vld1.16         {\d0},  [\src]
.elseif \n == 8
        vld1.16         {\q0},  [\src]
.else
        vld1.16         {\q0, \q1},  [\src]
.endif
.endm

.macro vst1_align_n d0, q0, q1, src, n
.if \n == 4
        vst1.16         {\d0},  [\src, :64]
.elseif \n == 8
        vst1.16         {\q0},  [\src, :128]
.else
        vst1.16         {\q0, \q1},  [\src, :128]
.endif
.endm

.macro vst1_n d0, q0, q1, src, n
.if \n == 4
        vst1.16         {\d0},  [\src]
.elseif \n == 8
        vst1.16         {\q0},  [\src]
.else
        vst1.16         {\q0, \q1},  [\src]
.endif
.endm

.macro vshr_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n
.if \n == 4
        vshr.u16        \d0,  \s0,  \s3
.else
        vshr.u16        \d1,  \s1,  \s4
.if \n == 16
        vshr.u16        \d2,  \s2,  \s5
.endif
.endif
.endm

.macro vadd_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n
.if \n == 4
        vadd.i16        \d0,  \s0,  \s3
.else
        vadd.i16        \d1,  \s1,  \s4
.if \n == 16
        vadd.i16        \d2,  \s2,  \s5
.endif
.endif
.endm

.macro vsub_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n
.if \n == 4
        vsub.i16        \d0,  \s0,  \s3
.else
        vsub.i16        \d1,  \s1,  \s4
.if \n == 16
        vsub.i16        \d2,  \s2,  \s5
.endif
.endif
.endm

.macro vand_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n
.if \n == 4
        vand            \d0,  \s0,  \s3
.else
        vand            \d1,  \s1,  \s4
.if \n == 16
        vand            \d2,  \s2,  \s5
.endif
.endif
.endm

.macro vcge_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n
.if \n == 4
        vcge.u16        \d0,  \s0,  \s3
.else
        vcge.u16        \d1,  \s1,  \s4
.if \n == 16
        vcge.u16        \d2,  \s2,  \s5
.endif
.endif
.endm

.macro vrhadd_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n
.if \n == 4
        vrhadd.u16      \d0,  \s0,  \s3
.else
        vrhadd.u16      \d1,  \s1,  \s4
.if \n == 16
        vrhadd.u16      \d2,  \s2,  \s5
.endif
.endif
.endm

.macro vshl_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n
.if \n == 4
        vshl.s16        \d0,  \s0,  \s3
.else
        vshl.s16        \d1,  \s1,  \s4
.if \n == 16
        vshl.s16        \d2,  \s2,  \s5
.endif
.endif
.endm

.macro vqdmulh_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n
.if \n == 4
        vqdmulh.s16     \d0,  \s0,  \s3
.else
        vqdmulh.s16     \d1,  \s1,  \s4
.if \n == 16
        vqdmulh.s16     \d2,  \s2,  \s5
.endif
.endif
.endm

// unsigned dav1d_msac_decode_symbol_adapt4_neon(MsacContext *s, uint16_t *cdf,
//                                               size_t n_symbols);

function msac_decode_symbol_adapt4_neon, export=1
.macro decode_update n
        push            {r4-r10,lr}
        sub             sp,  sp,  #48
        add             r8,  r0,  #RNG

        vld1_align_n    d0,  q0,  q1,  r1,  \n                         // cdf
        vld1.16         {d16[]}, [r8, :16]                             // rng
        movrel_local    r9,  coeffs, 30
        vmov.i16        d30, #0x7f00                                   // 0x7f00
        sub             r9,  r9,  r2, lsl #1
        vmvn.i16        q14, #0x3f                                     // 0xffc0
        add             r8,  sp,  #14
        vand            d22, d16, d30                                  // rng & 0x7f00
        vst1.16         {d16[0]}, [r8, :16]                            // store original u = s->rng
        vand_n          d4,  q2,  q3,  d0,  q0,  q1, d28, q14, q14, \n // cdf & 0xffc0
.if \n > 4
        vmov            d23, d22
.endif

        vld1_n          d16, q8,  q9,  r9,  \n                          // EC_MIN_PROB * (n_symbols - ret)
        vqdmulh_n       d20, q10, q11, d4,  q2,  q3,  d22, q11, q11, \n // ((cdf >> EC_PROB_SHIFT) * (r - 128)) >> 1
        add             r8,  r0,  #DIF + 2

        vadd_n          d16, q8,  q9,  d4,  q2,  q3,  d16, q8,  q9,  \n // v = cdf + EC_MIN_PROB * (n_symbols - ret)
.if \n == 4
        vmov.i16        d17, #0
.endif
        vadd_n          d16, q8,  q9,  d20, q10, q11, d16, q8,  q9,  \n // v = ((cdf >> EC_PROB_SHIFT) * r) >> 1 + EC_MIN_PROB * (n_symbols - ret)

        add             r9,  sp,  #16
        vld1.16         {d20[]}, [r8, :16]                              // dif >> (EC_WIN_SIZE - 16)
        movrel_local    r8,  bits
        vst1_n          q8,  q8,  q9,  r9,  \n                          // store v values to allow indexed access

        vmov            d21, d20
        vld1_align_n    q12, q12, q13, r8,  \n
.if \n == 16
        vmov            q11, q10
.endif

        vcge_n          q2,  q2,  q3,  q10, q10, q11, q8,  q8,  q9,  \n // c >= v

        vand_n          q10, q10, q11, q2,  q2,  q3,  q12, q12, q13, \n // One bit per halfword set in the mask
.if \n == 16
        vadd.i16        q10, q10, q11
.endif
        vadd.i16        d20, d20, d21                                   // Aggregate mask bits
        ldr             r4,  [r0, #ALLOW_UPDATE_CDF]
        vpadd.i16       d20, d20, d20
        lsl             r10, r2,  #1
        vpadd.i16       d20, d20, d20
        vmov.u16        r3,  d20[0]
        cmp             r4,  #0
        rbit            r3,  r3
        clz             lr,  r3                                         // ret

        beq             L(renorm)
        // update_cdf
        ldrh            r3,  [r1, r10]                                  // count = cdf[n_symbols]
        vmov.i8         q10, #0xff
.if \n == 16
        mov             r4,  #-5
.else
        mvn             r12, r2
        mov             r4,  #-4
        cmn             r12, #3                                         // set C if n_symbols <= 2
.endif
        vrhadd_n        d16, q8,  q9,  d20, q10, q10, d4,  q2,  q3,  \n // i >= val ? -1 : 32768
.if \n == 16
        sub             r4,  r4,  r3, lsr #4                            // -((count >> 4) + 5)
.else
        lsr             r12, r3,  #4                                    // count >> 4
        sbc             r4,  r4,  r12                                   // -((count >> 4) + (n_symbols > 2) + 4)
.endif
        vsub_n          d16, q8,  q9,  d16, q8,  q9,  d0,  q0,  q1,  \n // (32768 - cdf[i]) or (-1 - cdf[i])
.if \n == 4
        vdup.16         d20, r4                                         // -rate
.else
        vdup.16         q10, r4                                         // -rate
.endif

        sub             r3,  r3,  r3, lsr #5                            // count - (count == 32)
        vsub_n          d0,  q0,  q1,  d0,  q0,  q1,  d4,  q2,  q3,  \n // cdf + (i >= val ? 1 : 0)
        vshl_n          d16, q8,  q9,  d16, q8,  q9,  d20, q10, q10, \n // ({32768,-1} - cdf[i]) >> rate
        add             r3,  r3,  #1                                    // count + (count < 32)
        vadd_n          d0,  q0,  q1,  d0,  q0,  q1,  d16, q8,  q9,  \n // cdf + (32768 - cdf[i]) >> rate
        vst1_align_n    d0,  q0,  q1,  r1,  \n
        strh            r3,  [r1, r10]
.endm

        decode_update   4

L(renorm):
        add             r8,  sp,  #16
        add             r8,  r8,  lr, lsl #1
        ldrh            r3,  [r8]              // v
        ldrh            r4,  [r8, #-2]         // u
        ldr             r6,  [r0, #CNT]
        ldr             r7,  [r0, #DIF]
        sub             r4,  r4,  r3           // rng = u - v
        clz             r5,  r4                // clz(rng)
        eor             r5,  r5,  #16          // d = clz(rng) ^ 16
        sub             r7,  r7,  r3, lsl #16  // dif - (v << 16)
L(renorm2):
        lsl             r4,  r4,  r5           // rng << d
        subs            r6,  r6,  r5           // cnt -= d
        lsl             r7,  r7,  r5           // (dif - (v << 16)) << d
        str             r4,  [r0, #RNG]
        bhs             4f

        // refill
        ldr             r3,  [r0, #BUF_POS]    // BUF_POS
        ldr             r4,  [r0, #BUF_END]    // BUF_END
        add             r5,  r3,  #4
        subs            r5,  r5,  r4
        bhi             6f

        ldr             r8,  [r3]              // next_bits
        rsb             r5,  r6,  #16
        add             r4,  r6,  #16          // shift_bits = cnt + 16
        mvn             r8,  r8
        lsr             r5,  r5,  #3           // num_bytes_read
        rev             r8,  r8                // next_bits = bswap(next_bits)
        lsr             r8,  r8,  r4           // next_bits >>= shift_bits

2:      // refill_end
        add             r3,  r3,  r5
        add             r6,  r6,  r5, lsl #3   // cnt += num_bits_read
        str             r3,  [r0, #BUF_POS]

3:      // refill_end2
        orr             r7,  r7,  r8           // dif |= next_bits

4:      // end
        str             r6,  [r0, #CNT]
        str             r7,  [r0, #DIF]
        mov             r0,  lr
        add             sp,  sp,  #48
        pop             {r4-r10,pc}

5:      // pad_with_ones
        add             r8,  r6,  #-240
        lsr             r8,  r8,  r8
        b               3b

6:      // refill_eob
        cmp             r3,  r4
        bhs             5b

        ldr             r8,  [r4, #-4]
        lsl             r5,  r5,  #3
        lsr             r8,  r8,  r5
        add             r5,  r6,  #16
        mvn             r8,  r8
        sub             r4,  r4,  r3           // num_bytes_left
        rev             r8,  r8
        lsr             r8,  r8,  r5
        rsb             r5,  r6,  #16
        lsr             r5,  r5,  #3
        cmp             r5,  r4
        it              hs
        movhs           r5,  r4
        b               2b
endfunc

function msac_decode_symbol_adapt8_neon, export=1
        decode_update   8
        b               L(renorm)
endfunc

function msac_decode_symbol_adapt16_neon, export=1
        decode_update   16
        b               L(renorm)
endfunc

function msac_decode_hi_tok_neon, export=1
        push            {r4-r10,lr}
        vld1.16         {d0},  [r1, :64]       // cdf
        add             r4,  r0,  #RNG
        vmov.i16        d31, #0x7f00           // 0x7f00
        movrel_local    r5,  coeffs, 30-2*3
        vmvn.i16        d30, #0x3f             // 0xffc0
        ldrh            r9,  [r1, #6]          // count = cdf[n_symbols]
        vld1.16         {d1[]},  [r4, :16]     // rng
        movrel_local    r4,  bits
        vld1.16         {d29}, [r5]            // EC_MIN_PROB * (n_symbols - ret)
        add             r5,  r0,  #DIF + 2
        vld1.16         {q8}, [r4, :128]
        mov             r2,  #-24
        vand            d20, d0, d30           // cdf & 0xffc0
        ldr             r10, [r0, #ALLOW_UPDATE_CDF]
        vld1.16         {d2[]}, [r5, :16]      // dif >> (EC_WIN_SIZE - 16)
        sub             sp,  sp,  #48
        ldr             r6,  [r0, #CNT]
        ldr             r7,  [r0, #DIF]
        vmov            d3,  d2
1:
        vand            d23, d1,  d31          // rng & 0x7f00
        vqdmulh.s16     d18, d20, d23          // ((cdf >> EC_PROB_SHIFT) * (r - 128)) >> 1
        add             r12, sp,  #14
        vadd.i16        d6,  d20, d29          // v = cdf + EC_MIN_PROB * (n_symbols - ret)
        vadd.i16        d6,  d18, d6           // v = ((cdf >> EC_PROB_SHIFT) * r) >> 1 + EC_MIN_PROB * (n_symbols - ret)
        vmov.i16        d7,  #0
        vst1.16         {d1[0]}, [r12, :16]    // store original u = s->rng
        add             r12, sp,  #16
        vcge.u16        q2,  q1,  q3           // c >= v
        vst1.16         {q3},  [r12]           // store v values to allow indexed access
        vand            q9,  q2,  q8           // One bit per halfword set in the mask

        vadd.i16        d18, d18, d19          // Aggregate mask bits
        vpadd.i16       d18, d18, d18
        vpadd.i16       d18, d18, d18
        vmov.u16        r3,  d18[0]
        cmp             r10, #0
        add             r2,  r2,  #5
        rbit            r3,  r3
        add             r8,  sp,  #16
        clz             lr,  r3                // ret

        beq             2f
        // update_cdf
        vmov.i8         d22, #0xff
        mov             r4,  #-5
        vrhadd.u16      d6,  d22, d4           // i >= val ? -1 : 32768
        sub             r4,  r4,  r9, lsr #4   // -((count >> 4) + 5)
        vsub.i16        d6,  d6,  d0           // (32768 - cdf[i]) or (-1 - cdf[i])
        vdup.16         d18, r4                // -rate

        sub             r9,  r9,  r9, lsr #5   // count - (count == 32)
        vsub.i16        d0,  d0,  d4           // cdf + (i >= val ? 1 : 0)
        vshl.s16        d6,  d6,  d18          // ({32768,-1} - cdf[i]) >> rate
        add             r9,  r9,  #1           // count + (count < 32)
        vadd.i16        d0,  d0,  d6           // cdf + (32768 - cdf[i]) >> rate
        vst1.16         {d0},  [r1, :64]
        vand            d20, d0,  d30          // cdf & 0xffc0
        strh            r9,  [r1, #6]

2:
        add             r8,  r8,  lr, lsl #1
        ldrh            r3,  [r8]              // v
        ldrh            r4,  [r8, #-2]         // u
        sub             r4,  r4,  r3           // rng = u - v
        clz             r5,  r4                // clz(rng)
        eor             r5,  r5,  #16          // d = clz(rng) ^ 16
        sub             r7,  r7,  r3, lsl #16  // dif - (v << 16)
        lsl             r4,  r4,  r5           // rng << d
        subs            r6,  r6,  r5           // cnt -= d
        lsl             r7,  r7,  r5           // (dif - (v << 16)) << d
        str             r4,  [r0, #RNG]
        vdup.16         d1,  r4
        bhs             5f

        // refill
        ldr             r3,  [r0, #BUF_POS]    // BUF_POS
        ldr             r4,  [r0, #BUF_END]    // BUF_END
        add             r5,  r3,  #4
        subs            r5,  r5,  r4
        bhi             7f

        ldr             r8,  [r3]              // next_bits
        rsb             r5,  r6,  #16
        add             r4,  r6,  #16          // shift_bits = cnt + 16
        mvn             r8,  r8
        lsr             r5,  r5,  #3           // num_bytes_read
        rev             r8,  r8                // next_bits = bswap(next_bits)
        lsr             r8,  r8,  r4           // next_bits >>= shift_bits

3:      // refill_end
        add             r3,  r3,  r5
        add             r6,  r6,  r5, lsl #3   // cnt += num_bits_read
        str             r3,  [r0, #BUF_POS]

4:      // refill_end2
        orr             r7,  r7,  r8           // dif |= next_bits

5:      // end
        lsl             lr,  lr,  #1
        sub             lr,  lr,  #5
        lsr             r12, r7,  #16
        adds            r2,  r2,  lr           // carry = tok_br < 3 || tok == 15
        vdup.16         q1,  r12
        bcc             1b                     // loop if !carry
        add             r2,  r2,  #30
        str             r6,  [r0, #CNT]
        add             sp,  sp,  #48
        str             r7,  [r0, #DIF]
        lsr             r0,  r2,  #1
        pop             {r4-r10,pc}

6:      // pad_with_ones
        add             r8,  r6,  #-240
        lsr             r8,  r8,  r8
        b               4b

7:      // refill_eob
        cmp             r3,  r4
        bhs             6b

        ldr             r8,  [r4, #-4]
        lsl             r5,  r5,  #3
        lsr             r8,  r8,  r5
        add             r5,  r6,  #16
        mvn             r8,  r8
        sub             r4,  r4,  r3           // num_bytes_left
        rev             r8,  r8
        lsr             r8,  r8,  r5
        rsb             r5,  r6,  #16
        lsr             r5,  r5,  #3
        cmp             r5,  r4
        it              hs
        movhs           r5,  r4
        b               3b
endfunc

function msac_decode_bool_equi_neon, export=1
        push            {r4-r10,lr}
        ldr             r5,  [r0, #RNG]
        ldr             r6,  [r0, #CNT]
        sub             sp,  sp,  #48
        ldr             r7,  [r0, #DIF]
        bic             r4,  r5,  #0xff        // r &= 0xff00
        add             r4,  r4,  #8
        mov             r2,  #0
        subs            r8,  r7,  r4, lsl #15  // dif - vw
        lsr             r4,  r4,  #1           // v
        sub             r5,  r5,  r4           // r - v
        itee            lo
        movlo           r2,  #1
        movhs           r4,  r5                // if (ret) v = r - v;
        movhs           r7,  r8                // if (ret) dif = dif - vw;

        clz             r5,  r4                // clz(rng)
        eor             r5,  r5,  #16          // d = clz(rng) ^ 16
        mov             lr,  r2
        b               L(renorm2)
endfunc

function msac_decode_bool_neon, export=1
        push            {r4-r10,lr}
        ldr             r5,  [r0, #RNG]
        ldr             r6,  [r0, #CNT]
        sub             sp,  sp,  #48
        ldr             r7,  [r0, #DIF]
        lsr             r4,  r5,  #8           // r >> 8
        bic             r1,  r1,  #0x3f        // f &= ~63
        mul             r4,  r4,  r1
        mov             r2,  #0
        lsr             r4,  r4,  #7
        add             r4,  r4,  #4           // v
        subs            r8,  r7,  r4, lsl #16  // dif - vw
        sub             r5,  r5,  r4           // r - v
        itee            lo
        movlo           r2,  #1
        movhs           r4,  r5                // if (ret) v = r - v;
        movhs           r7,  r8                // if (ret) dif = dif - vw;

        clz             r5,  r4                // clz(rng)
        eor             r5,  r5,  #16          // d = clz(rng) ^ 16
        mov             lr,  r2
        b               L(renorm2)
endfunc

function msac_decode_bool_adapt_neon, export=1
        push            {r4-r10,lr}
        ldr             r9,  [r1]              // cdf[0-1]
        ldr             r5,  [r0, #RNG]
        movw            lr,  #0xffc0
        ldr             r6,  [r0, #CNT]
        sub             sp,  sp,  #48
        ldr             r7,  [r0, #DIF]
        lsr             r4,  r5,  #8           // r >> 8
        and             r2,  r9,  lr           // f &= ~63
        mul             r4,  r4,  r2
        mov             r2,  #0
        lsr             r4,  r4,  #7
        add             r4,  r4,  #4           // v
        subs            r8,  r7,  r4, lsl #16  // dif - vw
        sub             r5,  r5,  r4           // r - v
        ldr             r10, [r0, #ALLOW_UPDATE_CDF]
        itee            lo
        movlo           r2,  #1
        movhs           r4,  r5                // if (ret) v = r - v;
        movhs           r7,  r8                // if (ret) dif = dif - vw;

        cmp             r10, #0
        clz             r5,  r4                // clz(rng)
        eor             r5,  r5,  #16          // d = clz(rng) ^ 16
        mov             lr,  r2

        beq             L(renorm2)

        lsr             r2,  r9,  #16          // count = cdf[1]
        uxth            r9,  r9                // cdf[0]

        sub             r3,  r2,  r2,  lsr #5  // count - (count >= 32)
        lsr             r2,  r2,  #4           // count >> 4
        add             r10, r3,  #1           // count + (count < 32)
        add             r2,  r2,  #4           // rate = (count >> 4) | 4

        sub             r9,  r9,  lr           // cdf[0] -= bit
        sub             r3,  r9,  lr,  lsl #15 // {cdf[0], cdf[0] - 32769}
        asr             r3,  r3,  r2           // {cdf[0], cdf[0] - 32769} >> rate
        sub             r9,  r9,  r3           // cdf[0]

        strh            r9,  [r1]
        strh            r10, [r1, #2]

        b               L(renorm2)
endfunc