xref: /aosp_15_r20/external/libaom/av1/encoder/arm/pickrst_neon.h (revision 77c1e3ccc04c968bd2bc212e87364f250e820521)

/*
 * Copyright (c) 2023, Alliance for Open Media. All rights reserved.
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#ifndef AOM_AV1_ENCODER_ARM_PICKRST_NEON_H_
#define AOM_AV1_ENCODER_ARM_PICKRST_NEON_H_

#include <arm_neon.h>

#include "av1/common/restoration.h"

#define WIN_7 ((WIENER_WIN - 1) * 2)
#define WIN_CHROMA ((WIENER_WIN_CHROMA - 1) * 2)

// Aligned sizes for Wiener filters.
#define WIENER_WIN2_ALIGN2 ALIGN_POWER_OF_TWO(WIENER_WIN2, 2)
#define WIENER_WIN2_ALIGN3 ALIGN_POWER_OF_TWO(WIENER_WIN2, 3)
#define WIENER_WIN2_REDUCED ((WIENER_WIN_REDUCED) * (WIENER_WIN_REDUCED))
#define WIENER_WIN2_REDUCED_ALIGN2 ALIGN_POWER_OF_TWO(WIENER_WIN2_REDUCED, 2)
#define WIENER_WIN2_REDUCED_ALIGN3 ALIGN_POWER_OF_TWO(WIENER_WIN2_REDUCED, 3)

// Compute 8 values of M (cross correlation) for a single source pixel and
// accumulate.
static inline void update_M_1pixel(int32_t *M_s32, int16x4_t src_avg,
                                   int16x8_t dgd_avg) {
  int32x4_t lo = vld1q_s32(M_s32 + 0);
  int32x4_t hi = vld1q_s32(M_s32 + 4);

  lo = vmlal_s16(lo, vget_low_s16(dgd_avg), src_avg);
  hi = vmlal_s16(hi, vget_high_s16(dgd_avg), src_avg);

  vst1q_s32(M_s32 + 0, lo);
  vst1q_s32(M_s32 + 4, hi);
}

// Compute 8 values of M (cross correlation) for two source pixels and
// accumulate.
static inline void update_M_2pixels(int32_t *M_s32, int16x4_t src_avg0,
                                    int16x4_t src_avg1, int16x8_t dgd_avg0,
                                    int16x8_t dgd_avg1) {
  int32x4_t lo = vld1q_s32(M_s32 + 0);
  int32x4_t hi = vld1q_s32(M_s32 + 4);

  lo = vmlal_s16(lo, vget_low_s16(dgd_avg0), src_avg0);
  hi = vmlal_s16(hi, vget_high_s16(dgd_avg0), src_avg0);
  lo = vmlal_s16(lo, vget_low_s16(dgd_avg1), src_avg1);
  hi = vmlal_s16(hi, vget_high_s16(dgd_avg1), src_avg1);

  vst1q_s32(M_s32 + 0, lo);
  vst1q_s32(M_s32 + 4, hi);
}

static inline void update_H_1pixel(int32_t *H_s32, const int16_t *dgd_avg,
                                   int width, int height) {
  for (int i = 0; i < height; i += 4) {
    int16x4_t di = vld1_s16(dgd_avg + i);

    for (int j = i; j < width; j += 4) {
      int16x4_t dj = vld1_s16(dgd_avg + j);
      int32x4_t h0 = vld1q_s32(H_s32 + 0 * width + j);
      int32x4_t h1 = vld1q_s32(H_s32 + 1 * width + j);
      int32x4_t h2 = vld1q_s32(H_s32 + 2 * width + j);
      int32x4_t h3 = vld1q_s32(H_s32 + 3 * width + j);

      h0 = vmlal_lane_s16(h0, dj, di, 0);
      h1 = vmlal_lane_s16(h1, dj, di, 1);
      h2 = vmlal_lane_s16(h2, dj, di, 2);
      h3 = vmlal_lane_s16(h3, dj, di, 3);

      vst1q_s32(H_s32 + 0 * width + j, h0);
      vst1q_s32(H_s32 + 1 * width + j, h1);
      vst1q_s32(H_s32 + 2 * width + j, h2);
      vst1q_s32(H_s32 + 3 * width + j, h3);
    }
    H_s32 += 4 * width;
  }
}

static inline void update_H_5x5_2pixels(int32_t *H_s32, const int16_t *dgd_avg0,
                                        const int16_t *dgd_avg1) {
  for (int i = 0; i < 24; i += 4) {
    int16x4_t di0 = vld1_s16(dgd_avg0 + i);
    int16x4_t di1 = vld1_s16(dgd_avg1 + i);

    for (int j = i + 0; j < WIENER_WIN2_REDUCED_ALIGN2; j += 4) {
      int16x4_t dj0 = vld1_s16(dgd_avg0 + j);
      int16x4_t dj1 = vld1_s16(dgd_avg1 + j);
      int32x4_t h0 = vld1q_s32(H_s32 + 0 * WIENER_WIN2_REDUCED_ALIGN2 + j);
      int32x4_t h1 = vld1q_s32(H_s32 + 1 * WIENER_WIN2_REDUCED_ALIGN2 + j);
      int32x4_t h2 = vld1q_s32(H_s32 + 2 * WIENER_WIN2_REDUCED_ALIGN2 + j);
      int32x4_t h3 = vld1q_s32(H_s32 + 3 * WIENER_WIN2_REDUCED_ALIGN2 + j);

      h0 = vmlal_lane_s16(h0, dj0, di0, 0);
      h0 = vmlal_lane_s16(h0, dj1, di1, 0);
      h1 = vmlal_lane_s16(h1, dj0, di0, 1);
      h1 = vmlal_lane_s16(h1, dj1, di1, 1);
      h2 = vmlal_lane_s16(h2, dj0, di0, 2);
      h2 = vmlal_lane_s16(h2, dj1, di1, 2);
      h3 = vmlal_lane_s16(h3, dj0, di0, 3);
      h3 = vmlal_lane_s16(h3, dj1, di1, 3);

      vst1q_s32(H_s32 + 0 * WIENER_WIN2_REDUCED_ALIGN2 + j, h0);
      vst1q_s32(H_s32 + 1 * WIENER_WIN2_REDUCED_ALIGN2 + j, h1);
      vst1q_s32(H_s32 + 2 * WIENER_WIN2_REDUCED_ALIGN2 + j, h2);
      vst1q_s32(H_s32 + 3 * WIENER_WIN2_REDUCED_ALIGN2 + j, h3);
    }
    H_s32 += 4 * WIENER_WIN2_REDUCED_ALIGN2;
  }
}

static inline void update_H_7x7_2pixels(int32_t *H_s32, const int16_t *dgd_avg0,
                                        const int16_t *dgd_avg1) {
  for (int i = 0; i < 48; i += 4) {
    int16x4_t di0 = vld1_s16(dgd_avg0 + i);
    int16x4_t di1 = vld1_s16(dgd_avg1 + i);

    int32x4_t h0 = vld1q_s32(H_s32 + 0 * WIENER_WIN2_ALIGN2 + i);
    int32x4_t h1 = vld1q_s32(H_s32 + 1 * WIENER_WIN2_ALIGN2 + i);
    int32x4_t h2 = vld1q_s32(H_s32 + 2 * WIENER_WIN2_ALIGN2 + i);
    int32x4_t h3 = vld1q_s32(H_s32 + 3 * WIENER_WIN2_ALIGN2 + i);

    h0 = vmlal_lane_s16(h0, di0, di0, 0);
    h0 = vmlal_lane_s16(h0, di1, di1, 0);
    h1 = vmlal_lane_s16(h1, di0, di0, 1);
    h1 = vmlal_lane_s16(h1, di1, di1, 1);
    h2 = vmlal_lane_s16(h2, di0, di0, 2);
    h2 = vmlal_lane_s16(h2, di1, di1, 2);
    h3 = vmlal_lane_s16(h3, di0, di0, 3);
    h3 = vmlal_lane_s16(h3, di1, di1, 3);

    vst1q_s32(H_s32 + 0 * WIENER_WIN2_ALIGN2 + i, h0);
    vst1q_s32(H_s32 + 1 * WIENER_WIN2_ALIGN2 + i, h1);
    vst1q_s32(H_s32 + 2 * WIENER_WIN2_ALIGN2 + i, h2);
    vst1q_s32(H_s32 + 3 * WIENER_WIN2_ALIGN2 + i, h3);

    for (int j = i + 4; j < WIENER_WIN2_ALIGN2; j += 4) {
      int16x4_t dj0 = vld1_s16(dgd_avg0 + j);
      int16x4_t dj1 = vld1_s16(dgd_avg1 + j);
      h0 = vld1q_s32(H_s32 + 0 * WIENER_WIN2_ALIGN2 + j);
      h1 = vld1q_s32(H_s32 + 1 * WIENER_WIN2_ALIGN2 + j);
      h2 = vld1q_s32(H_s32 + 2 * WIENER_WIN2_ALIGN2 + j);
      h3 = vld1q_s32(H_s32 + 3 * WIENER_WIN2_ALIGN2 + j);

      h0 = vmlal_lane_s16(h0, dj0, di0, 0);
      h0 = vmlal_lane_s16(h0, dj1, di1, 0);
      h1 = vmlal_lane_s16(h1, dj0, di0, 1);
      h1 = vmlal_lane_s16(h1, dj1, di1, 1);
      h2 = vmlal_lane_s16(h2, dj0, di0, 2);
      h2 = vmlal_lane_s16(h2, dj1, di1, 2);
      h3 = vmlal_lane_s16(h3, dj0, di0, 3);
      h3 = vmlal_lane_s16(h3, dj1, di1, 3);

      vst1q_s32(H_s32 + 0 * WIENER_WIN2_ALIGN2 + j, h0);
      vst1q_s32(H_s32 + 1 * WIENER_WIN2_ALIGN2 + j, h1);
      vst1q_s32(H_s32 + 2 * WIENER_WIN2_ALIGN2 + j, h2);
      vst1q_s32(H_s32 + 3 * WIENER_WIN2_ALIGN2 + j, h3);
    }
    H_s32 += 4 * WIENER_WIN2_ALIGN2;
  }
}

// Widen 32-bit src data and accumulate into 64-bit dst. Clear src data.
static inline void accumulate_and_clear(int64_t *dst, int32_t *src,
                                        int length) {
  do {
    int32x4_t s32 = vld1q_s32(src);
    vst1q_s32(src, vdupq_n_s32(0));
    src += 4;

    int64x2_t d_lo = vld1q_s64(dst + 0);
    int64x2_t d_hi = vld1q_s64(dst + 2);

    d_lo = vaddw_s32(d_lo, vget_low_s32(s32));
    d_hi = vaddw_s32(d_hi, vget_high_s32(s32));

    vst1q_s64(dst + 0, d_lo);
    vst1q_s64(dst + 2, d_hi);

    dst += 4;
    length -= 4;
  } while (length > 0);
}

// clang-format off
// Constant pool to act as a mask to zero n top elements in an int16x8_t vector.
// The index we load from depends on n.
static const int16_t mask_16bit[32] = {
  0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
  0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
       0,      0,      0,      0,      0,      0,      0,      0,
       0,      0,      0,      0,      0,      0,      0,      0,
};
// clang-format on

static inline void madd_neon_pairwise(int32x4_t *sum, const int16x8_t src,
                                      const int16x8_t dgd) {
  const int32x4_t sd =
      horizontal_add_2d_s32(vmull_s16(vget_low_s16(src), vget_low_s16(dgd)),
                            vmull_s16(vget_high_s16(src), vget_high_s16(dgd)));
  *sum = vaddq_s32(*sum, sd);
}

static inline void madd_neon(int32x4_t *sum, const int16x8_t src,
                             const int16x8_t dgd) {
  *sum = vmlal_s16(*sum, vget_low_s16(src), vget_low_s16(dgd));
  *sum = vmlal_s16(*sum, vget_high_s16(src), vget_high_s16(dgd));
}

static inline void msub_neon(int32x4_t *sum, const int16x8_t src,
                             const int16x8_t dgd) {
  *sum = vmlsl_s16(*sum, vget_low_s16(src), vget_low_s16(dgd));
  *sum = vmlsl_s16(*sum, vget_high_s16(src), vget_high_s16(dgd));
}

static inline void compute_delta_step3(int32x4_t *sum0, int32x4_t *sum1,
                                       const int16x8_t src0,
                                       const int16x8_t src1,
                                       const int16x8_t dgd0,
                                       const int16x8_t dgd1) {
  *sum0 = vmlsl_s16(*sum0, vget_low_s16(src0), vget_low_s16(dgd0));
  *sum0 = vmlal_s16(*sum0, vget_low_s16(src1), vget_low_s16(dgd1));
  *sum1 = vmlsl_s16(*sum1, vget_high_s16(src0), vget_high_s16(dgd0));
  *sum1 = vmlal_s16(*sum1, vget_high_s16(src1), vget_high_s16(dgd1));
}

static inline int32x4_t hadd_four_32_neon(const int32x4_t src0,
                                          const int32x4_t src1,
                                          const int32x4_t src2,
                                          const int32x4_t src3) {
  int32x4_t src[4] = { src0, src1, src2, src3 };
  return horizontal_add_4d_s32x4(src);
}

static inline void update_4_stats_neon(const int64_t *const src,
                                       const int32x4_t delta,
                                       int64_t *const dst) {
  const int64x2_t s1 = vld1q_s64(src);
  const int64x2_t s2 = vld1q_s64(src + 2);

  const int64x2_t d1 = vaddw_s32(s1, vget_low_s32(delta));
  const int64x2_t d2 = vaddw_s32(s2, vget_high_s32(delta));

  vst1q_s64(dst, d1);
  vst1q_s64(dst + 2, d2);
}

static inline void load_more_16_neon(const int16_t *const src,
                                     const int32_t width,
                                     const int16x8_t org[2], int16x8_t dst[2]) {
  int16x8_t s0 = vld1q_dup_s16(src);
  int16x8_t s1 = vld1q_dup_s16(src + width);
  dst[0] = vextq_s16(org[0], s0, 1);
  dst[1] = vextq_s16(org[1], s1, 1);
}

static inline void stats_top_win5_neon(const int16x8_t src[2],
                                       const int16x8_t dgd[2],
                                       const int16_t *const d,
                                       const int32_t d_stride, int32x4_t *sum_m,
                                       int32x4_t *sum_h) {
  int16x8_t dgds[WIENER_WIN_CHROMA * 2];

  load_s16_8x5(d + 0, d_stride, &dgds[0], &dgds[2], &dgds[4], &dgds[6],
               &dgds[8]);
  load_s16_8x5(d + 8, d_stride, &dgds[1], &dgds[3], &dgds[5], &dgds[7],
               &dgds[9]);

  madd_neon(&sum_m[0], src[0], dgds[0]);
  madd_neon(&sum_m[0], src[1], dgds[1]);
  madd_neon(&sum_m[1], src[0], dgds[2]);
  madd_neon(&sum_m[1], src[1], dgds[3]);
  madd_neon(&sum_m[2], src[0], dgds[4]);
  madd_neon(&sum_m[2], src[1], dgds[5]);
  madd_neon(&sum_m[3], src[0], dgds[6]);
  madd_neon(&sum_m[3], src[1], dgds[7]);
  madd_neon(&sum_m[4], src[0], dgds[8]);
  madd_neon(&sum_m[4], src[1], dgds[9]);

  madd_neon(&sum_h[0], dgd[0], dgds[0]);
  madd_neon(&sum_h[0], dgd[1], dgds[1]);
  madd_neon(&sum_h[1], dgd[0], dgds[2]);
  madd_neon(&sum_h[1], dgd[1], dgds[3]);
  madd_neon(&sum_h[2], dgd[0], dgds[4]);
  madd_neon(&sum_h[2], dgd[1], dgds[5]);
  madd_neon(&sum_h[3], dgd[0], dgds[6]);
  madd_neon(&sum_h[3], dgd[1], dgds[7]);
  madd_neon(&sum_h[4], dgd[0], dgds[8]);
  madd_neon(&sum_h[4], dgd[1], dgds[9]);
}

static inline void stats_left_win5_neon(const int16x8_t src[2],
                                        const int16_t *d,
                                        const int32_t d_stride,
                                        int32x4_t *sum) {
  int16x8_t dgds[WIN_CHROMA];

  load_s16_8x4(d + d_stride + 0, d_stride, &dgds[0], &dgds[2], &dgds[4],
               &dgds[6]);
  load_s16_8x4(d + d_stride + 8, d_stride, &dgds[1], &dgds[3], &dgds[5],
               &dgds[7]);

  madd_neon(&sum[0], src[0], dgds[0]);
  madd_neon(&sum[0], src[1], dgds[1]);
  madd_neon(&sum[1], src[0], dgds[2]);
  madd_neon(&sum[1], src[1], dgds[3]);
  madd_neon(&sum[2], src[0], dgds[4]);
  madd_neon(&sum[2], src[1], dgds[5]);
  madd_neon(&sum[3], src[0], dgds[6]);
  madd_neon(&sum[3], src[1], dgds[7]);
}

static inline void derive_square_win5_neon(
    const int16x8_t *d_is, const int16x8_t *d_ie, const int16x8_t *d_js,
    const int16x8_t *d_je,
    int32x4_t deltas[WIENER_WIN_CHROMA - 1][WIENER_WIN_CHROMA - 1]) {
  msub_neon(&deltas[0][0], d_is[0], d_js[0]);
  msub_neon(&deltas[0][0], d_is[1], d_js[1]);
  msub_neon(&deltas[0][1], d_is[0], d_js[2]);
  msub_neon(&deltas[0][1], d_is[1], d_js[3]);
  msub_neon(&deltas[0][2], d_is[0], d_js[4]);
  msub_neon(&deltas[0][2], d_is[1], d_js[5]);
  msub_neon(&deltas[0][3], d_is[0], d_js[6]);
  msub_neon(&deltas[0][3], d_is[1], d_js[7]);

  msub_neon(&deltas[1][0], d_is[2], d_js[0]);
  msub_neon(&deltas[1][0], d_is[3], d_js[1]);
  msub_neon(&deltas[1][1], d_is[2], d_js[2]);
  msub_neon(&deltas[1][1], d_is[3], d_js[3]);
  msub_neon(&deltas[1][2], d_is[2], d_js[4]);
  msub_neon(&deltas[1][2], d_is[3], d_js[5]);
  msub_neon(&deltas[1][3], d_is[2], d_js[6]);
  msub_neon(&deltas[1][3], d_is[3], d_js[7]);

  msub_neon(&deltas[2][0], d_is[4], d_js[0]);
  msub_neon(&deltas[2][0], d_is[5], d_js[1]);
  msub_neon(&deltas[2][1], d_is[4], d_js[2]);
  msub_neon(&deltas[2][1], d_is[5], d_js[3]);
  msub_neon(&deltas[2][2], d_is[4], d_js[4]);
  msub_neon(&deltas[2][2], d_is[5], d_js[5]);
  msub_neon(&deltas[2][3], d_is[4], d_js[6]);
  msub_neon(&deltas[2][3], d_is[5], d_js[7]);

  msub_neon(&deltas[3][0], d_is[6], d_js[0]);
  msub_neon(&deltas[3][0], d_is[7], d_js[1]);
  msub_neon(&deltas[3][1], d_is[6], d_js[2]);
  msub_neon(&deltas[3][1], d_is[7], d_js[3]);
  msub_neon(&deltas[3][2], d_is[6], d_js[4]);
  msub_neon(&deltas[3][2], d_is[7], d_js[5]);
  msub_neon(&deltas[3][3], d_is[6], d_js[6]);
  msub_neon(&deltas[3][3], d_is[7], d_js[7]);

  madd_neon(&deltas[0][0], d_ie[0], d_je[0]);
  madd_neon(&deltas[0][0], d_ie[1], d_je[1]);
  madd_neon(&deltas[0][1], d_ie[0], d_je[2]);
  madd_neon(&deltas[0][1], d_ie[1], d_je[3]);
  madd_neon(&deltas[0][2], d_ie[0], d_je[4]);
  madd_neon(&deltas[0][2], d_ie[1], d_je[5]);
  madd_neon(&deltas[0][3], d_ie[0], d_je[6]);
  madd_neon(&deltas[0][3], d_ie[1], d_je[7]);

  madd_neon(&deltas[1][0], d_ie[2], d_je[0]);
  madd_neon(&deltas[1][0], d_ie[3], d_je[1]);
  madd_neon(&deltas[1][1], d_ie[2], d_je[2]);
  madd_neon(&deltas[1][1], d_ie[3], d_je[3]);
  madd_neon(&deltas[1][2], d_ie[2], d_je[4]);
  madd_neon(&deltas[1][2], d_ie[3], d_je[5]);
  madd_neon(&deltas[1][3], d_ie[2], d_je[6]);
  madd_neon(&deltas[1][3], d_ie[3], d_je[7]);

  madd_neon(&deltas[2][0], d_ie[4], d_je[0]);
  madd_neon(&deltas[2][0], d_ie[5], d_je[1]);
  madd_neon(&deltas[2][1], d_ie[4], d_je[2]);
  madd_neon(&deltas[2][1], d_ie[5], d_je[3]);
  madd_neon(&deltas[2][2], d_ie[4], d_je[4]);
  madd_neon(&deltas[2][2], d_ie[5], d_je[5]);
  madd_neon(&deltas[2][3], d_ie[4], d_je[6]);
  madd_neon(&deltas[2][3], d_ie[5], d_je[7]);

  madd_neon(&deltas[3][0], d_ie[6], d_je[0]);
  madd_neon(&deltas[3][0], d_ie[7], d_je[1]);
  madd_neon(&deltas[3][1], d_ie[6], d_je[2]);
  madd_neon(&deltas[3][1], d_ie[7], d_je[3]);
  madd_neon(&deltas[3][2], d_ie[6], d_je[4]);
  madd_neon(&deltas[3][2], d_ie[7], d_je[5]);
  madd_neon(&deltas[3][3], d_ie[6], d_je[6]);
  madd_neon(&deltas[3][3], d_ie[7], d_je[7]);
}

static inline void load_square_win5_neon(const int16_t *const di,
                                         const int16_t *const dj,
                                         const int32_t d_stride,
                                         const int32_t height, int16x8_t *d_is,
                                         int16x8_t *d_ie, int16x8_t *d_js,
                                         int16x8_t *d_je) {
  load_s16_8x4(di + 0, d_stride, &d_is[0], &d_is[2], &d_is[4], &d_is[6]);
  load_s16_8x4(di + 8, d_stride, &d_is[1], &d_is[3], &d_is[5], &d_is[7]);
  load_s16_8x4(dj + 0, d_stride, &d_js[0], &d_js[2], &d_js[4], &d_js[6]);
  load_s16_8x4(dj + 8, d_stride, &d_js[1], &d_js[3], &d_js[5], &d_js[7]);

  load_s16_8x4(di + height * d_stride + 0, d_stride, &d_ie[0], &d_ie[2],
               &d_ie[4], &d_ie[6]);
  load_s16_8x4(di + height * d_stride + 8, d_stride, &d_ie[1], &d_ie[3],
               &d_ie[5], &d_ie[7]);
  load_s16_8x4(dj + height * d_stride + 0, d_stride, &d_je[0], &d_je[2],
               &d_je[4], &d_je[6]);
  load_s16_8x4(dj + height * d_stride + 8, d_stride, &d_je[1], &d_je[3],
               &d_je[5], &d_je[7]);
}

static inline void update_5_stats_neon(const int64_t *const src,
                                       const int32x4_t delta,
                                       const int64_t delta4,
                                       int64_t *const dst) {
  update_4_stats_neon(src + 0, delta, dst + 0);
  dst[4] = src[4] + delta4;
}

static inline void compute_delta_step3_two_lines(int32x4_t *sum,
                                                 const int16x8_t src,
                                                 const int16x8_t dgd) {
  *sum = vmlsl_s16(*sum, vget_low_s16(src), vget_low_s16(dgd));
  *sum = vmlal_s16(*sum, vget_high_s16(src), vget_high_s16(dgd));
}

static inline void step3_win5_neon(const int16_t *d, const int32_t d_stride,
                                   const int32_t width, const int32_t height,
                                   int16x8_t *ds, int32x4_t *deltas) {
  int32_t y = height;
  do {
    ds[4] = load_unaligned_s16_4x2(d + 0 * d_stride, width);
    ds[5] = load_unaligned_s16_4x2(d + 1 * d_stride, width);

    compute_delta_step3_two_lines(&deltas[0], ds[0], ds[0]);
    compute_delta_step3_two_lines(&deltas[1], ds[0], ds[1]);
    compute_delta_step3_two_lines(&deltas[2], ds[0], ds[2]);
    compute_delta_step3_two_lines(&deltas[3], ds[0], ds[3]);
    compute_delta_step3_two_lines(&deltas[4], ds[0], ds[4]);
    compute_delta_step3_two_lines(&deltas[0], ds[1], ds[1]);
    compute_delta_step3_two_lines(&deltas[1], ds[1], ds[2]);
    compute_delta_step3_two_lines(&deltas[2], ds[1], ds[3]);
    compute_delta_step3_two_lines(&deltas[3], ds[1], ds[4]);
    compute_delta_step3_two_lines(&deltas[4], ds[1], ds[5]);

    ds[0] = ds[2];
    ds[1] = ds[3];
    ds[2] = ds[4];
    ds[3] = ds[5];

    d += 2 * d_stride;
    y -= 2;
  } while (y);
}

static inline void step3_win5_oneline_neon(const int16_t **const d,
                                           const int32_t d_stride,
                                           const int32_t width,
                                           const int32_t height, int16x8_t *ds,
                                           int32x4_t *deltas) {
  int32_t y = height;
  do {
    ds[8] = vld1q_s16(*d);
    ds[9] = vld1q_s16(*d + width);

    compute_delta_step3(&deltas[0], &deltas[4], ds[0], ds[1], ds[0], ds[1]);
    compute_delta_step3(&deltas[1], &deltas[5], ds[0], ds[1], ds[2], ds[3]);
    compute_delta_step3(&deltas[2], &deltas[6], ds[0], ds[1], ds[4], ds[5]);
    compute_delta_step3(&deltas[3], &deltas[7], ds[0], ds[1], ds[6], ds[7]);
    compute_delta_step3(&deltas[8], &deltas[12], ds[0], ds[1], ds[8], ds[9]);

    ds[0] = ds[2];
    ds[1] = ds[3];
    ds[2] = ds[4];
    ds[3] = ds[5];
    ds[4] = ds[6];
    ds[5] = ds[7];
    ds[6] = ds[8];
    ds[7] = ds[9];

    *d += d_stride;
  } while (--y);
}

static inline void derive_triangle_win5_neon(const int16x8_t *d_is,
                                             const int16x8_t *d_ie,
                                             int32x4_t *deltas) {
  msub_neon(&deltas[0], d_is[0], d_is[0]);
  msub_neon(&deltas[0], d_is[1], d_is[1]);
  msub_neon(&deltas[1], d_is[0], d_is[2]);
  msub_neon(&deltas[1], d_is[1], d_is[3]);
  msub_neon(&deltas[2], d_is[0], d_is[4]);
  msub_neon(&deltas[2], d_is[1], d_is[5]);
  msub_neon(&deltas[3], d_is[0], d_is[6]);
  msub_neon(&deltas[3], d_is[1], d_is[7]);
  msub_neon(&deltas[4], d_is[2], d_is[2]);
  msub_neon(&deltas[4], d_is[3], d_is[3]);
  msub_neon(&deltas[5], d_is[2], d_is[4]);
  msub_neon(&deltas[5], d_is[3], d_is[5]);
  msub_neon(&deltas[6], d_is[2], d_is[6]);
  msub_neon(&deltas[6], d_is[3], d_is[7]);
  msub_neon(&deltas[7], d_is[4], d_is[4]);
  msub_neon(&deltas[7], d_is[5], d_is[5]);
  msub_neon(&deltas[8], d_is[4], d_is[6]);
  msub_neon(&deltas[8], d_is[5], d_is[7]);
  msub_neon(&deltas[9], d_is[6], d_is[6]);
  msub_neon(&deltas[9], d_is[7], d_is[7]);

  madd_neon(&deltas[0], d_ie[0], d_ie[0]);
  madd_neon(&deltas[0], d_ie[1], d_ie[1]);
  madd_neon(&deltas[1], d_ie[0], d_ie[2]);
  madd_neon(&deltas[1], d_ie[1], d_ie[3]);
  madd_neon(&deltas[2], d_ie[0], d_ie[4]);
  madd_neon(&deltas[2], d_ie[1], d_ie[5]);
  madd_neon(&deltas[3], d_ie[0], d_ie[6]);
  madd_neon(&deltas[3], d_ie[1], d_ie[7]);
  madd_neon(&deltas[4], d_ie[2], d_ie[2]);
  madd_neon(&deltas[4], d_ie[3], d_ie[3]);
  madd_neon(&deltas[5], d_ie[2], d_ie[4]);
  madd_neon(&deltas[5], d_ie[3], d_ie[5]);
  madd_neon(&deltas[6], d_ie[2], d_ie[6]);
  madd_neon(&deltas[6], d_ie[3], d_ie[7]);
  madd_neon(&deltas[7], d_ie[4], d_ie[4]);
  madd_neon(&deltas[7], d_ie[5], d_ie[5]);
  madd_neon(&deltas[8], d_ie[4], d_ie[6]);
  madd_neon(&deltas[8], d_ie[5], d_ie[7]);
  madd_neon(&deltas[9], d_ie[6], d_ie[6]);
  madd_neon(&deltas[9], d_ie[7], d_ie[7]);
}

static inline void load_triangle_win5_neon(const int16_t *const di,
                                           const int32_t d_stride,
                                           const int32_t height,
                                           int16x8_t *d_is, int16x8_t *d_ie) {
  load_s16_8x4(di + 0, d_stride, &d_is[0], &d_is[2], &d_is[4], &d_is[6]);
  load_s16_8x4(di + 8, d_stride, &d_is[1], &d_is[3], &d_is[5], &d_is[7]);

  load_s16_8x4(di + height * d_stride + 0, d_stride, &d_ie[0], &d_ie[2],
               &d_ie[4], &d_ie[6]);
  load_s16_8x4(di + height * d_stride + 8, d_stride, &d_ie[1], &d_ie[3],
               &d_ie[5], &d_ie[7]);
}

static inline void sub_deltas_step4(int16x8_t *A, int16x8_t *B,
                                    int32x4_t *deltas) {
  deltas[0] = vmlsl_s16(deltas[0], vget_low_s16(A[0]), vget_low_s16(B[0]));
  deltas[0] = vmlsl_s16(deltas[0], vget_high_s16(A[0]), vget_high_s16(B[0]));
  deltas[1] = vmlsl_s16(deltas[1], vget_low_s16(A[0]), vget_low_s16(B[1]));
  deltas[1] = vmlsl_s16(deltas[1], vget_high_s16(A[0]), vget_high_s16(B[1]));
  deltas[2] = vmlsl_s16(deltas[2], vget_low_s16(A[0]), vget_low_s16(B[2]));
  deltas[2] = vmlsl_s16(deltas[2], vget_high_s16(A[0]), vget_high_s16(B[2]));
  deltas[3] = vmlsl_s16(deltas[3], vget_low_s16(A[0]), vget_low_s16(B[3]));
  deltas[3] = vmlsl_s16(deltas[3], vget_high_s16(A[0]), vget_high_s16(B[3]));
  deltas[4] = vmlsl_s16(deltas[4], vget_low_s16(A[0]), vget_low_s16(B[4]));
  deltas[4] = vmlsl_s16(deltas[4], vget_high_s16(A[0]), vget_high_s16(B[4]));
  deltas[5] = vmlsl_s16(deltas[5], vget_low_s16(A[1]), vget_low_s16(B[0]));
  deltas[5] = vmlsl_s16(deltas[5], vget_high_s16(A[1]), vget_high_s16(B[0]));
  deltas[6] = vmlsl_s16(deltas[6], vget_low_s16(A[2]), vget_low_s16(B[0]));
  deltas[6] = vmlsl_s16(deltas[6], vget_high_s16(A[2]), vget_high_s16(B[0]));
  deltas[7] = vmlsl_s16(deltas[7], vget_low_s16(A[3]), vget_low_s16(B[0]));
  deltas[7] = vmlsl_s16(deltas[7], vget_high_s16(A[3]), vget_high_s16(B[0]));
  deltas[8] = vmlsl_s16(deltas[8], vget_low_s16(A[4]), vget_low_s16(B[0]));
  deltas[8] = vmlsl_s16(deltas[8], vget_high_s16(A[4]), vget_high_s16(B[0]));
}

static inline void add_deltas_step4(int16x8_t *A, int16x8_t *B,
                                    int32x4_t *deltas) {
  deltas[0] = vmlal_s16(deltas[0], vget_low_s16(A[0]), vget_low_s16(B[0]));
  deltas[0] = vmlal_s16(deltas[0], vget_high_s16(A[0]), vget_high_s16(B[0]));
  deltas[1] = vmlal_s16(deltas[1], vget_low_s16(A[0]), vget_low_s16(B[1]));
  deltas[1] = vmlal_s16(deltas[1], vget_high_s16(A[0]), vget_high_s16(B[1]));
  deltas[2] = vmlal_s16(deltas[2], vget_low_s16(A[0]), vget_low_s16(B[2]));
  deltas[2] = vmlal_s16(deltas[2], vget_high_s16(A[0]), vget_high_s16(B[2]));
  deltas[3] = vmlal_s16(deltas[3], vget_low_s16(A[0]), vget_low_s16(B[3]));
  deltas[3] = vmlal_s16(deltas[3], vget_high_s16(A[0]), vget_high_s16(B[3]));
  deltas[4] = vmlal_s16(deltas[4], vget_low_s16(A[0]), vget_low_s16(B[4]));
  deltas[4] = vmlal_s16(deltas[4], vget_high_s16(A[0]), vget_high_s16(B[4]));
  deltas[5] = vmlal_s16(deltas[5], vget_low_s16(A[1]), vget_low_s16(B[0]));
  deltas[5] = vmlal_s16(deltas[5], vget_high_s16(A[1]), vget_high_s16(B[0]));
  deltas[6] = vmlal_s16(deltas[6], vget_low_s16(A[2]), vget_low_s16(B[0]));
  deltas[6] = vmlal_s16(deltas[6], vget_high_s16(A[2]), vget_high_s16(B[0]));
  deltas[7] = vmlal_s16(deltas[7], vget_low_s16(A[3]), vget_low_s16(B[0]));
  deltas[7] = vmlal_s16(deltas[7], vget_high_s16(A[3]), vget_high_s16(B[0]));
  deltas[8] = vmlal_s16(deltas[8], vget_low_s16(A[4]), vget_low_s16(B[0]));
  deltas[8] = vmlal_s16(deltas[8], vget_high_s16(A[4]), vget_high_s16(B[0]));
}

static inline void stats_top_win7_neon(const int16x8_t src[2],
                                       const int16x8_t dgd[2],
                                       const int16_t *const d,
                                       const int32_t d_stride, int32x4_t *sum_m,
                                       int32x4_t *sum_h) {
  int16x8_t dgds[WIENER_WIN * 2];

  load_s16_8x7(d + 0, d_stride, &dgds[0], &dgds[2], &dgds[4], &dgds[6],
               &dgds[8], &dgds[10], &dgds[12]);
  load_s16_8x7(d + 8, d_stride, &dgds[1], &dgds[3], &dgds[5], &dgds[7],
               &dgds[9], &dgds[11], &dgds[13]);

  madd_neon(&sum_m[0], src[0], dgds[0]);
  madd_neon(&sum_m[0], src[1], dgds[1]);
  madd_neon(&sum_m[1], src[0], dgds[2]);
  madd_neon(&sum_m[1], src[1], dgds[3]);
  madd_neon(&sum_m[2], src[0], dgds[4]);
  madd_neon(&sum_m[2], src[1], dgds[5]);
  madd_neon(&sum_m[3], src[0], dgds[6]);
  madd_neon(&sum_m[3], src[1], dgds[7]);
  madd_neon(&sum_m[4], src[0], dgds[8]);
  madd_neon(&sum_m[4], src[1], dgds[9]);
  madd_neon(&sum_m[5], src[0], dgds[10]);
  madd_neon(&sum_m[5], src[1], dgds[11]);
  madd_neon(&sum_m[6], src[0], dgds[12]);
  madd_neon(&sum_m[6], src[1], dgds[13]);

  madd_neon(&sum_h[0], dgd[0], dgds[0]);
  madd_neon(&sum_h[0], dgd[1], dgds[1]);
  madd_neon(&sum_h[1], dgd[0], dgds[2]);
  madd_neon(&sum_h[1], dgd[1], dgds[3]);
  madd_neon(&sum_h[2], dgd[0], dgds[4]);
  madd_neon(&sum_h[2], dgd[1], dgds[5]);
  madd_neon(&sum_h[3], dgd[0], dgds[6]);
  madd_neon(&sum_h[3], dgd[1], dgds[7]);
  madd_neon(&sum_h[4], dgd[0], dgds[8]);
  madd_neon(&sum_h[4], dgd[1], dgds[9]);
  madd_neon(&sum_h[5], dgd[0], dgds[10]);
  madd_neon(&sum_h[5], dgd[1], dgds[11]);
  madd_neon(&sum_h[6], dgd[0], dgds[12]);
  madd_neon(&sum_h[6], dgd[1], dgds[13]);
}

static inline void derive_square_win7_neon(const int16x8_t *d_is,
                                           const int16x8_t *d_ie,
                                           const int16x8_t *d_js,
                                           const int16x8_t *d_je,
                                           int32x4_t deltas[][WIN_7]) {
  msub_neon(&deltas[0][0], d_is[0], d_js[0]);
  msub_neon(&deltas[0][0], d_is[1], d_js[1]);
  msub_neon(&deltas[0][1], d_is[0], d_js[2]);
  msub_neon(&deltas[0][1], d_is[1], d_js[3]);
  msub_neon(&deltas[0][2], d_is[0], d_js[4]);
  msub_neon(&deltas[0][2], d_is[1], d_js[5]);
  msub_neon(&deltas[0][3], d_is[0], d_js[6]);
  msub_neon(&deltas[0][3], d_is[1], d_js[7]);
  msub_neon(&deltas[0][4], d_is[0], d_js[8]);
  msub_neon(&deltas[0][4], d_is[1], d_js[9]);
  msub_neon(&deltas[0][5], d_is[0], d_js[10]);
  msub_neon(&deltas[0][5], d_is[1], d_js[11]);

  msub_neon(&deltas[1][0], d_is[2], d_js[0]);
  msub_neon(&deltas[1][0], d_is[3], d_js[1]);
  msub_neon(&deltas[1][1], d_is[2], d_js[2]);
  msub_neon(&deltas[1][1], d_is[3], d_js[3]);
  msub_neon(&deltas[1][2], d_is[2], d_js[4]);
  msub_neon(&deltas[1][2], d_is[3], d_js[5]);
  msub_neon(&deltas[1][3], d_is[2], d_js[6]);
  msub_neon(&deltas[1][3], d_is[3], d_js[7]);
  msub_neon(&deltas[1][4], d_is[2], d_js[8]);
  msub_neon(&deltas[1][4], d_is[3], d_js[9]);
  msub_neon(&deltas[1][5], d_is[2], d_js[10]);
  msub_neon(&deltas[1][5], d_is[3], d_js[11]);

  msub_neon(&deltas[2][0], d_is[4], d_js[0]);
  msub_neon(&deltas[2][0], d_is[5], d_js[1]);
  msub_neon(&deltas[2][1], d_is[4], d_js[2]);
  msub_neon(&deltas[2][1], d_is[5], d_js[3]);
  msub_neon(&deltas[2][2], d_is[4], d_js[4]);
  msub_neon(&deltas[2][2], d_is[5], d_js[5]);
  msub_neon(&deltas[2][3], d_is[4], d_js[6]);
  msub_neon(&deltas[2][3], d_is[5], d_js[7]);
  msub_neon(&deltas[2][4], d_is[4], d_js[8]);
  msub_neon(&deltas[2][4], d_is[5], d_js[9]);
  msub_neon(&deltas[2][5], d_is[4], d_js[10]);
  msub_neon(&deltas[2][5], d_is[5], d_js[11]);

  msub_neon(&deltas[3][0], d_is[6], d_js[0]);
  msub_neon(&deltas[3][0], d_is[7], d_js[1]);
  msub_neon(&deltas[3][1], d_is[6], d_js[2]);
  msub_neon(&deltas[3][1], d_is[7], d_js[3]);
  msub_neon(&deltas[3][2], d_is[6], d_js[4]);
  msub_neon(&deltas[3][2], d_is[7], d_js[5]);
  msub_neon(&deltas[3][3], d_is[6], d_js[6]);
  msub_neon(&deltas[3][3], d_is[7], d_js[7]);
  msub_neon(&deltas[3][4], d_is[6], d_js[8]);
  msub_neon(&deltas[3][4], d_is[7], d_js[9]);
  msub_neon(&deltas[3][5], d_is[6], d_js[10]);
  msub_neon(&deltas[3][5], d_is[7], d_js[11]);

  msub_neon(&deltas[4][0], d_is[8], d_js[0]);
  msub_neon(&deltas[4][0], d_is[9], d_js[1]);
  msub_neon(&deltas[4][1], d_is[8], d_js[2]);
  msub_neon(&deltas[4][1], d_is[9], d_js[3]);
  msub_neon(&deltas[4][2], d_is[8], d_js[4]);
  msub_neon(&deltas[4][2], d_is[9], d_js[5]);
  msub_neon(&deltas[4][3], d_is[8], d_js[6]);
  msub_neon(&deltas[4][3], d_is[9], d_js[7]);
  msub_neon(&deltas[4][4], d_is[8], d_js[8]);
  msub_neon(&deltas[4][4], d_is[9], d_js[9]);
  msub_neon(&deltas[4][5], d_is[8], d_js[10]);
  msub_neon(&deltas[4][5], d_is[9], d_js[11]);

  msub_neon(&deltas[5][0], d_is[10], d_js[0]);
  msub_neon(&deltas[5][0], d_is[11], d_js[1]);
  msub_neon(&deltas[5][1], d_is[10], d_js[2]);
  msub_neon(&deltas[5][1], d_is[11], d_js[3]);
  msub_neon(&deltas[5][2], d_is[10], d_js[4]);
  msub_neon(&deltas[5][2], d_is[11], d_js[5]);
  msub_neon(&deltas[5][3], d_is[10], d_js[6]);
  msub_neon(&deltas[5][3], d_is[11], d_js[7]);
  msub_neon(&deltas[5][4], d_is[10], d_js[8]);
  msub_neon(&deltas[5][4], d_is[11], d_js[9]);
  msub_neon(&deltas[5][5], d_is[10], d_js[10]);
  msub_neon(&deltas[5][5], d_is[11], d_js[11]);

  madd_neon(&deltas[0][0], d_ie[0], d_je[0]);
  madd_neon(&deltas[0][0], d_ie[1], d_je[1]);
  madd_neon(&deltas[0][1], d_ie[0], d_je[2]);
  madd_neon(&deltas[0][1], d_ie[1], d_je[3]);
  madd_neon(&deltas[0][2], d_ie[0], d_je[4]);
  madd_neon(&deltas[0][2], d_ie[1], d_je[5]);
  madd_neon(&deltas[0][3], d_ie[0], d_je[6]);
  madd_neon(&deltas[0][3], d_ie[1], d_je[7]);
  madd_neon(&deltas[0][4], d_ie[0], d_je[8]);
  madd_neon(&deltas[0][4], d_ie[1], d_je[9]);
  madd_neon(&deltas[0][5], d_ie[0], d_je[10]);
  madd_neon(&deltas[0][5], d_ie[1], d_je[11]);

  madd_neon(&deltas[1][0], d_ie[2], d_je[0]);
  madd_neon(&deltas[1][0], d_ie[3], d_je[1]);
  madd_neon(&deltas[1][1], d_ie[2], d_je[2]);
  madd_neon(&deltas[1][1], d_ie[3], d_je[3]);
  madd_neon(&deltas[1][2], d_ie[2], d_je[4]);
  madd_neon(&deltas[1][2], d_ie[3], d_je[5]);
  madd_neon(&deltas[1][3], d_ie[2], d_je[6]);
  madd_neon(&deltas[1][3], d_ie[3], d_je[7]);
  madd_neon(&deltas[1][4], d_ie[2], d_je[8]);
  madd_neon(&deltas[1][4], d_ie[3], d_je[9]);
  madd_neon(&deltas[1][5], d_ie[2], d_je[10]);
  madd_neon(&deltas[1][5], d_ie[3], d_je[11]);

  madd_neon(&deltas[2][0], d_ie[4], d_je[0]);
  madd_neon(&deltas[2][0], d_ie[5], d_je[1]);
  madd_neon(&deltas[2][1], d_ie[4], d_je[2]);
  madd_neon(&deltas[2][1], d_ie[5], d_je[3]);
  madd_neon(&deltas[2][2], d_ie[4], d_je[4]);
  madd_neon(&deltas[2][2], d_ie[5], d_je[5]);
  madd_neon(&deltas[2][3], d_ie[4], d_je[6]);
  madd_neon(&deltas[2][3], d_ie[5], d_je[7]);
  madd_neon(&deltas[2][4], d_ie[4], d_je[8]);
  madd_neon(&deltas[2][4], d_ie[5], d_je[9]);
  madd_neon(&deltas[2][5], d_ie[4], d_je[10]);
  madd_neon(&deltas[2][5], d_ie[5], d_je[11]);

  madd_neon(&deltas[3][0], d_ie[6], d_je[0]);
  madd_neon(&deltas[3][0], d_ie[7], d_je[1]);
  madd_neon(&deltas[3][1], d_ie[6], d_je[2]);
  madd_neon(&deltas[3][1], d_ie[7], d_je[3]);
  madd_neon(&deltas[3][2], d_ie[6], d_je[4]);
  madd_neon(&deltas[3][2], d_ie[7], d_je[5]);
  madd_neon(&deltas[3][3], d_ie[6], d_je[6]);
  madd_neon(&deltas[3][3], d_ie[7], d_je[7]);
  madd_neon(&deltas[3][4], d_ie[6], d_je[8]);
  madd_neon(&deltas[3][4], d_ie[7], d_je[9]);
  madd_neon(&deltas[3][5], d_ie[6], d_je[10]);
  madd_neon(&deltas[3][5], d_ie[7], d_je[11]);

  madd_neon(&deltas[4][0], d_ie[8], d_je[0]);
  madd_neon(&deltas[4][0], d_ie[9], d_je[1]);
  madd_neon(&deltas[4][1], d_ie[8], d_je[2]);
  madd_neon(&deltas[4][1], d_ie[9], d_je[3]);
  madd_neon(&deltas[4][2], d_ie[8], d_je[4]);
  madd_neon(&deltas[4][2], d_ie[9], d_je[5]);
  madd_neon(&deltas[4][3], d_ie[8], d_je[6]);
  madd_neon(&deltas[4][3], d_ie[9], d_je[7]);
  madd_neon(&deltas[4][4], d_ie[8], d_je[8]);
  madd_neon(&deltas[4][4], d_ie[9], d_je[9]);
  madd_neon(&deltas[4][5], d_ie[8], d_je[10]);
  madd_neon(&deltas[4][5], d_ie[9], d_je[11]);

  madd_neon(&deltas[5][0], d_ie[10], d_je[0]);
  madd_neon(&deltas[5][0], d_ie[11], d_je[1]);
  madd_neon(&deltas[5][1], d_ie[10], d_je[2]);
  madd_neon(&deltas[5][1], d_ie[11], d_je[3]);
  madd_neon(&deltas[5][2], d_ie[10], d_je[4]);
  madd_neon(&deltas[5][2], d_ie[11], d_je[5]);
  madd_neon(&deltas[5][3], d_ie[10], d_je[6]);
  madd_neon(&deltas[5][3], d_ie[11], d_je[7]);
  madd_neon(&deltas[5][4], d_ie[10], d_je[8]);
  madd_neon(&deltas[5][4], d_ie[11], d_je[9]);
  madd_neon(&deltas[5][5], d_ie[10], d_je[10]);
  madd_neon(&deltas[5][5], d_ie[11], d_je[11]);
}

static inline void update_8_stats_neon(const int64_t *const src,
                                       const int32x4_t delta0,
                                       const int32x4_t delta1,
                                       int64_t *const dst) {
  update_4_stats_neon(src + 0, delta0, dst + 0);
  update_4_stats_neon(src + 4, delta1, dst + 4);
}

static inline void load_square_win7_neon(const int16_t *const di,
                                         const int16_t *const dj,
                                         const int32_t d_stride,
                                         const int32_t height, int16x8_t *d_is,
                                         int16x8_t *d_ie, int16x8_t *d_js,
                                         int16x8_t *d_je) {
  load_s16_8x6(di + 0, d_stride, &d_is[0], &d_is[2], &d_is[4], &d_is[6],
               &d_is[8], &d_is[10]);
  load_s16_8x6(di + 8, d_stride, &d_is[1], &d_is[3], &d_is[5], &d_is[7],
               &d_is[9], &d_is[11]);
  load_s16_8x6(dj + 0, d_stride, &d_js[0], &d_js[2], &d_js[4], &d_js[6],
               &d_js[8], &d_js[10]);
  load_s16_8x6(dj + 8, d_stride, &d_js[1], &d_js[3], &d_js[5], &d_js[7],
               &d_js[9], &d_js[11]);

  load_s16_8x6(di + height * d_stride + 0, d_stride, &d_ie[0], &d_ie[2],
               &d_ie[4], &d_ie[6], &d_ie[8], &d_ie[10]);
  load_s16_8x6(di + height * d_stride + 8, d_stride, &d_ie[1], &d_ie[3],
               &d_ie[5], &d_ie[7], &d_ie[9], &d_ie[11]);
  load_s16_8x6(dj + height * d_stride + 0, d_stride, &d_je[0], &d_je[2],
               &d_je[4], &d_je[6], &d_je[8], &d_je[10]);
  load_s16_8x6(dj + height * d_stride + 8, d_stride, &d_je[1], &d_je[3],
               &d_je[5], &d_je[7], &d_je[9], &d_je[11]);
}

static inline void load_triangle_win7_neon(const int16_t *const di,
                                           const int32_t d_stride,
                                           const int32_t height,
                                           int16x8_t *d_is, int16x8_t *d_ie) {
  load_s16_8x6(di, d_stride, &d_is[0], &d_is[2], &d_is[4], &d_is[6], &d_is[8],
               &d_is[10]);
  load_s16_8x6(di + 8, d_stride, &d_is[1], &d_is[3], &d_is[5], &d_is[7],
               &d_is[9], &d_is[11]);

  load_s16_8x6(di + height * d_stride, d_stride, &d_ie[0], &d_ie[2], &d_ie[4],
               &d_ie[6], &d_ie[8], &d_ie[10]);
  load_s16_8x6(di + height * d_stride + 8, d_stride, &d_ie[1], &d_ie[3],
               &d_ie[5], &d_ie[7], &d_ie[9], &d_ie[11]);
}

static inline void stats_left_win7_neon(const int16x8_t src[2],
                                        const int16_t *d,
                                        const int32_t d_stride,
                                        int32x4_t *sum) {
  int16x8_t dgds[WIN_7];

  load_s16_8x6(d + d_stride + 0, d_stride, &dgds[0], &dgds[2], &dgds[4],
               &dgds[6], &dgds[8], &dgds[10]);
  load_s16_8x6(d + d_stride + 8, d_stride, &dgds[1], &dgds[3], &dgds[5],
               &dgds[7], &dgds[9], &dgds[11]);

  madd_neon(&sum[0], src[0], dgds[0]);
  madd_neon(&sum[0], src[1], dgds[1]);
  madd_neon(&sum[1], src[0], dgds[2]);
  madd_neon(&sum[1], src[1], dgds[3]);
  madd_neon(&sum[2], src[0], dgds[4]);
  madd_neon(&sum[2], src[1], dgds[5]);
  madd_neon(&sum[3], src[0], dgds[6]);
  madd_neon(&sum[3], src[1], dgds[7]);
  madd_neon(&sum[4], src[0], dgds[8]);
  madd_neon(&sum[4], src[1], dgds[9]);
  madd_neon(&sum[5], src[0], dgds[10]);
  madd_neon(&sum[5], src[1], dgds[11]);
}

static inline void step3_win7_neon(const int16_t *d, const int32_t d_stride,
                                   const int32_t width, const int32_t height,
                                   int16x8_t *ds, int32x4_t *deltas) {
  int32_t y = height;
  do {
    ds[12] = vld1q_s16(d);
    ds[13] = vld1q_s16(d + width);

    compute_delta_step3(&deltas[0], &deltas[4], ds[0], ds[1], ds[0], ds[1]);
    compute_delta_step3(&deltas[1], &deltas[5], ds[0], ds[1], ds[2], ds[3]);
    compute_delta_step3(&deltas[2], &deltas[6], ds[0], ds[1], ds[4], ds[5]);
    compute_delta_step3(&deltas[3], &deltas[7], ds[0], ds[1], ds[6], ds[7]);
    compute_delta_step3(&deltas[8], &deltas[12], ds[0], ds[1], ds[8], ds[9]);
    compute_delta_step3(&deltas[9], &deltas[13], ds[0], ds[1], ds[10], ds[11]);
    compute_delta_step3(&deltas[10], &deltas[14], ds[0], ds[1], ds[12], ds[13]);

    ds[0] = ds[2];
    ds[1] = ds[3];
    ds[2] = ds[4];
    ds[3] = ds[5];
    ds[4] = ds[6];
    ds[5] = ds[7];
    ds[6] = ds[8];
    ds[7] = ds[9];
    ds[8] = ds[10];
    ds[9] = ds[11];
    ds[10] = ds[12];
    ds[11] = ds[13];

    d += d_stride;
  } while (--y);
}

static inline void derive_triangle_win7_neon(const int16x8_t *d_is,
                                             const int16x8_t *d_ie,
                                             int32x4_t *deltas) {
  msub_neon(&deltas[0], d_is[0], d_is[0]);
  msub_neon(&deltas[0], d_is[1], d_is[1]);
  msub_neon(&deltas[1], d_is[0], d_is[2]);
  msub_neon(&deltas[1], d_is[1], d_is[3]);
  msub_neon(&deltas[2], d_is[0], d_is[4]);
  msub_neon(&deltas[2], d_is[1], d_is[5]);
  msub_neon(&deltas[3], d_is[0], d_is[6]);
  msub_neon(&deltas[3], d_is[1], d_is[7]);
  msub_neon(&deltas[4], d_is[0], d_is[8]);
  msub_neon(&deltas[4], d_is[1], d_is[9]);
  msub_neon(&deltas[5], d_is[0], d_is[10]);
  msub_neon(&deltas[5], d_is[1], d_is[11]);

  msub_neon(&deltas[6], d_is[2], d_is[2]);
  msub_neon(&deltas[6], d_is[3], d_is[3]);
  msub_neon(&deltas[7], d_is[2], d_is[4]);
  msub_neon(&deltas[7], d_is[3], d_is[5]);
  msub_neon(&deltas[8], d_is[2], d_is[6]);
  msub_neon(&deltas[8], d_is[3], d_is[7]);
  msub_neon(&deltas[9], d_is[2], d_is[8]);
  msub_neon(&deltas[9], d_is[3], d_is[9]);
  msub_neon(&deltas[10], d_is[2], d_is[10]);
  msub_neon(&deltas[10], d_is[3], d_is[11]);

  msub_neon(&deltas[11], d_is[4], d_is[4]);
  msub_neon(&deltas[11], d_is[5], d_is[5]);
  msub_neon(&deltas[12], d_is[4], d_is[6]);
  msub_neon(&deltas[12], d_is[5], d_is[7]);
  msub_neon(&deltas[13], d_is[4], d_is[8]);
  msub_neon(&deltas[13], d_is[5], d_is[9]);
  msub_neon(&deltas[14], d_is[4], d_is[10]);
  msub_neon(&deltas[14], d_is[5], d_is[11]);

  msub_neon(&deltas[15], d_is[6], d_is[6]);
  msub_neon(&deltas[15], d_is[7], d_is[7]);
  msub_neon(&deltas[16], d_is[6], d_is[8]);
  msub_neon(&deltas[16], d_is[7], d_is[9]);
  msub_neon(&deltas[17], d_is[6], d_is[10]);
  msub_neon(&deltas[17], d_is[7], d_is[11]);

  msub_neon(&deltas[18], d_is[8], d_is[8]);
  msub_neon(&deltas[18], d_is[9], d_is[9]);
  msub_neon(&deltas[19], d_is[8], d_is[10]);
  msub_neon(&deltas[19], d_is[9], d_is[11]);

  msub_neon(&deltas[20], d_is[10], d_is[10]);
  msub_neon(&deltas[20], d_is[11], d_is[11]);

  madd_neon(&deltas[0], d_ie[0], d_ie[0]);
  madd_neon(&deltas[0], d_ie[1], d_ie[1]);
  madd_neon(&deltas[1], d_ie[0], d_ie[2]);
  madd_neon(&deltas[1], d_ie[1], d_ie[3]);
  madd_neon(&deltas[2], d_ie[0], d_ie[4]);
  madd_neon(&deltas[2], d_ie[1], d_ie[5]);
  madd_neon(&deltas[3], d_ie[0], d_ie[6]);
  madd_neon(&deltas[3], d_ie[1], d_ie[7]);
  madd_neon(&deltas[4], d_ie[0], d_ie[8]);
  madd_neon(&deltas[4], d_ie[1], d_ie[9]);
  madd_neon(&deltas[5], d_ie[0], d_ie[10]);
  madd_neon(&deltas[5], d_ie[1], d_ie[11]);

  madd_neon(&deltas[6], d_ie[2], d_ie[2]);
  madd_neon(&deltas[6], d_ie[3], d_ie[3]);
  madd_neon(&deltas[7], d_ie[2], d_ie[4]);
  madd_neon(&deltas[7], d_ie[3], d_ie[5]);
  madd_neon(&deltas[8], d_ie[2], d_ie[6]);
  madd_neon(&deltas[8], d_ie[3], d_ie[7]);
  madd_neon(&deltas[9], d_ie[2], d_ie[8]);
  madd_neon(&deltas[9], d_ie[3], d_ie[9]);
  madd_neon(&deltas[10], d_ie[2], d_ie[10]);
  madd_neon(&deltas[10], d_ie[3], d_ie[11]);

  madd_neon(&deltas[11], d_ie[4], d_ie[4]);
  madd_neon(&deltas[11], d_ie[5], d_ie[5]);
  madd_neon(&deltas[12], d_ie[4], d_ie[6]);
  madd_neon(&deltas[12], d_ie[5], d_ie[7]);
  madd_neon(&deltas[13], d_ie[4], d_ie[8]);
  madd_neon(&deltas[13], d_ie[5], d_ie[9]);
  madd_neon(&deltas[14], d_ie[4], d_ie[10]);
  madd_neon(&deltas[14], d_ie[5], d_ie[11]);

  madd_neon(&deltas[15], d_ie[6], d_ie[6]);
  madd_neon(&deltas[15], d_ie[7], d_ie[7]);
  madd_neon(&deltas[16], d_ie[6], d_ie[8]);
  madd_neon(&deltas[16], d_ie[7], d_ie[9]);
  madd_neon(&deltas[17], d_ie[6], d_ie[10]);
  madd_neon(&deltas[17], d_ie[7], d_ie[11]);

  madd_neon(&deltas[18], d_ie[8], d_ie[8]);
  madd_neon(&deltas[18], d_ie[9], d_ie[9]);
  madd_neon(&deltas[19], d_ie[8], d_ie[10]);
  madd_neon(&deltas[19], d_ie[9], d_ie[11]);

  madd_neon(&deltas[20], d_ie[10], d_ie[10]);
  madd_neon(&deltas[20], d_ie[11], d_ie[11]);
}

static inline void diagonal_copy_stats_neon(const int32_t wiener_win2,
                                            int64_t *const H) {
  for (int32_t i = 0; i < wiener_win2 - 1; i += 4) {
    int64x2_t in[8], out[8];

    in[0] = vld1q_s64(H + (i + 0) * wiener_win2 + i + 1);
    in[1] = vld1q_s64(H + (i + 0) * wiener_win2 + i + 3);
    in[2] = vld1q_s64(H + (i + 1) * wiener_win2 + i + 1);
    in[3] = vld1q_s64(H + (i + 1) * wiener_win2 + i + 3);
    in[4] = vld1q_s64(H + (i + 2) * wiener_win2 + i + 1);
    in[5] = vld1q_s64(H + (i + 2) * wiener_win2 + i + 3);
    in[6] = vld1q_s64(H + (i + 3) * wiener_win2 + i + 1);
    in[7] = vld1q_s64(H + (i + 3) * wiener_win2 + i + 3);

    transpose_arrays_s64_4x4(in, out);

    vst1_s64(H + (i + 1) * wiener_win2 + i, vget_low_s64(out[0]));
    vst1q_s64(H + (i + 2) * wiener_win2 + i, out[2]);
    vst1q_s64(H + (i + 3) * wiener_win2 + i, out[4]);
    vst1q_s64(H + (i + 3) * wiener_win2 + i + 2, out[5]);
    vst1q_s64(H + (i + 4) * wiener_win2 + i, out[6]);
    vst1q_s64(H + (i + 4) * wiener_win2 + i + 2, out[7]);

    for (int32_t j = i + 5; j < wiener_win2; j += 4) {
      in[0] = vld1q_s64(H + (i + 0) * wiener_win2 + j);
      in[1] = vld1q_s64(H + (i + 0) * wiener_win2 + j + 2);
      in[2] = vld1q_s64(H + (i + 1) * wiener_win2 + j);
      in[3] = vld1q_s64(H + (i + 1) * wiener_win2 + j + 2);
      in[4] = vld1q_s64(H + (i + 2) * wiener_win2 + j);
      in[5] = vld1q_s64(H + (i + 2) * wiener_win2 + j + 2);
      in[6] = vld1q_s64(H + (i + 3) * wiener_win2 + j);
      in[7] = vld1q_s64(H + (i + 3) * wiener_win2 + j + 2);

      transpose_arrays_s64_4x4(in, out);

      vst1q_s64(H + (j + 0) * wiener_win2 + i, out[0]);
      vst1q_s64(H + (j + 0) * wiener_win2 + i + 2, out[1]);
      vst1q_s64(H + (j + 1) * wiener_win2 + i, out[2]);
      vst1q_s64(H + (j + 1) * wiener_win2 + i + 2, out[3]);
      vst1q_s64(H + (j + 2) * wiener_win2 + i, out[4]);
      vst1q_s64(H + (j + 2) * wiener_win2 + i + 2, out[5]);
      vst1q_s64(H + (j + 3) * wiener_win2 + i, out[6]);
      vst1q_s64(H + (j + 3) * wiener_win2 + i + 2, out[7]);
    }
  }
}

static inline int64x2_t div4_neon(const int64x2_t src) {
#if AOM_ARCH_AARCH64
  uint64x2_t sign = vcltzq_s64(src);
  int64x2_t abs = vabsq_s64(src);
  // divide by 4
  abs = vshrq_n_s64(abs, 2);
  // re-apply sign
  return vbslq_s64(sign, vnegq_s64(abs), abs);
#else
  int64x2_t sign = vshrq_n_s64(src, 63);
  int64x2_t abs = vsubq_s64(veorq_s64(src, sign), sign);
  // divide by 4
  abs = vshrq_n_s64(abs, 2);
  // re-apply sign
  return vsubq_s64(veorq_s64(abs, sign), sign);
#endif  // AOM_ARCH_AARCH64
}

static inline void div4_4x4_neon(const int32_t wiener_win2, int64_t *const H,
                                 int64x2_t out[8]) {
  out[0] = vld1q_s64(H + 0 * wiener_win2 + 0);
  out[1] = vld1q_s64(H + 0 * wiener_win2 + 2);
  out[2] = vld1q_s64(H + 1 * wiener_win2 + 0);
  out[3] = vld1q_s64(H + 1 * wiener_win2 + 2);
  out[4] = vld1q_s64(H + 2 * wiener_win2 + 0);
  out[5] = vld1q_s64(H + 2 * wiener_win2 + 2);
  out[6] = vld1q_s64(H + 3 * wiener_win2 + 0);
  out[7] = vld1q_s64(H + 3 * wiener_win2 + 2);

  out[0] = div4_neon(out[0]);
  out[1] = div4_neon(out[1]);
  out[2] = div4_neon(out[2]);
  out[3] = div4_neon(out[3]);
  out[4] = div4_neon(out[4]);
  out[5] = div4_neon(out[5]);
  out[6] = div4_neon(out[6]);
  out[7] = div4_neon(out[7]);

  vst1q_s64(H + 0 * wiener_win2 + 0, out[0]);
  vst1q_s64(H + 0 * wiener_win2 + 2, out[1]);
  vst1q_s64(H + 1 * wiener_win2 + 0, out[2]);
  vst1q_s64(H + 1 * wiener_win2 + 2, out[3]);
  vst1q_s64(H + 2 * wiener_win2 + 0, out[4]);
  vst1q_s64(H + 2 * wiener_win2 + 2, out[5]);
  vst1q_s64(H + 3 * wiener_win2 + 0, out[6]);
  vst1q_s64(H + 3 * wiener_win2 + 2, out[7]);
}

static inline int64x2_t div16_neon(const int64x2_t src) {
#if AOM_ARCH_AARCH64
  uint64x2_t sign = vcltzq_s64(src);
  int64x2_t abs = vabsq_s64(src);
  // divide by 16
  abs = vshrq_n_s64(abs, 4);
  // re-apply sign
  return vbslq_s64(sign, vnegq_s64(abs), abs);
#else
  int64x2_t sign = vshrq_n_s64(src, 63);
  int64x2_t abs = vsubq_s64(veorq_s64(src, sign), sign);
  // divide by 16
  abs = vshrq_n_s64(abs, 4);
  // re-apply sign
  return vsubq_s64(veorq_s64(abs, sign), sign);
#endif  // AOM_ARCH_AARCH64
}

static inline void div16_4x4_neon(const int32_t wiener_win2, int64_t *const H,
                                  int64x2_t out[8]) {
  out[0] = vld1q_s64(H + 0 * wiener_win2 + 0);
  out[1] = vld1q_s64(H + 0 * wiener_win2 + 2);
  out[2] = vld1q_s64(H + 1 * wiener_win2 + 0);
  out[3] = vld1q_s64(H + 1 * wiener_win2 + 2);
  out[4] = vld1q_s64(H + 2 * wiener_win2 + 0);
  out[5] = vld1q_s64(H + 2 * wiener_win2 + 2);
  out[6] = vld1q_s64(H + 3 * wiener_win2 + 0);
  out[7] = vld1q_s64(H + 3 * wiener_win2 + 2);

  out[0] = div16_neon(out[0]);
  out[1] = div16_neon(out[1]);
  out[2] = div16_neon(out[2]);
  out[3] = div16_neon(out[3]);
  out[4] = div16_neon(out[4]);
  out[5] = div16_neon(out[5]);
  out[6] = div16_neon(out[6]);
  out[7] = div16_neon(out[7]);

  vst1q_s64(H + 0 * wiener_win2 + 0, out[0]);
  vst1q_s64(H + 0 * wiener_win2 + 2, out[1]);
  vst1q_s64(H + 1 * wiener_win2 + 0, out[2]);
  vst1q_s64(H + 1 * wiener_win2 + 2, out[3]);
  vst1q_s64(H + 2 * wiener_win2 + 0, out[4]);
  vst1q_s64(H + 2 * wiener_win2 + 2, out[5]);
  vst1q_s64(H + 3 * wiener_win2 + 0, out[6]);
  vst1q_s64(H + 3 * wiener_win2 + 2, out[7]);
}

static inline void div4_diagonal_copy_stats_neon(const int32_t wiener_win2,
                                                 int64_t *const H) {
  for (int32_t i = 0; i < wiener_win2 - 1; i += 4) {
    int64x2_t in[8], out[8];

    div4_4x4_neon(wiener_win2, H + i * wiener_win2 + i + 1, in);
    transpose_arrays_s64_4x4(in, out);

    vst1_s64(H + (i + 1) * wiener_win2 + i + 0, vget_low_s64(out[0]));
    vst1q_s64(H + (i + 2) * wiener_win2 + i + 0, out[2]);
    vst1q_s64(H + (i + 3) * wiener_win2 + i + 0, out[4]);
    vst1q_s64(H + (i + 3) * wiener_win2 + i + 2, out[5]);
    vst1q_s64(H + (i + 4) * wiener_win2 + i + 0, out[6]);
    vst1q_s64(H + (i + 4) * wiener_win2 + i + 2, out[7]);

    for (int32_t j = i + 5; j < wiener_win2; j += 4) {
      div4_4x4_neon(wiener_win2, H + i * wiener_win2 + j, in);
      transpose_arrays_s64_4x4(in, out);

      vst1q_s64(H + (j + 0) * wiener_win2 + i + 0, out[0]);
      vst1q_s64(H + (j + 0) * wiener_win2 + i + 2, out[1]);
      vst1q_s64(H + (j + 1) * wiener_win2 + i + 0, out[2]);
      vst1q_s64(H + (j + 1) * wiener_win2 + i + 2, out[3]);
      vst1q_s64(H + (j + 2) * wiener_win2 + i + 0, out[4]);
      vst1q_s64(H + (j + 2) * wiener_win2 + i + 2, out[5]);
      vst1q_s64(H + (j + 3) * wiener_win2 + i + 0, out[6]);
      vst1q_s64(H + (j + 3) * wiener_win2 + i + 2, out[7]);
    }
  }
}

static inline void div16_diagonal_copy_stats_neon(const int32_t wiener_win2,
                                                  int64_t *const H) {
  for (int32_t i = 0; i < wiener_win2 - 1; i += 4) {
    int64x2_t in[8], out[8];

    div16_4x4_neon(wiener_win2, H + i * wiener_win2 + i + 1, in);
    transpose_arrays_s64_4x4(in, out);

    vst1_s64(H + (i + 1) * wiener_win2 + i + 0, vget_low_s64(out[0]));
    vst1q_s64(H + (i + 2) * wiener_win2 + i + 0, out[2]);
    vst1q_s64(H + (i + 3) * wiener_win2 + i + 0, out[4]);
    vst1q_s64(H + (i + 3) * wiener_win2 + i + 2, out[5]);
    vst1q_s64(H + (i + 4) * wiener_win2 + i + 0, out[6]);
    vst1q_s64(H + (i + 4) * wiener_win2 + i + 2, out[7]);

    for (int32_t j = i + 5; j < wiener_win2; j += 4) {
      div16_4x4_neon(wiener_win2, H + i * wiener_win2 + j, in);
      transpose_arrays_s64_4x4(in, out);

      vst1q_s64(H + (j + 0) * wiener_win2 + i + 0, out[0]);
      vst1q_s64(H + (j + 0) * wiener_win2 + i + 2, out[1]);
      vst1q_s64(H + (j + 1) * wiener_win2 + i + 0, out[2]);
      vst1q_s64(H + (j + 1) * wiener_win2 + i + 2, out[3]);
      vst1q_s64(H + (j + 2) * wiener_win2 + i + 0, out[4]);
      vst1q_s64(H + (j + 2) * wiener_win2 + i + 2, out[5]);
      vst1q_s64(H + (j + 3) * wiener_win2 + i + 0, out[6]);
      vst1q_s64(H + (j + 3) * wiener_win2 + i + 2, out[7]);
    }
  }
}

#endif  // AOM_AV1_ENCODER_ARM_PICKRST_NEON_H_