xref: /aosp_15_r20/external/gemmlowp/internal/simd_wrappers.h (revision 5f39d1b313f0528e11bae88b3029b54b9e1033e7)

// Copyright 2017 The Gemmlowp Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// simd_wrappers.h: some inline functions wrapping SIMD intrinsics,
// extending the set of such functions from fixedpoint.h.

#ifndef GEMMLOWP_INTERNAL_SIMD_WRAPPERS_H_
#define GEMMLOWP_INTERNAL_SIMD_WRAPPERS_H_

#include <algorithm>
#include <type_traits>
#include "../fixedpoint/fixedpoint.h"

namespace gemmlowp {

template <typename ScalarType, int ScalarCount>
struct RegisterType {
  using Type = ScalarType;
};

inline std::int32_t Min(std::int32_t a, std::int32_t b) {
  return std::min(a, b);
}

inline std::int32_t Max(std::int32_t a, std::int32_t b) {
  return std::max(a, b);
}

inline void MulAdd(std::int32_t lhs, std::int32_t rhs, std::int32_t* acc) {
  *acc += lhs * rhs;
}

template <typename tScalarType, int tScalarCount>
struct RegisterBuffer {
  using ScalarType = tScalarType;
  static constexpr int kScalarCount = tScalarCount;
  using RegisterType = typename RegisterType<ScalarType, kScalarCount>::Type;
  static_assert((kScalarCount & (kScalarCount - 1)) == 0,
                "kScalarCount must be a power of two");
  static_assert(sizeof(RegisterType) % sizeof(ScalarType) == 0, "");
  static constexpr int kRegisterLanes =
      sizeof(RegisterType) / sizeof(ScalarType);
  static constexpr int kRegisterCount =
      (kScalarCount * sizeof(ScalarType) + sizeof(RegisterType) - 1) /
      sizeof(RegisterType);

  RegisterType reg[kRegisterCount];
};

template <typename tScalarType, int tRows, int tCols>
struct RegisterBlock {
  using ScalarType = tScalarType;
  static constexpr int kRows = tRows;
  static constexpr int kCols = tCols;
  static constexpr int kScalarCount = kRows * kCols;
  using BufferType = RegisterBuffer<ScalarType, kScalarCount>;
  using RegisterType = typename BufferType::RegisterType;
  static constexpr int kRegisterCount = BufferType::kRegisterCount;
  static constexpr int kRegisterLanes = BufferType::kRegisterLanes;

  BufferType buf;
};

template <typename RegisterBlockType>
struct RegisterBlockAddImpl {
  static RegisterBlockType Run(const RegisterBlockType& lhs,
                               const RegisterBlockType& rhs) {
    RegisterBlockType result;
    for (int i = 0; i < RegisterBlockType::kRegisterCount; i++) {
      result.buf.reg[i] = Add(lhs.buf.reg[i], rhs.buf.reg[i]);
    }
    return result;
  }
};

template <typename RegisterBlockType>
RegisterBlockType RegisterBlockAdd(const RegisterBlockType& lhs,
                                   const RegisterBlockType& rhs) {
  return RegisterBlockAddImpl<RegisterBlockType>::Run(lhs, rhs);
}

template <typename LhsType, typename RhsType>
struct ShouldFlipLhsRhs {
  static constexpr bool kValue =
      (LhsType::kScalarCount < RhsType::kScalarCount) ||
      (LhsType::kScalarCount == RhsType::kScalarCount &&
       (LhsType::kRows < RhsType::kRows));
};

template <typename LhsType, typename RhsType,
          bool Flip = ShouldFlipLhsRhs<LhsType, RhsType>::kValue>
struct FlipLhsRhs {
  using FlippedLhsType = LhsType;
  using FlippedRhsType = RhsType;
  static const FlippedLhsType& FlippedLhs(const LhsType& lhs,
                                          const RhsType& rhs) {
    (void)rhs;
    return lhs;
  }
  static const FlippedRhsType& FlippedRhs(const LhsType& lhs,
                                          const RhsType& rhs) {
    (void)lhs;
    return rhs;
  }
};

template <typename LhsType, typename RhsType>
struct FlipLhsRhs<LhsType, RhsType, true> {
  using FlippedLhsType = RhsType;
  using FlippedRhsType = LhsType;
  static const FlippedLhsType& FlippedLhs(const LhsType& lhs,
                                          const RhsType& rhs) {
    (void)lhs;
    return rhs;
  }
  static const FlippedRhsType& FlippedRhs(const LhsType& lhs,
                                          const RhsType& rhs) {
    (void)rhs;
    return lhs;
  }
};

template <typename Lhs, typename Rhs>
struct BroadcastBinaryOpShape {
  static constexpr int kRows =
      Lhs::kRows > Rhs::kRows ? Lhs::kRows : Rhs::kRows;
  static constexpr int kCols =
      Lhs::kCols > Rhs::kCols ? Lhs::kCols : Rhs::kCols;
};

template <typename Lhs, typename Rhs>
struct BroadcastBinaryOpRegisterBlock {
  using Shape = BroadcastBinaryOpShape<Lhs, Rhs>;
  using ScalarType = typename Lhs::ScalarType;
  using Type = RegisterBlock<ScalarType, Shape::kRows, Shape::kCols>;
};

template <typename Lhs, typename Rhs>
struct BroadcastAddImpl {
  using ResultBlockType =
      typename BroadcastBinaryOpRegisterBlock<Lhs, Rhs>::Type;
  static ResultBlockType Run(const Lhs& lhs, const Rhs& rhs) {
    ResultBlockType result;
    static constexpr int Rows = ResultBlockType::kRows;
    static constexpr int Cols = ResultBlockType::kCols;
    static constexpr int LhsRows = Lhs::kRows;
    static constexpr int LhsCols = Lhs::kCols;
    static constexpr int RhsRows = Rhs::kRows;
    static constexpr int RhsCols = Rhs::kCols;

    static_assert(LhsRows == Rows || LhsRows == 1, "");
    static_assert(RhsRows == Rows || RhsRows == 1, "");
    static_assert(LhsCols == Cols || LhsCols == 1, "");
    static_assert(RhsCols == Cols || RhsCols == 1, "");
    static_assert(ResultBlockType::kRegisterLanes == 1,
                  "This path is only for scalar values");
    static_assert(Lhs::kRegisterLanes == 1,
                  "This path is only for scalar values");
    static_assert(Rhs::kRegisterLanes == 1,
                  "This path is only for scalar values");

    for (int c = 0; c < Cols; c++) {
      const int lhs_c = LhsCols == Cols ? c : 0;
      const int rhs_c = RhsCols == Cols ? c : 0;
      for (int r = 0; r < Rows; r++) {
        const int lhs_r = LhsRows == Rows ? r : 0;
        const int rhs_r = RhsRows == Rows ? r : 0;
        result.buf.reg[r + c * Rows] =
            Add(lhs.buf.reg[lhs_r + lhs_c * LhsRows],
                rhs.buf.reg[rhs_r + rhs_c * RhsRows]);
      }
    }
    return result;
  }
};

template <typename Lhs, typename Rhs>
typename BroadcastBinaryOpRegisterBlock<Lhs, Rhs>::Type BroadcastAdd(
    const Lhs& lhs, const Rhs& rhs) {
  using Flip = FlipLhsRhs<Lhs, Rhs>;
  return BroadcastAddImpl<
      typename Flip::FlippedLhsType,
      typename Flip::FlippedRhsType>::Run(Flip::FlippedLhs(lhs, rhs),
                                          Flip::FlippedRhs(lhs, rhs));
}

template <typename Lhs, typename Rhs>
struct BroadcastShiftLeftImpl {
  using ResultBlockType =
      typename BroadcastBinaryOpRegisterBlock<Lhs, Rhs>::Type;
  static ResultBlockType Run(const Lhs& lhs, const Rhs& rhs) {
    ResultBlockType result;
    static constexpr int Rows = ResultBlockType::kRows;
    static constexpr int Cols = ResultBlockType::kCols;
    static constexpr int LhsRows = Lhs::kRows;
    static constexpr int LhsCols = Lhs::kCols;
    static constexpr int RhsRows = Rhs::kRows;
    static constexpr int RhsCols = Rhs::kCols;

    static_assert(LhsRows == Rows || LhsRows == 1, "");
    static_assert(RhsRows == Rows || RhsRows == 1, "");
    static_assert(LhsCols == Cols || LhsCols == 1, "");
    static_assert(RhsCols == Cols || RhsCols == 1, "");
    static_assert(ResultBlockType::kRegisterLanes == 1,
                  "This path is only for scalar values");
    static_assert(Lhs::kRegisterLanes == 1,
                  "This path is only for scalar values");
    static_assert(Rhs::kRegisterLanes == 1,
                  "This path is only for scalar values");

    for (int c = 0; c < Cols; c++) {
      const int lhs_c = LhsCols == Cols ? c : 0;
      const int rhs_c = RhsCols == Cols ? c : 0;
      for (int r = 0; r < Rows; r++) {
        const int lhs_r = LhsRows == Rows ? r : 0;
        const int rhs_r = RhsRows == Rows ? r : 0;
        result.buf.reg[r + c * Rows] =
            ShiftLeft(lhs.buf.reg[lhs_r + lhs_c * LhsRows],
                      rhs.buf.reg[rhs_r + rhs_c * RhsRows]);
      }
    }
    return result;
  }
};

template <typename Lhs, typename Rhs>
typename BroadcastBinaryOpRegisterBlock<Lhs, Rhs>::Type BroadcastShiftLeft(
    const Lhs& lhs, const Rhs& rhs) {
  using Flip = FlipLhsRhs<Lhs, Rhs>;
  return BroadcastShiftLeftImpl<
      typename Flip::FlippedLhsType,
      typename Flip::FlippedRhsType>::Run(Flip::FlippedLhs(lhs, rhs),
                                          Flip::FlippedRhs(lhs, rhs));
}

template <typename Lhs, typename Rhs>
struct BroadcastSaturatingRoundingDoublingHighMulImpl {
  using ResultBlockType =
      typename BroadcastBinaryOpRegisterBlock<Lhs, Rhs>::Type;
  static ResultBlockType Run(const Lhs& lhs, const Rhs& rhs) {
    ResultBlockType result;
    static constexpr int Rows = ResultBlockType::kRows;
    static constexpr int Cols = ResultBlockType::kCols;
    static constexpr int LhsRows = Lhs::kRows;
    static constexpr int LhsCols = Lhs::kCols;
    static constexpr int RhsRows = Rhs::kRows;
    static constexpr int RhsCols = Rhs::kCols;

    static_assert(LhsRows == Rows || LhsRows == 1, "");
    static_assert(RhsRows == Rows || RhsRows == 1, "");
    static_assert(LhsCols == Cols || LhsCols == 1, "");
    static_assert(RhsCols == Cols || RhsCols == 1, "");
    static_assert(ResultBlockType::kRegisterLanes == 1,
                  "This path is only for scalar values");
    static_assert(Lhs::kRegisterLanes == 1,
                  "This path is only for scalar values");
    static_assert(Rhs::kRegisterLanes == 1,
                  "This path is only for scalar values");

    for (int c = 0; c < Cols; c++) {
      const int lhs_c = LhsCols == Cols ? c : 0;
      const int rhs_c = RhsCols == Cols ? c : 0;
      for (int r = 0; r < Rows; r++) {
        const int lhs_r = LhsRows == Rows ? r : 0;
        const int rhs_r = RhsRows == Rows ? r : 0;
        result.buf.reg[r + c * Rows] = SaturatingRoundingDoublingHighMul(
            lhs.buf.reg[lhs_r + lhs_c * LhsRows],
            rhs.buf.reg[rhs_r + rhs_c * RhsRows]);
      }
    }
    return result;
  }
};

template <typename Lhs, typename Rhs>
typename BroadcastBinaryOpRegisterBlock<Lhs, Rhs>::Type
BroadcastSaturatingRoundingDoublingHighMul(const Lhs& lhs, const Rhs& rhs) {
  using Flip = FlipLhsRhs<Lhs, Rhs>;
  return BroadcastSaturatingRoundingDoublingHighMulImpl<
      typename Flip::FlippedLhsType,
      typename Flip::FlippedRhsType>::Run(Flip::FlippedLhs(lhs, rhs),
                                          Flip::FlippedRhs(lhs, rhs));
}

template <typename Lhs, typename Rhs>
struct BroadcastRoundingDivideByPOTImpl {
  using ResultBlockType =
      typename BroadcastBinaryOpRegisterBlock<Lhs, Rhs>::Type;
  static ResultBlockType Run(const Lhs& lhs, const Rhs& rhs) {
    ResultBlockType result;
    static constexpr int Rows = ResultBlockType::kRows;
    static constexpr int Cols = ResultBlockType::kCols;
    static constexpr int LhsRows = Lhs::kRows;
    static constexpr int LhsCols = Lhs::kCols;
    static constexpr int RhsRows = Rhs::kRows;
    static constexpr int RhsCols = Rhs::kCols;

    static_assert(LhsRows == Rows || LhsRows == 1, "");
    static_assert(RhsRows == Rows || RhsRows == 1, "");
    static_assert(LhsCols == Cols || LhsCols == 1, "");
    static_assert(RhsCols == Cols || RhsCols == 1, "");
    static_assert(ResultBlockType::kRegisterLanes == 1,
                  "This path is only for scalar values");
    static_assert(Lhs::kRegisterLanes == 1,
                  "This path is only for scalar values");
    static_assert(Rhs::kRegisterLanes == 1,
                  "This path is only for scalar values");

    for (int c = 0; c < Cols; c++) {
      const int lhs_c = LhsCols == Cols ? c : 0;
      const int rhs_c = RhsCols == Cols ? c : 0;
      for (int r = 0; r < Rows; r++) {
        const int lhs_r = LhsRows == Rows ? r : 0;
        const int rhs_r = RhsRows == Rows ? r : 0;
        result.buf.reg[r + c * Rows] =
            RoundingDivideByPOT(lhs.buf.reg[lhs_r + lhs_c * LhsRows],
                                rhs.buf.reg[rhs_r + rhs_c * RhsRows]);
      }
    }
    return result;
  }
};

template <typename Lhs, typename Rhs>
typename BroadcastBinaryOpRegisterBlock<Lhs, Rhs>::Type
BroadcastRoundingDivideByPOT(const Lhs& lhs, const Rhs& rhs) {
  using Flip = FlipLhsRhs<Lhs, Rhs>;
  return BroadcastRoundingDivideByPOTImpl<
      typename Flip::FlippedLhsType,
      typename Flip::FlippedRhsType>::Run(Flip::FlippedLhs(lhs, rhs),
                                          Flip::FlippedRhs(lhs, rhs));
}

template <typename Lhs, typename Rhs>
struct BroadcastMulImpl {
  using ResultBlockType =
      typename BroadcastBinaryOpRegisterBlock<Lhs, Rhs>::Type;
  static ResultBlockType Run(const Lhs& lhs, const Rhs& rhs) {
    ResultBlockType result;
    static constexpr int Rows = ResultBlockType::kRows;
    static constexpr int Cols = ResultBlockType::kCols;
    static constexpr int LhsRows = Lhs::kRows;
    static constexpr int LhsCols = Lhs::kCols;
    static constexpr int RhsRows = Rhs::kRows;
    static constexpr int RhsCols = Rhs::kCols;
    static_assert(ResultBlockType::kRegisterLanes == 1,
                  "This path is only for scalar values");
    static_assert(Lhs::kRegisterLanes == 1,
                  "This path is only for scalar values");
    static_assert(Rhs::kRegisterLanes == 1,
                  "This path is only for scalar values");

    static_assert(LhsRows == Rows || LhsRows == 1, "");
    static_assert(RhsRows == Rows || RhsRows == 1, "");
    static_assert(LhsCols == Cols || LhsCols == 1, "");
    static_assert(RhsCols == Cols || RhsCols == 1, "");
    for (int c = 0; c < Cols; c++) {
      const int lhs_c = LhsCols == Cols ? c : 0;
      const int rhs_c = RhsCols == Cols ? c : 0;
      for (int r = 0; r < Rows; r++) {
        const int lhs_r = LhsRows == Rows ? r : 0;
        const int rhs_r = RhsRows == Rows ? r : 0;
        result.buf.reg[r + c * Rows] =
            Mul(lhs.buf.reg[lhs_r + lhs_c * LhsRows],
                rhs.buf.reg[rhs_r + rhs_c * RhsRows]);
      }
    }
    return result;
  }
};

template <typename Lhs, typename Rhs>
typename BroadcastBinaryOpRegisterBlock<Lhs, Rhs>::Type BroadcastMul(
    const Lhs& lhs, const Rhs& rhs) {
  using Flip = FlipLhsRhs<Lhs, Rhs>;
  return BroadcastMulImpl<
      typename Flip::FlippedLhsType,
      typename Flip::FlippedRhsType>::Run(Flip::FlippedLhs(lhs, rhs),
                                          Flip::FlippedRhs(lhs, rhs));
}

template <typename Lhs, typename Rhs, typename Acc>
struct BroadcastMulAddImpl {
  static void Run(const Lhs& lhs, const Rhs& rhs, Acc* acc) {
    static constexpr int Rows = Acc::kRows;
    static constexpr int Cols = Acc::kCols;
    static constexpr int LhsRows = Lhs::kRows;
    static constexpr int LhsCols = Lhs::kCols;
    static constexpr int RhsRows = Rhs::kRows;
    static constexpr int RhsCols = Rhs::kCols;
    static_assert(Acc::kRegisterLanes == 1,
                  "This path is only for scalar values");
    static_assert(Lhs::kRegisterLanes == 1,
                  "This path is only for scalar values");
    static_assert(Rhs::kRegisterLanes == 1,
                  "This path is only for scalar values");

    static_assert(LhsRows == Rows || LhsRows == 1, "");
    static_assert(RhsRows == Rows || RhsRows == 1, "");
    static_assert(LhsCols == Cols || LhsCols == 1, "");
    static_assert(RhsCols == Cols || RhsCols == 1, "");
    for (int c = 0; c < Cols; c++) {
      const int lhs_c = LhsCols == Cols ? c : 0;
      const int rhs_c = RhsCols == Cols ? c : 0;
      for (int r = 0; r < Rows; r++) {
        const int lhs_r = LhsRows == Rows ? r : 0;
        const int rhs_r = RhsRows == Rows ? r : 0;
        MulAdd(lhs.buf.reg[lhs_r + lhs_c * LhsRows],
               rhs.buf.reg[rhs_r + rhs_c * RhsRows],
               &acc->buf.reg[r + c * Rows]);
      }
    }
  }
};

template <typename Lhs, typename Rhs, typename Acc>
void BroadcastMulAdd(const Lhs& lhs, const Rhs& rhs, Acc* acc) {
  using Flip = FlipLhsRhs<Lhs, Rhs>;
  BroadcastMulAddImpl<typename Flip::FlippedLhsType,
                      typename Flip::FlippedRhsType,
                      Acc>::Run(Flip::FlippedLhs(lhs, rhs),
                                Flip::FlippedRhs(lhs, rhs), acc);
}

template <typename RegisterBlockType, typename SrcObjectType>
struct LoadImpl {
  static_assert(std::is_same<SrcObjectType, void>::value,
                "This generic impl should never be hit");
};

template <typename ScalarType, int Rows, int Cols, typename SrcScalarType>
struct LoadImpl<RegisterBlock<ScalarType, Rows, Cols>,
                MatrixMap<SrcScalarType, MapOrder::ColMajor>> {
  using RegisterBlockType = RegisterBlock<ScalarType, Rows, Cols>;
  using SrcObjectType = MatrixMap<SrcScalarType, MapOrder::ColMajor>;
  static RegisterBlockType Run(const SrcObjectType& src, int row, int col) {
    RegisterBlockType result;
    int i = 0;
    for (int c = 0; c < Cols; c++) {
      const ScalarType* src_ptr = src.data(row, col + c);
      for (int r = 0; r < Rows; r++) {
        result.buf.reg[i++] = *src_ptr++;
      }
    }
    return result;
  }
};

template <typename ScalarType, int Rows, int Cols, typename SrcScalarType,
          VectorShape Shape>
struct LoadImpl<RegisterBlock<ScalarType, Rows, Cols>,
                VectorMap<SrcScalarType, Shape>> {
  using RegisterBlockType = RegisterBlock<ScalarType, Rows, Cols>;
  using SrcObjectType = VectorMap<SrcScalarType, Shape>;
  static RegisterBlockType Run(const SrcObjectType& src, int pos) {
    static_assert(Shape == VectorShape::Col || Rows == 1, "");
    static_assert(Shape == VectorShape::Row || Cols == 1, "");
    RegisterBlockType result;
    for (int i = 0; i < Rows * Cols; i++) {
      result.buf.reg[i] = src(pos + i);
    }
    return result;
  }
};

template <typename ScalarType, int Rows, int Cols, typename SrcScalarType,
          VectorShape Shape>
struct LoadImpl<RegisterBlock<ScalarType, Rows, Cols>,
                VectorDup<SrcScalarType, Shape>> {
  using RegisterBlockType = RegisterBlock<ScalarType, Rows, Cols>;
  using SrcObjectType = VectorDup<SrcScalarType, Shape>;
  static RegisterBlockType Run(const SrcObjectType& src, int) {
    static_assert(Shape == VectorShape::Col || Rows == 1, "");
    static_assert(Shape == VectorShape::Row || Cols == 1, "");
    RegisterBlockType result;
    for (int i = 0; i < Rows * Cols; i++) {
      result.buf.reg[i] = src(0);
    }
    return result;
  }
};

template <typename RegisterBlockType, typename SrcObjectType>
RegisterBlockType Load(const SrcObjectType& src, int row, int col) {
  return LoadImpl<RegisterBlockType, SrcObjectType>::Run(src, row, col);
}

template <typename RegisterBlockType, typename SrcObjectType>
RegisterBlockType Load(const SrcObjectType& src, int pos) {
  return LoadImpl<RegisterBlockType, SrcObjectType>::Run(src, pos);
}

template <typename RegisterBlockType>
struct LoadContiguousImpl {
  using ScalarType = typename RegisterBlockType::ScalarType;
  static_assert(RegisterBlockType::kRegisterLanes == 1,
                "This path is only for scalar values");
  static RegisterBlockType Run(const ScalarType* src) {
    RegisterBlockType result;
    for (int i = 0; i < RegisterBlockType::kScalarCount; i++) {
      result.buf.reg[i] = src[i];
    }
    return result;
  }
};

template <typename RegisterBlockType>
RegisterBlockType LoadContiguous(
    const typename RegisterBlockType::ScalarType* src) {
  return LoadContiguousImpl<RegisterBlockType>::Run(src);
}

template <int BroadcastRows, int BroadcastCols, typename SrcObjectType>
struct LoadForBroadcastingShape {};

template <int BroadcastRows, int BroadcastCols, typename ScalarType,
          VectorShape Shape>
struct LoadForBroadcastingShape<BroadcastRows, BroadcastCols,
                                VectorMap<ScalarType, Shape>> {
  static constexpr int kRows = Shape == VectorShape::Col ? BroadcastRows : 1;
  static constexpr int kCols = Shape == VectorShape::Row ? BroadcastCols : 1;
};

template <int BroadcastRows, int BroadcastCols, typename ScalarType,
          VectorShape Shape>
struct LoadForBroadcastingShape<BroadcastRows, BroadcastCols,
                                VectorDup<ScalarType, Shape>> {
  static constexpr int kRows = 1;
  static constexpr int kCols = 1;
};

template <typename RegisterBlockType, typename SrcObjectType>
struct LoadForBroadcastingRegisterBlock {
  using Shape =
      LoadForBroadcastingShape<RegisterBlockType::kRows,
                               RegisterBlockType::kCols, SrcObjectType>;
  using ScalarType = typename RegisterBlockType::ScalarType;
  using Type = RegisterBlock<ScalarType, Shape::kRows, Shape::kCols>;
};

template <typename RegisterBlockType, typename SrcObjectType>
struct LoadForBroadcastingImpl {
  static_assert(std::is_same<SrcObjectType, void>::value,
                "This generic impl should never be hit");
};

template <typename ScalarType, int Rows, int Cols, typename SrcScalarType,
          VectorShape Shape>
struct LoadForBroadcastingImpl<RegisterBlock<ScalarType, Rows, Cols>,
                               VectorMap<SrcScalarType, Shape>> {
  using RegisterBlockType = RegisterBlock<ScalarType, Rows, Cols>;
  using SrcObjectType = VectorMap<SrcScalarType, Shape>;
  using ResultBlockType =
      typename LoadForBroadcastingRegisterBlock<RegisterBlockType,
                                                SrcObjectType>::Type;
  static_assert(ResultBlockType::kRegisterLanes == 1,
                "This path is only for scalar values");
  static ResultBlockType Run(const SrcObjectType& src, int pos) {
    ResultBlockType result;
    for (int c = 0; c < ResultBlockType::kCols; c++) {
      for (int r = 0; r < ResultBlockType::kRows; r++) {
        const int i = Shape == VectorShape::Col ? r : c;
        result.buf.reg[r + c * ResultBlockType::kRows] = src(pos + i);
      }
    }
    return result;
  }
};

template <typename ScalarType, int Rows, int Cols, typename SrcScalarType,
          VectorShape Shape>
struct LoadForBroadcastingImpl<RegisterBlock<ScalarType, Rows, Cols>,
                               VectorDup<SrcScalarType, Shape>> {
  using RegisterBlockType = RegisterBlock<ScalarType, Rows, Cols>;
  using SrcObjectType = VectorDup<SrcScalarType, Shape>;
  using ResultBlockType =
      typename LoadForBroadcastingRegisterBlock<RegisterBlockType,
                                                SrcObjectType>::Type;
  static_assert(ResultBlockType::kRegisterLanes == 1,
                "This path is only for scalar values");
  static ResultBlockType Run(const SrcObjectType& src, int) {
    ResultBlockType result;
    for (int c = 0; c < ResultBlockType::kCols; c++) {
      for (int r = 0; r < ResultBlockType::kRows; r++) {
        result.buf.reg[r + c * ResultBlockType::kRows] = src(0);
      }
    }
    return result;
  }
};

template <typename RegisterBlockType, typename SrcObjectType>
typename LoadForBroadcastingRegisterBlock<RegisterBlockType,
                                          SrcObjectType>::Type
LoadForBroadcasting(const SrcObjectType& src, int row, int col) {
  return LoadForBroadcastingImpl<RegisterBlockType, SrcObjectType>::Run(
      src, row, col);
}

template <typename RegisterBlockType, typename SrcObjectType>
typename LoadForBroadcastingRegisterBlock<RegisterBlockType,
                                          SrcObjectType>::Type
LoadForBroadcasting(const SrcObjectType& src, int pos) {
  return LoadForBroadcastingImpl<RegisterBlockType, SrcObjectType>::Run(src,
                                                                        pos);
}

template <int ConstantValue, typename RegisterBlockType>
struct AddConstantImpl {
  static void Run(RegisterBlockType* block) {
    using RegisterType = typename RegisterBlockType::RegisterType;
    const RegisterType dup = Dup<RegisterType>(ConstantValue);
    for (int i = 0; i < RegisterBlockType::kRegisterCount; i++) {
      block->buf.reg[i] = Add(block->buf.reg[i], dup);
    }
  }
};

template <typename RegisterBlockType>
struct AddConstantImpl<0, RegisterBlockType> {
  static void Run(RegisterBlockType*) {
    // This is a no-op.
  }
};

template <int ConstantValue, typename RegisterBlockType>
void AddConstant(RegisterBlockType* block) {
  AddConstantImpl<ConstantValue, RegisterBlockType>::Run(block);
}

template <int N>
using RegBufferInt32 = RegisterBuffer<std::int32_t, N>;
template <int N>
using RegBufferInt16 = RegisterBuffer<std::int16_t, N>;
template <int N>
using RegBufferUint8 = RegisterBuffer<std::uint8_t, N>;
template <int N>
using RegBufferInt8 = RegisterBuffer<std::int8_t, N>;
template <int R, int C>
using RegBlockInt32 = RegisterBlock<std::int32_t, R, C>;
template <int R, int C>
using RegBlockInt16 = RegisterBlock<std::int16_t, R, C>;
template <int R, int C>
using RegBlockUint8 = RegisterBlock<std::uint8_t, R, C>;
template <int R, int C>
using RegBlockInt8 = RegisterBlock<std::int8_t, R, C>;

}  // end namespace gemmlowp

#if defined GEMMLOWP_NEON
#include "simd_wrappers_neon.h"
#elif defined GEMMLOWP_SSE4
#include "simd_wrappers_sse.h"
#elif defined GEMMLOWP_MSA
#include "simd_wrappers_msa.h"
#endif

#endif  // GEMMLOWP_INTERNAL_SIMD_WRAPPERS_H_