xref: /aosp_15_r20/external/llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp (revision 9880d6810fe72a1726cb53787c6711e909410d58)

//===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// Define several functions to decode x86 specific shuffle semantics using
// constants from the constant pool.
//
//===----------------------------------------------------------------------===//

#include "X86ShuffleDecodeConstantPool.h"
#include "Utils/X86ShuffleDecode.h"
#include "llvm/CodeGen/MachineValueType.h"
#include "llvm/IR/Constants.h"

//===----------------------------------------------------------------------===//
//  Vector Mask Decoding
//===----------------------------------------------------------------------===//

namespace llvm {

void DecodePSHUFBMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
  Type *MaskTy = C->getType();
  // It is not an error for the PSHUFB mask to not be a vector of i8 because the
  // constant pool uniques constants by their bit representation.
  // e.g. the following take up the same space in the constant pool:
  //   i128 -170141183420855150465331762880109871104
  //
  //   <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
  //
  //   <4 x i32> <i32 -2147483648, i32 -2147483648,
  //              i32 -2147483648, i32 -2147483648>

#ifndef NDEBUG
  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
  assert(MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512);
#endif

  if (!MaskTy->isVectorTy())
    return;
  int NumElts = MaskTy->getVectorNumElements();

  Type *EltTy = MaskTy->getVectorElementType();
  if (!EltTy->isIntegerTy())
    return;

  // The shuffle mask requires a byte vector - decode cases with
  // wider elements as well.
  unsigned BitWidth = cast<IntegerType>(EltTy)->getBitWidth();
  if ((BitWidth % 8) != 0)
    return;

  int Scale = BitWidth / 8;
  int NumBytes = NumElts * Scale;
  ShuffleMask.reserve(NumBytes);

  for (int i = 0; i != NumElts; ++i) {
    Constant *COp = C->getAggregateElement(i);
    if (!COp) {
      ShuffleMask.clear();
      return;
    } else if (isa<UndefValue>(COp)) {
      ShuffleMask.append(Scale, SM_SentinelUndef);
      continue;
    }

    APInt APElt = cast<ConstantInt>(COp)->getValue();
    for (int j = 0; j != Scale; ++j) {
      // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
      // lane of the vector we're inside.
      int Base = ((i * Scale) + j) & ~0xf;

      uint64_t Element = APElt.getLoBits(8).getZExtValue();
      APElt = APElt.lshr(8);

      // If the high bit (7) of the byte is set, the element is zeroed.
      if (Element & (1 << 7))
        ShuffleMask.push_back(SM_SentinelZero);
      else {
        // Only the least significant 4 bits of the byte are used.
        int Index = Base + (Element & 0xf);
        ShuffleMask.push_back(Index);
      }
    }
  }

  assert(NumBytes == (int)ShuffleMask.size() && "Unexpected shuffle mask size");
}

void DecodeVPERMILPMask(const Constant *C, unsigned ElSize,
                        SmallVectorImpl<int> &ShuffleMask) {
  Type *MaskTy = C->getType();
  // It is not an error for the PSHUFB mask to not be a vector of i8 because the
  // constant pool uniques constants by their bit representation.
  // e.g. the following take up the same space in the constant pool:
  //   i128 -170141183420855150465331762880109871104
  //
  //   <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
  //
  //   <4 x i32> <i32 -2147483648, i32 -2147483648,
  //              i32 -2147483648, i32 -2147483648>

  if (ElSize != 32 && ElSize != 64)
    return;

  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
  if (MaskTySize != 128 && MaskTySize != 256 && MaskTySize != 512)
    return;

  // Only support vector types.
  if (!MaskTy->isVectorTy())
    return;

  // Make sure its an integer type.
  Type *VecEltTy = MaskTy->getVectorElementType();
  if (!VecEltTy->isIntegerTy())
    return;

  // Support any element type from byte up to element size.
  // This is necessary primarily because 64-bit elements get split to 32-bit
  // in the constant pool on 32-bit target.
  unsigned EltTySize = VecEltTy->getIntegerBitWidth();
  if (EltTySize < 8 || EltTySize > ElSize)
    return;

  unsigned NumElements = MaskTySize / ElSize;
  assert((NumElements == 2 || NumElements == 4 || NumElements == 8 ||
          NumElements == 16) &&
         "Unexpected number of vector elements.");
  ShuffleMask.reserve(NumElements);
  unsigned NumElementsPerLane = 128 / ElSize;
  unsigned Factor = ElSize / EltTySize;

  for (unsigned i = 0; i < NumElements; ++i) {
    Constant *COp = C->getAggregateElement(i * Factor);
    if (!COp) {
      ShuffleMask.clear();
      return;
    } else if (isa<UndefValue>(COp)) {
      ShuffleMask.push_back(SM_SentinelUndef);
      continue;
    }
    int Index = i & ~(NumElementsPerLane - 1);
    uint64_t Element = cast<ConstantInt>(COp)->getZExtValue();
    if (ElSize == 64)
      Index += (Element >> 1) & 0x1;
    else
      Index += Element & 0x3;
    ShuffleMask.push_back(Index);
  }

  // TODO: Handle funny-looking vectors too.
}

void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize,
                         SmallVectorImpl<int> &ShuffleMask) {
  Type *MaskTy = C->getType();

  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
  if (MaskTySize != 128 && MaskTySize != 256)
    return;

  // Only support vector types.
  if (!MaskTy->isVectorTy())
    return;

  // Make sure its an integer type.
  Type *VecEltTy = MaskTy->getVectorElementType();
  if (!VecEltTy->isIntegerTy())
    return;

  // Support any element type from byte up to element size.
  // This is necessary primarily because 64-bit elements get split to 32-bit
  // in the constant pool on 32-bit target.
  unsigned EltTySize = VecEltTy->getIntegerBitWidth();
  if (EltTySize < 8 || EltTySize > ElSize)
    return;

  unsigned NumElements = MaskTySize / ElSize;
  assert((NumElements == 2 || NumElements == 4 || NumElements == 8) &&
         "Unexpected number of vector elements.");
  ShuffleMask.reserve(NumElements);
  unsigned NumElementsPerLane = 128 / ElSize;
  unsigned Factor = ElSize / EltTySize;

  for (unsigned i = 0; i < NumElements; ++i) {
    Constant *COp = C->getAggregateElement(i * Factor);
    if (!COp) {
      ShuffleMask.clear();
      return;
    } else if (isa<UndefValue>(COp)) {
      ShuffleMask.push_back(SM_SentinelUndef);
      continue;
    }

    // VPERMIL2 Operation.
    // Bits[3] - Match Bit.
    // Bits[2:1] - (Per Lane) PD Shuffle Mask.
    // Bits[2:0] - (Per Lane) PS Shuffle Mask.
    uint64_t Selector = cast<ConstantInt>(COp)->getZExtValue();
    unsigned MatchBit = (Selector >> 3) & 0x1;

    // M2Z[0:1]     MatchBit
    //   0Xb           X        Source selected by Selector index.
    //   10b           0        Source selected by Selector index.
    //   10b           1        Zero.
    //   11b           0        Zero.
    //   11b           1        Source selected by Selector index.
    if ((M2Z & 0x2) != 0u && MatchBit != (M2Z & 0x1)) {
      ShuffleMask.push_back(SM_SentinelZero);
      continue;
    }

    int Index = i & ~(NumElementsPerLane - 1);
    if (ElSize == 64)
      Index += (Selector >> 1) & 0x1;
    else
      Index += Selector & 0x3;

    int Src = (Selector >> 2) & 0x1;
    Index += Src * NumElements;
    ShuffleMask.push_back(Index);
  }

  // TODO: Handle funny-looking vectors too.
}

void DecodeVPPERMMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
  Type *MaskTy = C->getType();
  assert(MaskTy->getPrimitiveSizeInBits() == 128);

  // Only support vector types.
  if (!MaskTy->isVectorTy())
    return;

  // Make sure its an integer type.
  Type *VecEltTy = MaskTy->getVectorElementType();
  if (!VecEltTy->isIntegerTy())
    return;

  // The shuffle mask requires a byte vector - decode cases with
  // wider elements as well.
  unsigned BitWidth = cast<IntegerType>(VecEltTy)->getBitWidth();
  if ((BitWidth % 8) != 0)
    return;

  int NumElts = MaskTy->getVectorNumElements();
  int Scale = BitWidth / 8;
  int NumBytes = NumElts * Scale;
  ShuffleMask.reserve(NumBytes);

  for (int i = 0; i != NumElts; ++i) {
    Constant *COp = C->getAggregateElement(i);
    if (!COp) {
      ShuffleMask.clear();
      return;
    } else if (isa<UndefValue>(COp)) {
      ShuffleMask.append(Scale, SM_SentinelUndef);
      continue;
    }

    // VPPERM Operation
    // Bits[4:0] - Byte Index (0 - 31)
    // Bits[7:5] - Permute Operation
    //
    // Permute Operation:
    // 0 - Source byte (no logical operation).
    // 1 - Invert source byte.
    // 2 - Bit reverse of source byte.
    // 3 - Bit reverse of inverted source byte.
    // 4 - 00h (zero - fill).
    // 5 - FFh (ones - fill).
    // 6 - Most significant bit of source byte replicated in all bit positions.
    // 7 - Invert most significant bit of source byte and replicate in all bit positions.
    APInt MaskElt = cast<ConstantInt>(COp)->getValue();
    for (int j = 0; j != Scale; ++j) {
      APInt Index = MaskElt.getLoBits(5);
      APInt PermuteOp = MaskElt.lshr(5).getLoBits(3);
      MaskElt = MaskElt.lshr(8);

      if (PermuteOp == 4) {
        ShuffleMask.push_back(SM_SentinelZero);
        continue;
      }
      if (PermuteOp != 0) {
        ShuffleMask.clear();
        return;
      }
      ShuffleMask.push_back((int)Index.getZExtValue());
    }
  }

  assert(NumBytes == (int)ShuffleMask.size() && "Unexpected shuffle mask size");
}

void DecodeVPERMVMask(const Constant *C, MVT VT,
                      SmallVectorImpl<int> &ShuffleMask) {
  Type *MaskTy = C->getType();
  if (MaskTy->isVectorTy()) {
    unsigned NumElements = MaskTy->getVectorNumElements();
    if (NumElements == VT.getVectorNumElements()) {
      unsigned EltMaskSize = Log2_64(NumElements);
      for (unsigned i = 0; i < NumElements; ++i) {
        Constant *COp = C->getAggregateElement(i);
        if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp))) {
          ShuffleMask.clear();
          return;
        }
        if (isa<UndefValue>(COp))
          ShuffleMask.push_back(SM_SentinelUndef);
        else {
          APInt Element = cast<ConstantInt>(COp)->getValue();
          Element = Element.getLoBits(EltMaskSize);
          ShuffleMask.push_back(Element.getZExtValue());
        }
      }
    }
    return;
  }
  // Scalar value; just broadcast it
  if (!isa<ConstantInt>(C))
    return;
  uint64_t Element = cast<ConstantInt>(C)->getZExtValue();
  int NumElements = VT.getVectorNumElements();
  Element &= (1 << NumElements) - 1;
  for (int i = 0; i < NumElements; ++i)
    ShuffleMask.push_back(Element);
}

void DecodeVPERMV3Mask(const Constant *C, MVT VT,
                       SmallVectorImpl<int> &ShuffleMask) {
  Type *MaskTy = C->getType();
  unsigned NumElements = MaskTy->getVectorNumElements();
  if (NumElements == VT.getVectorNumElements()) {
    unsigned EltMaskSize = Log2_64(NumElements * 2);
    for (unsigned i = 0; i < NumElements; ++i) {
      Constant *COp = C->getAggregateElement(i);
      if (!COp) {
        ShuffleMask.clear();
        return;
      }
      if (isa<UndefValue>(COp))
        ShuffleMask.push_back(SM_SentinelUndef);
      else {
        APInt Element = cast<ConstantInt>(COp)->getValue();
        Element = Element.getLoBits(EltMaskSize);
        ShuffleMask.push_back(Element.getZExtValue());
      }
    }
  }
}
} // llvm namespace