This is an archive of the discontinued LLVM Phabricator instance.

Differential D42723

[AArch64] Refactor instructions using SIMD immediates
ClosedPublic

Authored by evandro on Jan 30 2018, 5:24 PM.

Details

Reviewers
efriedma
sdesmalen
SjoerdMeijer
Commits
rG7dc0f1ec451a: [AArch64] Refactor identification of SIMD immediates
rL324903: [AArch64] Refactor identification of SIMD immediates
Summary

Get rid of icky goto loops and make the code easier to maintain.

Diff Detail

Repository
rL LLVM

Event Timeline

evandro created this revision.Jan 30 2018, 5:24 PM
Herald added subscribers: llvm-commits, hiraditya, kristof.beyls and 3 others. · View Herald TranscriptJan 30 2018, 5:24 PM
evandro added a child revision: D42133: [AArch64] Improve code generation of constant vectors.Jan 30 2018, 5:25 PM
evandro added a comment.Feb 7 2018, 8:42 AM

Ping! 🔔

evandro added a subscriber: SjoerdMeijer.Feb 7 2018, 8:43 AM
SjoerdMeijer added inline comments.Feb 7 2018, 12:58 PM
llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
540

drive-by comment: why are you changing the test?

The clean up looks very reasonable to me, but it's a big patch,
so I need some more time to digest the rest.

evandro marked an inline comment as done.Feb 7 2018, 2:35 PM
evandro added inline comments.
llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
540

Oops, it's a git leak. 🙂

SjoerdMeijer accepted this revision.Feb 12 2018, 5:27 AM

Looks like a reasonable clean up to me.

This revision is now accepted and ready to land.Feb 12 2018, 5:27 AM
Closed by commit rL324903: [AArch64] Refactor identification of SIMD immediates (authored by evandro). · Explain WhyFeb 12 2018, 8:43 AM
This revision was automatically updated to reflect the committed changes.
evandro marked an inline comment as done.

Revision Contents

PathSize
llvm/
lib/
Target/
AArch64/
648 lines
test/
CodeGen/
AArch64/
4 lines

Diff 132085

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 6,224 Lines • ▼ Show 20 Lines if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
} }
return true; return true;
} }
return false; return false;
} }
SDValue AArch64TargetLowering::LowerVectorAND(SDValue Op, // Try 64-bit splatted SIMD immediate.
SelectionDAG &DAG) const { static SDValue tryAdvSIMDModImm64(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
BuildVectorSDNode *BVN = const APInt &Bits) {
dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode()); if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
SDValue LHS = Op.getOperand(0); uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
SDLoc dl(Op);
EVT VT = Op.getValueType(); EVT VT = Op.getValueType();
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v2i64 : MVT::f64;
if (!BVN) if (AArch64_AM::isAdvSIMDModImmType10(Value)) {
return Op; Value = AArch64_AM::encodeAdvSIMDModImmType10(Value);
APInt CnstBits(VT.getSizeInBits(), 0); SDLoc dl(Op);
APInt UndefBits(VT.getSizeInBits(), 0); SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
if (resolveBuildVector(BVN, CnstBits, UndefBits)) { DAG.getConstant(Value, dl, MVT::i32));
// We only have BIC vector immediate instruction, which is and-not. return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
CnstBits = ~CnstBits; }
}
// We make use of a little bit of goto ickiness in order to avoid having to return SDValue();
// duplicate the immediate matching logic for the undef toggled case. }
bool SecondTry = false;
AttemptModImm:
if (CnstBits.getHiBits(64) == CnstBits.getLoBits(64)) {
CnstBits = CnstBits.zextOrTrunc(64);
uint64_t CnstVal = CnstBits.getZExtValue();
if (AArch64_AM::isAdvSIMDModImmType1(CnstVal)) { // Try 32-bit splatted SIMD immediate.
CnstVal = AArch64_AM::encodeAdvSIMDModImmType1(CnstVal); static SDValue tryAdvSIMDModImm32(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
const APInt &Bits,
const SDValue *LHS = nullptr) {
if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
EVT VT = Op.getValueType();
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS, bool isAdvSIMDModImm = false;
DAG.getConstant(CnstVal, dl, MVT::i32), uint64_t Shift;
DAG.getConstant(0, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType1(Value))) {
Value = AArch64_AM::encodeAdvSIMDModImmType1(Value);
Shift = 0;
}
else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType2(Value))) {
Value = AArch64_AM::encodeAdvSIMDModImmType2(Value);
Shift = 8;
}
else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType3(Value))) {
Value = AArch64_AM::encodeAdvSIMDModImmType3(Value);
Shift = 16;
}
else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType4(Value))) {
Value = AArch64_AM::encodeAdvSIMDModImmType4(Value);
Shift = 24;
} }
if (AArch64_AM::isAdvSIMDModImmType2(CnstVal)) { if (isAdvSIMDModImm) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType2(CnstVal); SDLoc dl(Op);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; SDValue Mov;
SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS,
DAG.getConstant(CnstVal, dl, MVT::i32), if (LHS)
DAG.getConstant(8, dl, MVT::i32)); Mov = DAG.getNode(NewOp, dl, MovTy, *LHS,
DAG.getConstant(Value, dl, MVT::i32),
DAG.getConstant(Shift, dl, MVT::i32));
else
Mov = DAG.getNode(NewOp, dl, MovTy,
DAG.getConstant(Value, dl, MVT::i32),
DAG.getConstant(Shift, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
} }
}
return SDValue();
}
// Try 16-bit splatted SIMD immediate.
static SDValue tryAdvSIMDModImm16(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
const APInt &Bits,
const SDValue *LHS = nullptr) {
if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
EVT VT = Op.getValueType();
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
bool isAdvSIMDModImm = false;
uint64_t Shift;
if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType5(Value))) {
Value = AArch64_AM::encodeAdvSIMDModImmType5(Value);
Shift = 0;
}
else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType6(Value))) {
Value = AArch64_AM::encodeAdvSIMDModImmType6(Value);
Shift = 8;
}
if (isAdvSIMDModImm) {
SDLoc dl(Op);
SDValue Mov;
if (LHS)
Mov = DAG.getNode(NewOp, dl, MovTy, *LHS,
DAG.getConstant(Value, dl, MVT::i32),
DAG.getConstant(Shift, dl, MVT::i32));
else
Mov = DAG.getNode(NewOp, dl, MovTy,
DAG.getConstant(Value, dl, MVT::i32),
DAG.getConstant(Shift, dl, MVT::i32));
if (AArch64_AM::isAdvSIMDModImmType3(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType3(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(16, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
} }
}
return SDValue();
}
if (AArch64_AM::isAdvSIMDModImmType4(CnstVal)) { // Try 32-bit splatted SIMD immediate with shifted ones.
CnstVal = AArch64_AM::encodeAdvSIMDModImmType4(CnstVal); static SDValue tryAdvSIMDModImm321s(unsigned NewOp, SDValue Op,
SelectionDAG &DAG, const APInt &Bits) {
if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
EVT VT = Op.getValueType();
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS, bool isAdvSIMDModImm = false;
DAG.getConstant(CnstVal, dl, MVT::i32), uint64_t Shift;
DAG.getConstant(24, dl, MVT::i32));
if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType7(Value))) {
Value = AArch64_AM::encodeAdvSIMDModImmType7(Value);
Shift = 264;
}
else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType8(Value))) {
Value = AArch64_AM::encodeAdvSIMDModImmType8(Value);
Shift = 272;
}
if (isAdvSIMDModImm) {
SDLoc dl(Op);
SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
DAG.getConstant(Value, dl, MVT::i32),
DAG.getConstant(Shift, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
} }
}
if (AArch64_AM::isAdvSIMDModImmType5(CnstVal)) { return SDValue();
CnstVal = AArch64_AM::encodeAdvSIMDModImmType5(CnstVal); }
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS, // Try 8-bit splatted SIMD immediate.
DAG.getConstant(CnstVal, dl, MVT::i32), static SDValue tryAdvSIMDModImm8(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
DAG.getConstant(0, dl, MVT::i32)); const APInt &Bits) {
if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
EVT VT = Op.getValueType();
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v16i8 : MVT::v8i8;
if (AArch64_AM::isAdvSIMDModImmType9(Value)) {
Value = AArch64_AM::encodeAdvSIMDModImmType9(Value);
SDLoc dl(Op);
SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
DAG.getConstant(Value, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
} }
}
if (AArch64_AM::isAdvSIMDModImmType6(CnstVal)) { return SDValue();
CnstVal = AArch64_AM::encodeAdvSIMDModImmType6(CnstVal); }
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS, // Try FP splatted SIMD immediate.
DAG.getConstant(CnstVal, dl, MVT::i32), static SDValue tryAdvSIMDModImmFP(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
DAG.getConstant(8, dl, MVT::i32)); const APInt &Bits) {
if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
EVT VT = Op.getValueType();
bool isWide = (VT.getSizeInBits() == 128);
MVT MovTy;
bool isAdvSIMDModImm = false;
if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType11(Value))) {
Value = AArch64_AM::encodeAdvSIMDModImmType11(Value);
MovTy = isWide ? MVT::v4f32 : MVT::v2f32;
}
else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType12(Value))) {
Value = AArch64_AM::encodeAdvSIMDModImmType12(Value);
MovTy = isWide ? MVT::v2f64 : MVT::f64;
}
if (isAdvSIMDModImm) {
SDLoc dl(Op);
SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
DAG.getConstant(Value, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
} }
} }
if (SecondTry) return SDValue();
goto FailedModImm; }
SecondTry = true;
CnstBits = ~UndefBits; SDValue AArch64TargetLowering::LowerVectorAND(SDValue Op,
goto AttemptModImm; SelectionDAG &DAG) const {
SDValue LHS = Op.getOperand(0);
EVT VT = Op.getValueType();
BuildVectorSDNode *BVN =
dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode());
if (!BVN) {
// AND commutes, so try swapping the operands.
LHS = Op.getOperand(1);
BVN = dyn_cast<BuildVectorSDNode>(Op.getOperand(0).getNode());
}
if (!BVN)
return Op;
APInt DefBits(VT.getSizeInBits(), 0);
APInt UndefBits(VT.getSizeInBits(), 0);
if (resolveBuildVector(BVN, DefBits, UndefBits)) {
SDValue NewOp;
// We only have BIC vector immediate instruction, which is and-not.
DefBits = ~DefBits;
if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, Op, DAG,
DefBits, &LHS)) ||
(NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, Op, DAG,
DefBits, &LHS)))
return NewOp;
else {
DefBits = ~UndefBits;
if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, Op, DAG,
DefBits, &LHS)) ||
(NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, Op, DAG,
DefBits, &LHS)))
return NewOp;
}
} }
// We can always fall back to a non-immediate AND. // We can always fall back to a non-immediate AND.
FailedModImm:
return Op; return Op;
} }
// Specialized code to quickly find if PotentialBVec is a BuildVector that // Specialized code to quickly find if PotentialBVec is a BuildVector that
// consists of only the same constant int value, returned in reference arg // consists of only the same constant int value, returned in reference arg
// ConstVal // ConstVal
static bool isAllConstantBuildVector(const SDValue &PotentialBVec, static bool isAllConstantBuildVector(const SDValue &PotentialBVec,
uint64_t &ConstVal) { uint64_t &ConstVal) {
▲ Show 20 Lines • Show All 94 Lines • ▼ Show 20 Lines
SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op, SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
SelectionDAG &DAG) const { SelectionDAG &DAG) const {
// Attempt to form a vector S[LR]I from (or (and X, C1), (lsl Y, C2)) // Attempt to form a vector S[LR]I from (or (and X, C1), (lsl Y, C2))
if (EnableAArch64SlrGeneration) { if (EnableAArch64SlrGeneration) {
if (SDValue Res = tryLowerToSLI(Op.getNode(), DAG)) if (SDValue Res = tryLowerToSLI(Op.getNode(), DAG))
return Res; return Res;
} }
BuildVectorSDNode *BVN = SDValue LHS = Op.getOperand(0);
dyn_cast<BuildVectorSDNode>(Op.getOperand(0).getNode());
SDValue LHS = Op.getOperand(1);
SDLoc dl(Op);
EVT VT = Op.getValueType(); EVT VT = Op.getValueType();
// OR commutes, so try swapping the operands. BuildVectorSDNode *BVN =
dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode());
if (!BVN) { if (!BVN) {
LHS = Op.getOperand(0); // OR commutes, so try swapping the operands.
BVN = dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode()); LHS = Op.getOperand(1);
BVN = dyn_cast<BuildVectorSDNode>(Op.getOperand(0).getNode());
} }
if (!BVN) if (!BVN)
return Op; return Op;
APInt CnstBits(VT.getSizeInBits(), 0); APInt DefBits(VT.getSizeInBits(), 0);
APInt UndefBits(VT.getSizeInBits(), 0); APInt UndefBits(VT.getSizeInBits(), 0);
if (resolveBuildVector(BVN, CnstBits, UndefBits)) { if (resolveBuildVector(BVN, DefBits, UndefBits)) {
// We make use of a little bit of goto ickiness in order to avoid having to SDValue NewOp;
// duplicate the immediate matching logic for the undef toggled case.
bool SecondTry = false;
AttemptModImm:
if (CnstBits.getHiBits(64) == CnstBits.getLoBits(64)) {
CnstBits = CnstBits.zextOrTrunc(64);
uint64_t CnstVal = CnstBits.getZExtValue();
if (AArch64_AM::isAdvSIMDModImmType1(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType1(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(0, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType2(CnstVal)) { if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::ORRi, Op, DAG,
CnstVal = AArch64_AM::encodeAdvSIMDModImmType2(CnstVal); DefBits, &LHS)) ||
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; (NewOp = tryAdvSIMDModImm16(AArch64ISD::ORRi, Op, DAG,
SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS, DefBits, &LHS)))
DAG.getConstant(CnstVal, dl, MVT::i32), return NewOp;
DAG.getConstant(8, dl, MVT::i32)); else {
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); DefBits = UndefBits;
} if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::ORRi, Op, DAG,
DefBits, &LHS)) ||
if (AArch64_AM::isAdvSIMDModImmType3(CnstVal)) { (NewOp = tryAdvSIMDModImm16(AArch64ISD::ORRi, Op, DAG,
CnstVal = AArch64_AM::encodeAdvSIMDModImmType3(CnstVal); DefBits, &LHS)))
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; return NewOp;
SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(16, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType4(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType4(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(24, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType5(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType5(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(0, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType6(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType6(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(8, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
} }
if (SecondTry)
goto FailedModImm;
SecondTry = true;
CnstBits = UndefBits;
goto AttemptModImm;
} }
// We can always fall back to a non-immediate OR. // We can always fall back to a non-immediate OR.
FailedModImm:
return Op; return Op;
} }
// Normalize the operands of BUILD_VECTOR. The value of constant operands will // Normalize the operands of BUILD_VECTOR. The value of constant operands will
// be truncated to fit element width. // be truncated to fit element width.
static SDValue NormalizeBuildVector(SDValue Op, static SDValue NormalizeBuildVector(SDValue Op,
SelectionDAG &DAG) { SelectionDAG &DAG) {
assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!"); assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
Show All 15 Linesstatic SDValue NormalizeBuildVector(SDValue Op,
} }
return DAG.getBuildVector(VT, dl, Ops); return DAG.getBuildVector(VT, dl, Ops);
} }
SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op, SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
SelectionDAG &DAG) const { SelectionDAG &DAG) const {
SDLoc dl(Op); SDLoc dl(Op);
EVT VT = Op.getValueType(); EVT VT = Op.getValueType();
Op = NormalizeBuildVector(Op, DAG); Op = NormalizeBuildVector(Op, DAG);
BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode()); BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode());
APInt DefBits(VT.getSizeInBits(), 0);
APInt CnstBits(VT.getSizeInBits(), 0);
APInt UndefBits(VT.getSizeInBits(), 0); APInt UndefBits(VT.getSizeInBits(), 0);
if (resolveBuildVector(BVN, CnstBits, UndefBits)) { if (resolveBuildVector(BVN, DefBits, UndefBits)) {
// We make use of a little bit of goto ickiness in order to avoid having to
// duplicate the immediate matching logic for the undef toggled case.
bool SecondTry = false;
AttemptModImm:
if (CnstBits.getHiBits(64) == CnstBits.getLoBits(64)) {
CnstBits = CnstBits.zextOrTrunc(64);
uint64_t CnstVal = CnstBits.getZExtValue();
// Certain magic vector constants (used to express things like NOT // Certain magic vector constants (used to express things like NOT
// and NEG) are passed through unmodified. This allows codegen patterns // and NEG) are passed through unmodified. This allows codegen patterns
// for these operations to match. Special-purpose patterns will lower // for these operations to match. Special-purpose patterns will lower
// these immediates to MOVIs if it proves necessary. // these immediates to MOVI if it proves necessary.
if (VT.isInteger() && (CnstVal == 0 || CnstVal == ~0ULL)) uint64_t DefVal = DefBits.zextOrTrunc(64).getZExtValue();
if (DefBits.getHiBits(64) == DefBits.getLoBits(64) &&
VT.isInteger() && (DefVal == 0 || DefVal == UINT64_MAX))
return Op; return Op;
// The many faces of MOVI... SDValue NewOp;
if (AArch64_AM::isAdvSIMDModImmType10(CnstVal)) { if ((NewOp = tryAdvSIMDModImm64(AArch64ISD::MOVIedit, Op, DAG, DefBits)) ||
CnstVal = AArch64_AM::encodeAdvSIMDModImmType10(CnstVal); (NewOp = tryAdvSIMDModImm32(AArch64ISD::MOVIshift, Op, DAG, DefBits)) ||
if (VT.getSizeInBits() == 128) { (NewOp = tryAdvSIMDModImm16(AArch64ISD::MOVIshift, Op, DAG, DefBits)) ||
SDValue Mov = DAG.getNode(AArch64ISD::MOVIedit, dl, MVT::v2i64, (NewOp = tryAdvSIMDModImm321s(AArch64ISD::MOVImsl, Op, DAG, DefBits)) ||
DAG.getConstant(CnstVal, dl, MVT::i32)); (NewOp = tryAdvSIMDModImm8(AArch64ISD::MOVI, Op, DAG, DefBits)) ||
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); (NewOp = tryAdvSIMDModImmFP(AArch64ISD::FMOV, Op, DAG, DefBits)))
} return NewOp;
// Support the V64 version via subregister insertion. DefBits = ~DefBits;
SDValue Mov = DAG.getNode(AArch64ISD::MOVIedit, dl, MVT::f64, if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::MVNIshift, Op, DAG, DefBits)) ||
DAG.getConstant(CnstVal, dl, MVT::i32)); (NewOp = tryAdvSIMDModImm16(AArch64ISD::MVNIshift, Op, DAG, DefBits)) ||
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); (NewOp = tryAdvSIMDModImm321s(AArch64ISD::MVNImsl, Op, DAG, DefBits)))
} return NewOp;
if (AArch64_AM::isAdvSIMDModImmType1(CnstVal)) { DefBits = UndefBits;
CnstVal = AArch64_AM::encodeAdvSIMDModImmType1(CnstVal); if ((NewOp = tryAdvSIMDModImm64(AArch64ISD::MOVIedit, Op, DAG, DefBits)) ||
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; (NewOp = tryAdvSIMDModImm32(AArch64ISD::MOVIshift, Op, DAG, DefBits)) ||
SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy, (NewOp = tryAdvSIMDModImm16(AArch64ISD::MOVIshift, Op, DAG, DefBits)) ||
DAG.getConstant(CnstVal, dl, MVT::i32), (NewOp = tryAdvSIMDModImm321s(AArch64ISD::MOVImsl, Op, DAG, DefBits)) ||
DAG.getConstant(0, dl, MVT::i32)); (NewOp = tryAdvSIMDModImm8(AArch64ISD::MOVI, Op, DAG, DefBits)) ||
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); (NewOp = tryAdvSIMDModImmFP(AArch64ISD::FMOV, Op, DAG, DefBits)))
} return NewOp;
if (AArch64_AM::isAdvSIMDModImmType2(CnstVal)) { DefBits = ~UndefBits;
CnstVal = AArch64_AM::encodeAdvSIMDModImmType2(CnstVal); if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::MVNIshift, Op, DAG, DefBits)) ||
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; (NewOp = tryAdvSIMDModImm16(AArch64ISD::MVNIshift, Op, DAG, DefBits)) ||
SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy, (NewOp = tryAdvSIMDModImm321s(AArch64ISD::MVNImsl, Op, DAG, DefBits)))
DAG.getConstant(CnstVal, dl, MVT::i32), return NewOp;
DAG.getConstant(8, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType3(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType3(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(16, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType4(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType4(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(24, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType5(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType5(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(0, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType6(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType6(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(8, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType7(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType7(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
SDValue Mov = DAG.getNode(AArch64ISD::MOVImsl, dl, MovTy,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(264, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType8(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType8(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
SDValue Mov = DAG.getNode(AArch64ISD::MOVImsl, dl, MovTy,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(272, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType9(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType9(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v16i8 : MVT::v8i8;
SDValue Mov = DAG.getNode(AArch64ISD::MOVI, dl, MovTy,
DAG.getConstant(CnstVal, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
// The few faces of FMOV...
if (AArch64_AM::isAdvSIMDModImmType11(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType11(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4f32 : MVT::v2f32;
SDValue Mov = DAG.getNode(AArch64ISD::FMOV, dl, MovTy,
DAG.getConstant(CnstVal, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType12(CnstVal) &&
VT.getSizeInBits() == 128) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType12(CnstVal);
SDValue Mov = DAG.getNode(AArch64ISD::FMOV, dl, MVT::v2f64,
DAG.getConstant(CnstVal, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
// The many faces of MVNI...
CnstVal = ~CnstVal;
if (AArch64_AM::isAdvSIMDModImmType1(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType1(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(0, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType2(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType2(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(8, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType3(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType3(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(16, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType4(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType4(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(24, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType5(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType5(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(0, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType6(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType6(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(8, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType7(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType7(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
SDValue Mov = DAG.getNode(AArch64ISD::MVNImsl, dl, MovTy,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(264, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
if (AArch64_AM::isAdvSIMDModImmType8(CnstVal)) {
CnstVal = AArch64_AM::encodeAdvSIMDModImmType8(CnstVal);
MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
SDValue Mov = DAG.getNode(AArch64ISD::MVNImsl, dl, MovTy,
DAG.getConstant(CnstVal, dl, MVT::i32),
DAG.getConstant(272, dl, MVT::i32));
return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
}
}
if (SecondTry)
goto FailedModImm;
SecondTry = true;
CnstBits = UndefBits;
goto AttemptModImm;
} }
FailedModImm:
// Scan through the operands to find some interesting properties we can // Scan through the operands to find some interesting properties we can
// exploit: // exploit:
// 1) If only one value is used, we can use a DUP, or // 1) If only one value is used, we can use a DUP, or
// 2) if only the low element is not undef, we can just insert that, or // 2) if only the low element is not undef, we can just insert that, or
// 3) if only one constant value is used (w/ some non-constant lanes), // 3) if only one constant value is used (w/ some non-constant lanes),
// we can splat the constant value into the whole vector then fill // we can splat the constant value into the whole vector then fill
// in the non-constant lanes. // in the non-constant lanes.
▲ Show 20 Lines • Show All 4,283 Lines • Show Last 20 Lines

llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll

Show First 20 Lines • Show All 531 Lines • ▼ Show 20 Lines; CHECK: bsl {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
%tmp2 = and <4 x i16> %b, < i16 0, i16 -1,i16 0, i16 -1 > %tmp2 = and <4 x i16> %b, < i16 0, i16 -1,i16 0, i16 -1 >
%tmp3 = or <4 x i16> %tmp1, %tmp2 %tmp3 = or <4 x i16> %tmp1, %tmp2
ret <4 x i16> %tmp3 ret <4 x i16> %tmp3
} }
define <1 x i64> @bsl1xi64_const(<1 x i64> %a, <1 x i64> %b) { define <1 x i64> @bsl1xi64_const(<1 x i64> %a, <1 x i64> %b) {
; CHECK-LABEL: bsl1xi64_const: ; CHECK-LABEL: bsl1xi64_const:
; CHECK: bsl {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b ; CHECK: bsl {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
%tmp1 = and <1 x i64> %a, < i64 -16 > %tmp1 = and <1 x i64> %a, < i64 -256 >
SjoerdMeijerUnsubmitted
Done
ReplyInline Actions

drive-by comment: why are you changing the test?

The clean up looks very reasonable to me, but it's a big patch,
so I need some more time to digest the rest.

SjoerdMeijer: drive-by comment: why are you changing the test? The clean up looks very reasonable to me, but…
evandroAuthorUnsubmitted
Not Done
ReplyInline Actions

Oops, it's a git leak. 🙂

evandro: Oops, it's a git leak. 🙂
%tmp2 = and <1 x i64> %b, < i64 15 > %tmp2 = and <1 x i64> %b, < i64 255 >
%tmp3 = or <1 x i64> %tmp1, %tmp2 %tmp3 = or <1 x i64> %tmp1, %tmp2
ret <1 x i64> %tmp3 ret <1 x i64> %tmp3
} }
define <4 x i32> @bsl4xi32_const(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @bsl4xi32_const(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: bsl4xi32_const: ; CHECK-LABEL: bsl4xi32_const:
; CHECK: bsl {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b ; CHECK: bsl {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b
%tmp1 = and <4 x i32> %a, < i32 -1, i32 0, i32 -1, i32 0 > %tmp1 = and <4 x i32> %a, < i32 -1, i32 0, i32 -1, i32 0 >
▲ Show 20 Lines • Show All 701 Lines • Show Last 20 Lines