diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h --- a/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/llvm/include/llvm/CodeGen/MachineFunction.h @@ -796,6 +796,8 @@ /// Allocate and initialize a register mask with @p NumRegister bits. uint32_t *allocateRegMask(); + ArrayRef allocateShuffleMask(ArrayRef Mask); + /// Allocate and construct an extra info structure for a `MachineInstr`. /// /// This is allocated on the function's allocator and so lives the life of diff --git a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h --- a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h +++ b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h @@ -250,7 +250,7 @@ return *this; } - const MachineInstrBuilder &addShuffleMask(const Constant *Val) const { + const MachineInstrBuilder &addShuffleMask(ArrayRef Val) const { MI->addOperand(*MF, MachineOperand::CreateShuffleMask(Val)); return *this; } diff --git a/llvm/include/llvm/CodeGen/MachineOperand.h b/llvm/include/llvm/CodeGen/MachineOperand.h --- a/llvm/include/llvm/CodeGen/MachineOperand.h +++ b/llvm/include/llvm/CodeGen/MachineOperand.h @@ -163,7 +163,8 @@ MachineInstr *ParentMI; /// Contents union - This contains the payload for the various operand types. - union { + union ContentsUnion { + ContentsUnion() {} MachineBasicBlock *MBB; // For MO_MachineBasicBlock. const ConstantFP *CFP; // For MO_FPImmediate. const ConstantInt *CI; // For MO_CImmediate. Integers > 64bit. @@ -174,7 +175,7 @@ unsigned CFIIndex; // For MO_CFI. Intrinsic::ID IntrinsicID; // For MO_IntrinsicID. unsigned Pred; // For MO_Predicate - const Constant *ShuffleMask; // For MO_ShuffleMask + ArrayRef ShuffleMask; // For MO_ShuffleMask struct { // For MO_Register. // Register number is in SmallContents.RegNo. @@ -587,7 +588,7 @@ return Contents.Pred; } - const Constant *getShuffleMask() const { + ArrayRef getShuffleMask() const { assert(isShuffleMask() && "Wrong MachineOperand accessor"); return Contents.ShuffleMask; } @@ -915,9 +916,9 @@ return Op; } - static MachineOperand CreateShuffleMask(const Constant *C) { + static MachineOperand CreateShuffleMask(ArrayRef Mask) { MachineOperand Op(MachineOperand::MO_ShuffleMask); - Op.Contents.ShuffleMask = C; + Op.Contents.ShuffleMask = Mask; return Op; } diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -239,8 +239,7 @@ // vectors. unsigned NumConcat = DstNumElts / SrcNumElts; SmallVector ConcatSrcs(NumConcat, -1); - SmallVector Mask; - ShuffleVectorInst::getShuffleMask(MI.getOperand(3).getShuffleMask(), Mask); + ArrayRef Mask = MI.getOperand(3).getShuffleMask(); for (unsigned i = 0; i != DstNumElts; ++i) { int Idx = Mask[i]; // Undef value. diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1933,11 +1933,14 @@ bool IRTranslator::translateShuffleVector(const User &U, MachineIRBuilder &MIRBuilder) { + SmallVector Mask; + ShuffleVectorInst::getShuffleMask(cast(U.getOperand(2)), Mask); + ArrayRef MaskAlloc = MF->allocateShuffleMask(Mask); MIRBuilder.buildInstr(TargetOpcode::G_SHUFFLE_VECTOR) .addDef(getOrCreateVReg(U)) .addUse(getOrCreateVReg(*U.getOperand(0))) .addUse(getOrCreateVReg(*U.getOperand(1))) - .addShuffleMask(cast(U.getOperand(2))); + .addShuffleMask(MaskAlloc); return true; } diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -4202,10 +4202,7 @@ LLT DstTy = MRI.getType(DstReg); LLT IdxTy = LLT::scalar(32); - const Constant *ShufMask = MI.getOperand(3).getShuffleMask(); - - SmallVector Mask; - ShuffleVectorInst::getShuffleMask(ShufMask, Mask); + ArrayRef Mask = MI.getOperand(3).getShuffleMask(); if (DstTy.isScalar()) { if (Src0Ty.isVector()) diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp --- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp +++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp @@ -2421,23 +2421,13 @@ if (expectAndConsume(MIToken::lparen)) return error("expected syntax shufflemask(, ...)"); - SmallVector ShufMask; - LLVMContext &Ctx = MF.getFunction().getContext(); - Type *I32Ty = Type::getInt32Ty(Ctx); - - bool AllZero = true; - bool AllUndef = true; - + SmallVector ShufMask; do { if (Token.is(MIToken::kw_undef)) { - ShufMask.push_back(UndefValue::get(I32Ty)); - AllZero = false; + ShufMask.push_back(-1); } else if (Token.is(MIToken::IntegerLiteral)) { - AllUndef = false; const APSInt &Int = Token.integerValue(); - if (!Int.isNullValue()) - AllZero = false; - ShufMask.push_back(ConstantInt::get(I32Ty, Int.getExtValue())); + ShufMask.push_back(Int.getExtValue()); } else return error("expected integer constant"); @@ -2447,13 +2437,8 @@ if (expectAndConsume(MIToken::rparen)) return error("shufflemask should be terminated by ')'."); - if (AllZero || AllUndef) { - VectorType *VT = VectorType::get(I32Ty, ShufMask.size()); - Constant *C = AllZero ? Constant::getNullValue(VT) : UndefValue::get(VT); - Dest = MachineOperand::CreateShuffleMask(C); - } else - Dest = MachineOperand::CreateShuffleMask(ConstantVector::get(ShufMask)); - + ArrayRef MaskAlloc = MF.allocateShuffleMask(ShufMask); + Dest = MachineOperand::CreateShuffleMask(MaskAlloc); return false; } diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp --- a/llvm/lib/CodeGen/MachineFunction.cpp +++ b/llvm/lib/CodeGen/MachineFunction.cpp @@ -484,6 +484,12 @@ return Mask; } +ArrayRef MachineFunction::allocateShuffleMask(ArrayRef Mask) { + int* AllocMask = Allocator.Allocate(Mask.size()); + copy(Mask, AllocMask); + return {AllocMask, Mask.size()}; +} + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD void MachineFunction::dump() const { print(dbgs()); diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp --- a/llvm/lib/CodeGen/MachineOperand.cpp +++ b/llvm/lib/CodeGen/MachineOperand.cpp @@ -930,13 +930,13 @@ } case MachineOperand::MO_ShuffleMask: OS << "shufflemask("; - const Constant* C = getShuffleMask(); - const int NumElts = C->getType()->getVectorNumElements(); - + ArrayRef Mask = getShuffleMask(); StringRef Separator; - for (int I = 0; I != NumElts; ++I) { - OS << Separator; - C->getAggregateElement(I)->printAsOperand(OS, false, MST); + for (int Elt : Mask) { + if (Elt == -1) + OS << Separator << "undef"; + else + OS << Separator << Elt; Separator = ", "; } diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -1407,18 +1407,6 @@ break; } - const Constant *Mask = MaskOp.getShuffleMask(); - auto *MaskVT = dyn_cast(Mask->getType()); - if (!MaskVT || !MaskVT->getElementType()->isIntegerTy(32)) { - report("Invalid shufflemask constant type", MI); - break; - } - - if (!Mask->getAggregateElement(0u)) { - report("Invalid shufflemask constant type", MI); - break; - } - LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); LLT Src0Ty = MRI->getType(MI->getOperand(1).getReg()); LLT Src1Ty = MRI->getType(MI->getOperand(2).getReg()); @@ -1434,8 +1422,7 @@ int SrcNumElts = Src0Ty.isVector() ? Src0Ty.getNumElements() : 1; int DstNumElts = DstTy.isVector() ? DstTy.getNumElements() : 1; - SmallVector MaskIdxes; - ShuffleVectorInst::getShuffleMask(Mask, MaskIdxes); + ArrayRef MaskIdxes = MaskOp.getShuffleMask(); if (static_cast(MaskIdxes.size()) != DstNumElts) report("Wrong result type for shufflemask", MI); diff --git a/llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp --- a/llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp @@ -3699,8 +3699,8 @@ return false; // The shuffle's second operand doesn't matter if the mask is all zero. - const Constant *Mask = I.getOperand(3).getShuffleMask(); - if (!isa(Mask)) + ArrayRef Mask = I.getOperand(3).getShuffleMask(); + if (!all_of(Mask, [](int Elem) { return Elem == 0; })) return false; // We're done, now find out what kind of splat we need. @@ -3778,15 +3778,12 @@ const LLT Src1Ty = MRI.getType(Src1Reg); Register Src2Reg = I.getOperand(2).getReg(); const LLT Src2Ty = MRI.getType(Src2Reg); - const Constant *ShuffleMask = I.getOperand(3).getShuffleMask(); + ArrayRef Mask = I.getOperand(3).getShuffleMask(); MachineBasicBlock &MBB = *I.getParent(); MachineFunction &MF = *MBB.getParent(); LLVMContext &Ctx = MF.getFunction().getContext(); - SmallVector Mask; - ShuffleVectorInst::getShuffleMask(ShuffleMask, Mask); - // G_SHUFFLE_VECTOR is weird in that the source operands can be scalars, if // it's originated from a <1 x T> type. Those should have been lowered into // G_BUILD_VECTOR earlier. diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-shuffle-vector.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-shuffle-vector.mir --- a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-shuffle-vector.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-shuffle-vector.mir @@ -135,7 +135,7 @@ ; CHECK: liveins: $d0, $d1 ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0 ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1 - ; CHECK: [[SHUF:%[0-9]+]]:_(<6 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<2 x s32>), [[COPY1]], shufflemask(2, 3, 0, 1, 3, -1) + ; CHECK: [[SHUF:%[0-9]+]]:_(<6 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<2 x s32>), [[COPY1]], shufflemask(2, 3, 0, 1, 3, undef) ; CHECK: RET_ReallyLR implicit [[SHUF]](<6 x s32>) %0:_(<2 x s32>) = COPY $d0 %1:_(<2 x s32>) = COPY $d1 @@ -277,7 +277,7 @@ ; CHECK: liveins: $q0, $q1 ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1 - ; CHECK: [[SHUF:%[0-9]+]]:_(<12 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<4 x s32>), [[COPY1]], shufflemask(4, 5, 6, 7, 0, 1, 2, 3, 6, 7, -1, -1) + ; CHECK: [[SHUF:%[0-9]+]]:_(<12 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<4 x s32>), [[COPY1]], shufflemask(4, 5, 6, 7, 0, 1, 2, 3, 6, 7, undef, undef) ; CHECK: RET_ReallyLR implicit [[SHUF]](<12 x s32>) %0:_(<4 x s32>) = COPY $q0 %1:_(<4 x s32>) = COPY $q1