Index: llvm/trunk/include/llvm/CodeGen/GlobalISel/IRTranslator.h =================================================================== --- llvm/trunk/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ llvm/trunk/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -399,6 +399,9 @@ bool translateShuffleVector(const User &U, MachineIRBuilder &MIRBuilder); + bool translateAtomicCmpXchg(const User &U, MachineIRBuilder &MIRBuilder); + bool translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder); + // Stubs to keep the compiler happy while we implement the rest of the // translation. bool translateResume(const User &U, MachineIRBuilder &MIRBuilder) { @@ -416,12 +419,6 @@ bool translateFence(const User &U, MachineIRBuilder &MIRBuilder) { return false; } - bool translateAtomicCmpXchg(const User &U, MachineIRBuilder &MIRBuilder) { - return false; - } - bool translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder) { - return false; - } bool translateAddrSpaceCast(const User &U, MachineIRBuilder &MIRBuilder) { return translateCast(TargetOpcode::G_ADDRSPACE_CAST, U, MIRBuilder); } Index: llvm/trunk/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h =================================================================== --- llvm/trunk/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ llvm/trunk/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -717,7 +717,28 @@ MachineInstrBuilder buildExtractVectorElement(unsigned Res, unsigned Val, unsigned Idx); - /// Build and insert `OldValRes = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal, + /// Build and insert `OldValRes, SuccessRes = + /// G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr, CmpVal, NewVal, MMO`. + /// + /// Atomically replace the value at \p Addr with \p NewVal if it is currently + /// \p CmpVal otherwise leaves it unchanged. Puts the original value from \p + /// Addr in \p Res, along with an s1 indicating whether it was replaced. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register of scalar type. + /// \pre \p SuccessRes must be a generic virtual register of scalar type. It + /// will be assigned 0 on failure and 1 on success. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, \p CmpVal, and \p NewVal must be generic virtual + /// registers of the same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder + buildAtomicCmpXchgWithSuccess(unsigned OldValRes, unsigned SuccessRes, + unsigned Addr, unsigned CmpVal, unsigned NewVal, + MachineMemOperand &MMO); + + /// Build and insert `OldValRes = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal, /// MMO`. /// /// Atomically replace the value at \p Addr with \p NewVal if it is currently @@ -734,6 +755,193 @@ MachineInstrBuilder buildAtomicCmpXchg(unsigned OldValRes, unsigned Addr, unsigned CmpVal, unsigned NewVal, MachineMemOperand &MMO); + + /// Build and insert `OldValRes = G_ATOMICRMW_ Addr, Val, MMO`. + /// + /// Atomically read-modify-update the value at \p Addr with \p Val. Puts the + /// original value from \p Addr in \p OldValRes. The modification is + /// determined by the opcode. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMW(unsigned Opcode, unsigned OldValRes, + unsigned Addr, unsigned Val, + MachineMemOperand &MMO); + + /// Build and insert `OldValRes = G_ATOMICRMW_XCHG Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with \p Val. Puts the original + /// value from \p Addr in \p OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWXchg(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes = G_ATOMICRMW_ADD Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the addition of \p Val and + /// the original value. Puts the original value from \p Addr in \p OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWAdd(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes = G_ATOMICRMW_SUB Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the subtraction of \p Val and + /// the original value. Puts the original value from \p Addr in \p OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWSub(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes = G_ATOMICRMW_AND Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the bitwise and of \p Val and + /// the original value. Puts the original value from \p Addr in \p OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWAnd(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes = G_ATOMICRMW_NAND Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the bitwise nand of \p Val + /// and the original value. Puts the original value from \p Addr in \p + /// OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWNand(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes = G_ATOMICRMW_OR Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the bitwise or of \p Val and + /// the original value. Puts the original value from \p Addr in \p OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWOr(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes = G_ATOMICRMW_XOR Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the bitwise xor of \p Val and + /// the original value. Puts the original value from \p Addr in \p OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWXor(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes = G_ATOMICRMW_MAX Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the signed maximum of \p + /// Val and the original value. Puts the original value from \p Addr in \p + /// OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWMax(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes = G_ATOMICRMW_MIN Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the signed minimum of \p + /// Val and the original value. Puts the original value from \p Addr in \p + /// OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWMin(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes = G_ATOMICRMW_UMAX Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the unsigned maximum of \p + /// Val and the original value. Puts the original value from \p Addr in \p + /// OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWUmax(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes = G_ATOMICRMW_UMIN Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the unsigned minimum of \p + /// Val and the original value. Puts the original value from \p Addr in \p + /// OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWUmin(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); }; /// A CRTP class that contains methods for building instructions that can Index: llvm/trunk/lib/CodeGen/GlobalISel/IRTranslator.cpp =================================================================== --- llvm/trunk/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ llvm/trunk/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -228,6 +228,20 @@ } else if (const LoadInst *LI = dyn_cast(&I)) { Alignment = LI->getAlignment(); ValTy = LI->getType(); + } else if (const AtomicCmpXchgInst *AI = dyn_cast(&I)) { + // TODO(PR27168): This instruction has no alignment attribute, but unlike + // the default alignment for load/store, the default here is to assume + // it has NATURAL alignment, not DataLayout-specified alignment. + const DataLayout &DL = AI->getModule()->getDataLayout(); + Alignment = DL.getTypeStoreSize(AI->getCompareOperand()->getType()); + ValTy = AI->getCompareOperand()->getType(); + } else if (const AtomicRMWInst *AI = dyn_cast(&I)) { + // TODO(PR27168): This instruction has no alignment attribute, but unlike + // the default alignment for load/store, the default here is to assume + // it has NATURAL alignment, not DataLayout-specified alignment. + const DataLayout &DL = AI->getModule()->getDataLayout(); + Alignment = DL.getTypeStoreSize(AI->getValOperand()->getType()); + ValTy = AI->getType(); } else { OptimizationRemarkMissed R("gisel-irtranslator", "", &I); R << "unable to translate memop: " << ore::NV("Opcode", &I); @@ -1285,6 +1299,100 @@ return true; } +bool IRTranslator::translateAtomicCmpXchg(const User &U, + MachineIRBuilder &MIRBuilder) { + const AtomicCmpXchgInst &I = cast(U); + + if (I.isWeak()) + return false; + + auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile + : MachineMemOperand::MONone; + Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore; + + Type *ResType = I.getType(); + Type *ValType = ResType->Type::getStructElementType(0); + + auto Res = getOrCreateVRegs(I); + unsigned OldValRes = Res[0]; + unsigned SuccessRes = Res[1]; + unsigned Addr = getOrCreateVReg(*I.getPointerOperand()); + unsigned Cmp = getOrCreateVReg(*I.getCompareOperand()); + unsigned NewVal = getOrCreateVReg(*I.getNewValOperand()); + + MIRBuilder.buildAtomicCmpXchgWithSuccess( + OldValRes, SuccessRes, Addr, Cmp, NewVal, + *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), + Flags, DL->getTypeStoreSize(ValType), + getMemOpAlignment(I), AAMDNodes(), nullptr, + I.getSyncScopeID(), I.getSuccessOrdering(), + I.getFailureOrdering())); + return true; +} + +bool IRTranslator::translateAtomicRMW(const User &U, + MachineIRBuilder &MIRBuilder) { + const AtomicRMWInst &I = cast(U); + + auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile + : MachineMemOperand::MONone; + Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore; + + Type *ResType = I.getType(); + + unsigned Res = getOrCreateVReg(I); + unsigned Addr = getOrCreateVReg(*I.getPointerOperand()); + unsigned Val = getOrCreateVReg(*I.getValOperand()); + + unsigned Opcode = 0; + switch (I.getOperation()) { + default: + llvm_unreachable("Unknown atomicrmw op"); + return false; + case AtomicRMWInst::Xchg: + Opcode = TargetOpcode::G_ATOMICRMW_XCHG; + break; + case AtomicRMWInst::Add: + Opcode = TargetOpcode::G_ATOMICRMW_ADD; + break; + case AtomicRMWInst::Sub: + Opcode = TargetOpcode::G_ATOMICRMW_SUB; + break; + case AtomicRMWInst::And: + Opcode = TargetOpcode::G_ATOMICRMW_AND; + break; + case AtomicRMWInst::Nand: + Opcode = TargetOpcode::G_ATOMICRMW_NAND; + break; + case AtomicRMWInst::Or: + Opcode = TargetOpcode::G_ATOMICRMW_OR; + break; + case AtomicRMWInst::Xor: + Opcode = TargetOpcode::G_ATOMICRMW_XOR; + break; + case AtomicRMWInst::Max: + Opcode = TargetOpcode::G_ATOMICRMW_MAX; + break; + case AtomicRMWInst::Min: + Opcode = TargetOpcode::G_ATOMICRMW_MIN; + break; + case AtomicRMWInst::UMax: + Opcode = TargetOpcode::G_ATOMICRMW_UMAX; + break; + case AtomicRMWInst::UMin: + Opcode = TargetOpcode::G_ATOMICRMW_UMIN; + break; + } + + MIRBuilder.buildAtomicRMW( + Opcode, Res, Addr, Val, + *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), + Flags, DL->getTypeStoreSize(ResType), + getMemOpAlignment(I), AAMDNodes(), nullptr, + I.getSyncScopeID(), I.getOrdering())); + return true; +} + void IRTranslator::finishPendingPhis() { for (auto &Phi : PendingPHIs) { const PHINode *PI = Phi.first; Index: llvm/trunk/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp =================================================================== --- llvm/trunk/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ llvm/trunk/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -669,6 +669,33 @@ .addUse(Idx); } +MachineInstrBuilder MachineIRBuilderBase::buildAtomicCmpXchgWithSuccess( + unsigned OldValRes, unsigned SuccessRes, unsigned Addr, unsigned CmpVal, + unsigned NewVal, MachineMemOperand &MMO) { +#ifndef NDEBUG + LLT OldValResTy = getMRI()->getType(OldValRes); + LLT SuccessResTy = getMRI()->getType(SuccessRes); + LLT AddrTy = getMRI()->getType(Addr); + LLT CmpValTy = getMRI()->getType(CmpVal); + LLT NewValTy = getMRI()->getType(NewVal); + assert(OldValResTy.isScalar() && "invalid operand type"); + assert(SuccessResTy.isScalar() && "invalid operand type"); + assert(AddrTy.isPointer() && "invalid operand type"); + assert(CmpValTy.isValid() && "invalid operand type"); + assert(NewValTy.isValid() && "invalid operand type"); + assert(OldValResTy == CmpValTy && "type mismatch"); + assert(OldValResTy == NewValTy && "type mismatch"); +#endif + + return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS) + .addDef(OldValRes) + .addDef(SuccessRes) + .addUse(Addr) + .addUse(CmpVal) + .addUse(NewVal) + .addMemOperand(&MMO); +} + MachineInstrBuilder MachineIRBuilderBase::buildAtomicCmpXchg(unsigned OldValRes, unsigned Addr, unsigned CmpVal, unsigned NewVal, @@ -694,6 +721,94 @@ .addMemOperand(&MMO); } +MachineInstrBuilder +MachineIRBuilderBase::buildAtomicRMW(unsigned Opcode, unsigned OldValRes, + unsigned Addr, unsigned Val, + MachineMemOperand &MMO) { +#ifndef NDEBUG + LLT OldValResTy = getMRI()->getType(OldValRes); + LLT AddrTy = getMRI()->getType(Addr); + LLT ValTy = getMRI()->getType(Val); + assert(OldValResTy.isScalar() && "invalid operand type"); + assert(AddrTy.isPointer() && "invalid operand type"); + assert(ValTy.isValid() && "invalid operand type"); + assert(OldValResTy == ValTy && "type mismatch"); +#endif + + return buildInstr(Opcode) + .addDef(OldValRes) + .addUse(Addr) + .addUse(Val) + .addMemOperand(&MMO); +} + +MachineInstrBuilder +MachineIRBuilderBase::buildAtomicRMWXchg(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO) { + return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XCHG, OldValRes, Addr, Val, + MMO); +} +MachineInstrBuilder +MachineIRBuilderBase::buildAtomicRMWAdd(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO) { + return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_ADD, OldValRes, Addr, Val, + MMO); +} +MachineInstrBuilder +MachineIRBuilderBase::buildAtomicRMWSub(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO) { + return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_SUB, OldValRes, Addr, Val, + MMO); +} +MachineInstrBuilder +MachineIRBuilderBase::buildAtomicRMWAnd(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO) { + return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_AND, OldValRes, Addr, Val, + MMO); +} +MachineInstrBuilder +MachineIRBuilderBase::buildAtomicRMWNand(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO) { + return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_NAND, OldValRes, Addr, Val, + MMO); +} +MachineInstrBuilder +MachineIRBuilderBase::buildAtomicRMWOr(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO) { + return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_OR, OldValRes, Addr, Val, + MMO); +} +MachineInstrBuilder +MachineIRBuilderBase::buildAtomicRMWXor(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO) { + return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XOR, OldValRes, Addr, Val, + MMO); +} +MachineInstrBuilder +MachineIRBuilderBase::buildAtomicRMWMax(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO) { + return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MAX, OldValRes, Addr, Val, + MMO); +} +MachineInstrBuilder +MachineIRBuilderBase::buildAtomicRMWMin(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO) { + return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MIN, OldValRes, Addr, Val, + MMO); +} +MachineInstrBuilder +MachineIRBuilderBase::buildAtomicRMWUmax(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO) { + return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMAX, OldValRes, Addr, Val, + MMO); +} +MachineInstrBuilder +MachineIRBuilderBase::buildAtomicRMWUmin(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO) { + return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMIN, OldValRes, Addr, Val, + MMO); +} + void MachineIRBuilderBase::validateTruncExt(unsigned Dst, unsigned Src, bool IsExtend) { #ifndef NDEBUG @@ -701,7 +816,7 @@ LLT DstTy = getMRI()->getType(Dst); if (DstTy.isVector()) { - assert(SrcTy.isVector() && "mismatched cast between vecot and non-vector"); + assert(SrcTy.isVector() && "mismatched cast between vector and non-vector"); assert(SrcTy.getNumElements() == DstTy.getNumElements() && "different number of elements in a trunc/ext"); } else Index: llvm/trunk/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll =================================================================== --- llvm/trunk/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll +++ llvm/trunk/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll @@ -1882,3 +1882,268 @@ ; CHECK: RET_ReallyLR implicit $w0 ret i1 true } + +; Try one cmpxchg +define i32 @test_atomic_cmpxchg_1(i32* %addr) { +; CHECK-LABEL: name: test_atomic_cmpxchg_1 +; CHECK: bb.1.entry: +; CHECK-NEXT: successors: %bb.{{[^)]+}} +; CHECK-NEXT: liveins: $x0 +; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 +; CHECK-NEXT: [[OLDVAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 +; CHECK-NEXT: [[NEWVAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK: bb.2.repeat: +; CHECK-NEXT: successors: %bb.3({{[^)]+}}), %bb.2({{[^)]+}}) +; CHECK: [[OLDVALRES:%[0-9]+]]:_(s32), [[SUCCESS:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[ADDR]](p0), [[OLDVAL]], [[NEWVAL]] :: (load store monotonic monotonic 4 on %ir.addr) +; CHECK-NEXT: G_BRCOND [[SUCCESS]](s1), %bb.3 +; CHECK-NEXT: G_BR %bb.2 +; CHECK: bb.3.done: +entry: + br label %repeat +repeat: + %val_success = cmpxchg i32* %addr, i32 0, i32 1 monotonic monotonic + %value_loaded = extractvalue { i32, i1 } %val_success, 0 + %success = extractvalue { i32, i1 } %val_success, 1 + br i1 %success, label %done, label %repeat +done: + ret i32 %value_loaded +} + +; Try one cmpxchg with a small type and high atomic ordering. +define i16 @test_atomic_cmpxchg_2(i16* %addr) { +; CHECK-LABEL: name: test_atomic_cmpxchg_2 +; CHECK: bb.1.entry: +; CHECK-NEXT: successors: %bb.2({{[^)]+}}) +; CHECK-NEXT: liveins: $x0 +; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 +; CHECK-NEXT: [[OLDVAL:%[0-9]+]]:_(s16) = G_CONSTANT i16 0 +; CHECK-NEXT: [[NEWVAL:%[0-9]+]]:_(s16) = G_CONSTANT i16 1 +; CHECK: bb.2.repeat: +; CHECK-NEXT: successors: %bb.3({{[^)]+}}), %bb.2({{[^)]+}}) +; CHECK: [[OLDVALRES:%[0-9]+]]:_(s16), [[SUCCESS:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[ADDR]](p0), [[OLDVAL]], [[NEWVAL]] :: (load store seq_cst seq_cst 2 on %ir.addr) +; CHECK-NEXT: G_BRCOND [[SUCCESS]](s1), %bb.3 +; CHECK-NEXT: G_BR %bb.2 +; CHECK: bb.3.done: +entry: + br label %repeat +repeat: + %val_success = cmpxchg i16* %addr, i16 0, i16 1 seq_cst seq_cst + %value_loaded = extractvalue { i16, i1 } %val_success, 0 + %success = extractvalue { i16, i1 } %val_success, 1 + br i1 %success, label %done, label %repeat +done: + ret i16 %value_loaded +} + +; Try one cmpxchg where the success order and failure order differ. +define i64 @test_atomic_cmpxchg_3(i64* %addr) { +; CHECK-LABEL: name: test_atomic_cmpxchg_3 +; CHECK: bb.1.entry: +; CHECK-NEXT: successors: %bb.2({{[^)]+}}) +; CHECK-NEXT: liveins: $x0 +; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 +; CHECK-NEXT: [[OLDVAL:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 +; CHECK-NEXT: [[NEWVAL:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 +; CHECK: bb.2.repeat: +; CHECK-NEXT: successors: %bb.3({{[^)]+}}), %bb.2({{[^)]+}}) +; CHECK: [[OLDVALRES:%[0-9]+]]:_(s64), [[SUCCESS:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[ADDR]](p0), [[OLDVAL]], [[NEWVAL]] :: (load store seq_cst acquire 8 on %ir.addr) +; CHECK-NEXT: G_BRCOND [[SUCCESS]](s1), %bb.3 +; CHECK-NEXT: G_BR %bb.2 +; CHECK: bb.3.done: +entry: + br label %repeat +repeat: + %val_success = cmpxchg i64* %addr, i64 0, i64 1 seq_cst acquire + %value_loaded = extractvalue { i64, i1 } %val_success, 0 + %success = extractvalue { i64, i1 } %val_success, 1 + br i1 %success, label %done, label %repeat +done: + ret i64 %value_loaded +} + +; Try a monotonic atomicrmw xchg +; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. +define i32 @test_atomicrmw_xchg(i256* %addr) { +; CHECK-LABEL: name: test_atomicrmw_xchg +; CHECK: bb.1 (%ir-block.{{[0-9]+}}): +; CHECK-NEXT: liveins: $x0 +; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_XCHG [[ADDR]](p0), [[VAL]] :: (load store monotonic 32 on %ir.addr) +; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] + %oldval = atomicrmw xchg i256* %addr, i256 1 monotonic + ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this + ; test so work around it by truncating to i32 for now. + %oldval.trunc = trunc i256 %oldval to i32 + ret i32 %oldval.trunc +} + +; Try an acquire atomicrmw add +; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. +define i32 @test_atomicrmw_add(i256* %addr) { +; CHECK-LABEL: name: test_atomicrmw_add +; CHECK: bb.1 (%ir-block.{{[0-9]+}}): +; CHECK-NEXT: liveins: $x0 +; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_ADD [[ADDR]](p0), [[VAL]] :: (load store acquire 32 on %ir.addr) +; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] + %oldval = atomicrmw add i256* %addr, i256 1 acquire + ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this + ; test so work around it by truncating to i32 for now. + %oldval.trunc = trunc i256 %oldval to i32 + ret i32 %oldval.trunc +} + +; Try a release atomicrmw sub +; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. +define i32 @test_atomicrmw_sub(i256* %addr) { +; CHECK-LABEL: name: test_atomicrmw_sub +; CHECK: bb.1 (%ir-block.{{[0-9]+}}): +; CHECK-NEXT: liveins: $x0 +; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_SUB [[ADDR]](p0), [[VAL]] :: (load store release 32 on %ir.addr) +; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] + %oldval = atomicrmw sub i256* %addr, i256 1 release + ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this + ; test so work around it by truncating to i32 for now. + %oldval.trunc = trunc i256 %oldval to i32 + ret i32 %oldval.trunc +} + +; Try an acq_rel atomicrmw and +; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. +define i32 @test_atomicrmw_and(i256* %addr) { +; CHECK-LABEL: name: test_atomicrmw_and +; CHECK: bb.1 (%ir-block.{{[0-9]+}}): +; CHECK-NEXT: liveins: $x0 +; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_AND [[ADDR]](p0), [[VAL]] :: (load store acq_rel 32 on %ir.addr) +; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] + %oldval = atomicrmw and i256* %addr, i256 1 acq_rel + ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this + ; test so work around it by truncating to i32 for now. + %oldval.trunc = trunc i256 %oldval to i32 + ret i32 %oldval.trunc +} + +; Try an seq_cst atomicrmw nand +; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. +define i32 @test_atomicrmw_nand(i256* %addr) { +; CHECK-LABEL: name: test_atomicrmw_nand +; CHECK: bb.1 (%ir-block.{{[0-9]+}}): +; CHECK-NEXT: liveins: $x0 +; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_NAND [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr) +; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] + %oldval = atomicrmw nand i256* %addr, i256 1 seq_cst + ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this + ; test so work around it by truncating to i32 for now. + %oldval.trunc = trunc i256 %oldval to i32 + ret i32 %oldval.trunc +} + +; Try an seq_cst atomicrmw or +; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. +define i32 @test_atomicrmw_or(i256* %addr) { +; CHECK-LABEL: name: test_atomicrmw_or +; CHECK: bb.1 (%ir-block.{{[0-9]+}}): +; CHECK-NEXT: liveins: $x0 +; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_OR [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr) +; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] + %oldval = atomicrmw or i256* %addr, i256 1 seq_cst + ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this + ; test so work around it by truncating to i32 for now. + %oldval.trunc = trunc i256 %oldval to i32 + ret i32 %oldval.trunc +} + +; Try an seq_cst atomicrmw xor +; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. +define i32 @test_atomicrmw_xor(i256* %addr) { +; CHECK-LABEL: name: test_atomicrmw_xor +; CHECK: bb.1 (%ir-block.{{[0-9]+}}): +; CHECK-NEXT: liveins: $x0 +; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_XOR [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr) +; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] + %oldval = atomicrmw xor i256* %addr, i256 1 seq_cst + ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this + ; test so work around it by truncating to i32 for now. + %oldval.trunc = trunc i256 %oldval to i32 + ret i32 %oldval.trunc +} + +; Try an seq_cst atomicrmw min +; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. +define i32 @test_atomicrmw_min(i256* %addr) { +; CHECK-LABEL: name: test_atomicrmw_min +; CHECK: bb.1 (%ir-block.{{[0-9]+}}): +; CHECK-NEXT: liveins: $x0 +; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_MIN [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr) +; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] + %oldval = atomicrmw min i256* %addr, i256 1 seq_cst + ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this + ; test so work around it by truncating to i32 for now. + %oldval.trunc = trunc i256 %oldval to i32 + ret i32 %oldval.trunc +} + +; Try an seq_cst atomicrmw max +; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. +define i32 @test_atomicrmw_max(i256* %addr) { +; CHECK-LABEL: name: test_atomicrmw_max +; CHECK: bb.1 (%ir-block.{{[0-9]+}}): +; CHECK-NEXT: liveins: $x0 +; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_MAX [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr) +; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] + %oldval = atomicrmw max i256* %addr, i256 1 seq_cst + ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this + ; test so work around it by truncating to i32 for now. + %oldval.trunc = trunc i256 %oldval to i32 + ret i32 %oldval.trunc +} + +; Try an seq_cst atomicrmw unsigned min +; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. +define i32 @test_atomicrmw_umin(i256* %addr) { +; CHECK-LABEL: name: test_atomicrmw_umin +; CHECK: bb.1 (%ir-block.{{[0-9]+}}): +; CHECK-NEXT: liveins: $x0 +; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_UMIN [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr) +; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] + %oldval = atomicrmw umin i256* %addr, i256 1 seq_cst + ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this + ; test so work around it by truncating to i32 for now. + %oldval.trunc = trunc i256 %oldval to i32 + ret i32 %oldval.trunc +} + +; Try an seq_cst atomicrmw unsigned max +; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. +define i32 @test_atomicrmw_umax(i256* %addr) { +; CHECK-LABEL: name: test_atomicrmw_umax +; CHECK: bb.1 (%ir-block.{{[0-9]+}}): +; CHECK-NEXT: liveins: $x0 +; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_UMAX [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr) +; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] + %oldval = atomicrmw umax i256* %addr, i256 1 seq_cst + ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this + ; test so work around it by truncating to i32 for now. + %oldval.trunc = trunc i256 %oldval to i32 + ret i32 %oldval.trunc +}