Index: llvm/include/llvm/CodeGen/TargetLowering.h =================================================================== --- llvm/include/llvm/CodeGen/TargetLowering.h +++ llvm/include/llvm/CodeGen/TargetLowering.h @@ -295,6 +295,8 @@ const DataLayout &DL) const; MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const; + MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, + const DataLayout &DL) const; virtual bool isSelectSupported(SelectSupportKind /*kind*/) const { return true; Index: llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp =================================================================== --- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1966,9 +1966,8 @@ if (I.isWeak()) return false; - auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile - : MachineMemOperand::MONone; - Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore; + auto &TLI = *MF->getSubtarget().getTargetLowering(); + auto Flags = TLI.getAtomicMemOperandFlags(I, *DL); Type *ResType = I.getType(); Type *ValType = ResType->Type::getStructElementType(0); @@ -1996,10 +1995,8 @@ bool IRTranslator::translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder) { const AtomicRMWInst &I = cast(U); - - auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile - : MachineMemOperand::MONone; - Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore; + auto &TLI = *MF->getSubtarget().getTargetLowering(); + auto Flags = TLI.getAtomicMemOperandFlags(I, *DL); Type *ResType = I.getType(); Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4599,11 +4599,8 @@ SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other); auto Alignment = DAG.getEVTAlignment(MemVT); - - auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; - if (I.isVolatile()) - Flags |= MachineMemOperand::MOVolatile; - Flags |= DAG.getTargetLoweringInfo().getTargetMMOFlags(I); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout()); MachineFunction &MF = DAG.getMachineFunction(); MachineMemOperand *MMO = @@ -4650,11 +4647,8 @@ auto MemVT = getValue(I.getValOperand()).getSimpleValueType(); auto Alignment = DAG.getEVTAlignment(MemVT); - - auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; - if (I.isVolatile()) - Flags |= MachineMemOperand::MOVolatile; - Flags |= DAG.getTargetLoweringInfo().getTargetMMOFlags(I); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout()); MachineFunction &MF = DAG.getMachineFunction(); MachineMemOperand *MMO = Index: llvm/lib/CodeGen/TargetLoweringBase.cpp =================================================================== --- llvm/lib/CodeGen/TargetLoweringBase.cpp +++ llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -2042,3 +2042,22 @@ Flags |= getTargetMMOFlags(SI); return Flags; } + +MachineMemOperand::Flags +TargetLoweringBase::getAtomicMemOperandFlags(const Instruction &AI, + const DataLayout &DL) const { + auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; + + if (const AtomicRMWInst *RMW = dyn_cast(&AI)) { + if (RMW->isVolatile()) + Flags |= MachineMemOperand::MOVolatile; + } else if (const AtomicCmpXchgInst *CmpX = dyn_cast(&AI)) { + if (CmpX->isVolatile()) + Flags |= MachineMemOperand::MOVolatile; + } else + llvm_unreachable("not an atomic instruction"); + + // FIXME: Not preserving dereferenceable + Flags |= getTargetMMOFlags(AI); + return Flags; +} Index: llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-atomic-metadata.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-atomic-metadata.ll @@ -0,0 +1,88 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +; RUN: llc -mtriple=aarch64-- -mcpu=falkor -mattr=+lse -O0 -aarch64-enable-atomic-cfg-tidy=0 -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - | FileCheck %s + +define i32 @atomicrmw_volatile(i32* %ptr) { + ; CHECK-LABEL: name: atomicrmw_volatile + ; CHECK: bb.1 (%ir-block.0): + ; CHECK: liveins: $x0 + ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[C]] :: (volatile load store monotonic 4 on %ir.ptr) + ; CHECK: $w0 = COPY [[ATOMICRMW_ADD]](s32) + ; CHECK: RET_ReallyLR implicit $w0 + %oldval = atomicrmw volatile add i32* %ptr, i32 1 monotonic + ret i32 %oldval +} + +define i32 @atomicrmw_falkor(i32* %ptr) { + ; CHECK-LABEL: name: atomicrmw_falkor + ; CHECK: bb.1 (%ir-block.0): + ; CHECK: liveins: $x0 + ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[C]] :: ("aarch64-strided-access" load store monotonic 4 on %ir.ptr) + ; CHECK: $w0 = COPY [[ATOMICRMW_ADD]](s32) + ; CHECK: RET_ReallyLR implicit $w0 + %oldval = atomicrmw add i32* %ptr, i32 1 monotonic, !falkor.strided.access !0 + ret i32 %oldval +} + +define i32 @atomicrmw_volatile_falkor(i32* %ptr) { + ; CHECK-LABEL: name: atomicrmw_volatile_falkor + ; CHECK: bb.1 (%ir-block.0): + ; CHECK: liveins: $x0 + ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[C]] :: (volatile "aarch64-strided-access" load store monotonic 4 on %ir.ptr) + ; CHECK: $w0 = COPY [[ATOMICRMW_ADD]](s32) + ; CHECK: RET_ReallyLR implicit $w0 + %oldval = atomicrmw volatile add i32* %ptr, i32 1 monotonic, !falkor.strided.access !0 + ret i32 %oldval +} + +define i32 @cmpxchg_volatile(i32* %addr) { + ; CHECK-LABEL: name: cmpxchg_volatile + ; CHECK: bb.1 (%ir-block.0): + ; CHECK: liveins: $x0 + ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 + ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; CHECK: [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(s32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p0), [[C]], [[C1]] :: (volatile load store monotonic monotonic 4 on %ir.addr) + ; CHECK: $w0 = COPY [[ATOMIC_CMPXCHG_WITH_SUCCESS]](s32) + ; CHECK: RET_ReallyLR implicit $w0 + %val_success = cmpxchg volatile i32* %addr, i32 0, i32 1 monotonic monotonic + %value_loaded = extractvalue { i32, i1 } %val_success, 0 + ret i32 %value_loaded +} + +define i32 @cmpxchg_falkor(i32* %addr) { + ; CHECK-LABEL: name: cmpxchg_falkor + ; CHECK: bb.1 (%ir-block.0): + ; CHECK: liveins: $x0 + ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 + ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; CHECK: [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(s32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p0), [[C]], [[C1]] :: ("aarch64-strided-access" load store monotonic monotonic 4 on %ir.addr) + ; CHECK: $w0 = COPY [[ATOMIC_CMPXCHG_WITH_SUCCESS]](s32) + ; CHECK: RET_ReallyLR implicit $w0 + %val_success = cmpxchg i32* %addr, i32 0, i32 1 monotonic monotonic, !falkor.strided.access !0 + %value_loaded = extractvalue { i32, i1 } %val_success, 0 + ret i32 %value_loaded +} + +define i32 @cmpxchg_volatile_falkor(i32* %addr) { + ; CHECK-LABEL: name: cmpxchg_volatile_falkor + ; CHECK: bb.1 (%ir-block.0): + ; CHECK: liveins: $x0 + ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 + ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; CHECK: [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(s32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p0), [[C]], [[C1]] :: (volatile "aarch64-strided-access" load store monotonic monotonic 4 on %ir.addr) + ; CHECK: $w0 = COPY [[ATOMIC_CMPXCHG_WITH_SUCCESS]](s32) + ; CHECK: RET_ReallyLR implicit $w0 + %val_success = cmpxchg volatile i32* %addr, i32 0, i32 1 monotonic monotonic, !falkor.strided.access !0 + %value_loaded = extractvalue { i32, i1 } %val_success, 0 + ret i32 %value_loaded +} + +!0 = !{}