Index: clang/lib/CodeGen/CGClass.cpp =================================================================== --- clang/lib/CodeGen/CGClass.cpp +++ clang/lib/CodeGen/CGClass.cpp @@ -1472,7 +1472,7 @@ // Initialize the vtable pointers before entering the body. if (!CanSkipVTablePointerInitialization(*this, Dtor)) { - // Insert the llvm.invariant.group.barrier intrinsic before initializing + // Insert the llvm.launder.invariant.group intrinsic before initializing // the vptrs to cancel any previous assumptions we might have made. if (CGM.getCodeGenOpts().StrictVTablePointers && CGM.getCodeGenOpts().OptimizationLevel > 0) Index: clang/lib/CodeGen/CGExprCXX.cpp =================================================================== --- clang/lib/CodeGen/CGExprCXX.cpp +++ clang/lib/CodeGen/CGExprCXX.cpp @@ -1687,7 +1687,7 @@ llvm::Type *elementTy = ConvertTypeForMem(allocType); Address result = Builder.CreateElementBitCast(allocation, elementTy); - // Passing pointer through invariant.group.barrier to avoid propagation of + // Passing pointer through launder.invariant.group to avoid propagation of // vptrs information which may be included in previous type. // To not break LTO with different optimizations levels, we do it regardless // of optimization level. Index: clang/test/CodeGenCXX/invariant.group-for-vptrs.cpp =================================================================== --- clang/test/CodeGenCXX/invariant.group-for-vptrs.cpp +++ clang/test/CodeGenCXX/invariant.group-for-vptrs.cpp @@ -56,7 +56,7 @@ // Checking D::D() // CHECK-LABEL: define linkonce_odr void @_ZN1DC2Ev( -// CHECK: = call i8* @llvm.invariant.group.barrier.p0i8(i8* +// CHECK: = call i8* @llvm.launder.invariant.group.p0i8(i8* // CHECK: call void @_ZN1AC2Ev(%struct.A* // CHECK: store {{.*}} !invariant.group ![[MD]] Index: clang/test/CodeGenCXX/strict-vtable-pointers.cpp =================================================================== --- clang/test/CodeGenCXX/strict-vtable-pointers.cpp +++ clang/test/CodeGenCXX/strict-vtable-pointers.cpp @@ -53,7 +53,7 @@ }; // CHECK-NEW-LABEL: define void @_Z12LocalObjectsv() -// CHECK-NEW-NOT: @llvm.invariant.group.barrier.p0i8( +// CHECK-NEW-NOT: @llvm.launder.invariant.group.p0i8( // CHECK-NEW-LABEL: {{^}}} void LocalObjects() { DynamicBase1 DB; @@ -81,20 +81,20 @@ struct DynamicFromVirtualStatic1; // CHECK-CTORS-LABEL: define linkonce_odr void @_ZN25DynamicFromVirtualStatic1C1Ev -// CHECK-CTORS-NOT: @llvm.invariant.group.barrier.p0i8( +// CHECK-CTORS-NOT: @llvm.launder.invariant.group.p0i8( // CHECK-CTORS-LABEL: {{^}}} struct DynamicFrom2Virtuals; // CHECK-CTORS-LABEL: define linkonce_odr void @_ZN20DynamicFrom2VirtualsC1Ev -// CHECK-CTORS: call i8* @llvm.invariant.group.barrier.p0i8( +// CHECK-CTORS: call i8* @llvm.launder.invariant.group.p0i8( // CHECK-CTORS-LABEL: {{^}}} // CHECK-NEW-LABEL: define void @_Z9Pointers1v() -// CHECK-NEW-NOT: @llvm.invariant.group.barrier.p0i8( +// CHECK-NEW-NOT: @llvm.launder.invariant.group.p0i8( // CHECK-NEW-LABEL: call void @_ZN12DynamicBase1C1Ev( -// CHECK-NEW: %[[THIS3:.*]] = call i8* @llvm.invariant.group.barrier.p0i8(i8* %[[THIS2:.*]]) +// CHECK-NEW: %[[THIS3:.*]] = call i8* @llvm.launder.invariant.group.p0i8(i8* %[[THIS2:.*]]) // CHECK-NEW: %[[THIS4:.*]] = bitcast i8* %[[THIS3]] to %[[DynamicDerived:.*]]* // CHECK-NEW: call void @_ZN14DynamicDerivedC1Ev(%[[DynamicDerived:.*]]* %[[THIS4]]) // CHECK-NEW-LABEL: {{^}}} @@ -109,9 +109,9 @@ // CHECK-NEW-LABEL: define void @_Z14HackingObjectsv() // CHECK-NEW: call void @_ZN12DynamicBase1C1Ev -// CHECK-NEW: call i8* @llvm.invariant.group.barrier.p0i8( +// CHECK-NEW: call i8* @llvm.launder.invariant.group.p0i8( // CHECK-NEW: call void @_ZN14DynamicDerivedC1Ev( -// CHECK-NEW: call i8* @llvm.invariant.group.barrier.p0i8( +// CHECK-NEW: call i8* @llvm.launder.invariant.group.p0i8( // CHECK-NEW: call void @_ZN12DynamicBase1C1Ev( // CHECK-NEW-LABEL: {{^}}} void HackingObjects() { @@ -131,7 +131,7 @@ /*** Testing Constructors ***/ struct DynamicBase1; // CHECK-CTORS-LABEL: define linkonce_odr void @_ZN12DynamicBase1C2Ev( -// CHECK-CTORS-NOT: call i8* @llvm.invariant.group.barrier.p0i8( +// CHECK-CTORS-NOT: call i8* @llvm.launder.invariant.group.p0i8( // CHECK-CTORS-LABEL: {{^}}} @@ -140,7 +140,7 @@ // CHECK-CTORS-LABEL: define linkonce_odr void @_ZN14DynamicDerivedC2Ev( // CHECK-CTORS: %[[THIS0:.*]] = load %[[DynamicDerived:.*]]*, %[[DynamicDerived]]** {{.*}} // CHECK-CTORS: %[[THIS1:.*]] = bitcast %[[DynamicDerived:.*]]* %[[THIS0]] to i8* -// CHECK-CTORS: %[[THIS2:.*]] = call i8* @llvm.invariant.group.barrier.p0i8(i8* %[[THIS1:.*]]) +// CHECK-CTORS: %[[THIS2:.*]] = call i8* @llvm.launder.invariant.group.p0i8(i8* %[[THIS1:.*]]) // CHECK-CTORS: %[[THIS3:.*]] = bitcast i8* %[[THIS2]] to %[[DynamicDerived]]* // CHECK-CTORS: %[[THIS4:.*]] = bitcast %[[DynamicDerived]]* %[[THIS3]] to %[[DynamicBase:.*]]* // CHECK-CTORS: call void @_ZN12DynamicBase1C2Ev(%[[DynamicBase]]* %[[THIS4]]) @@ -154,15 +154,15 @@ // CHECK-CTORS: %[[THIS0:.*]] = load %[[CLASS:.*]]*, %[[CLASS]]** {{.*}} // CHECK-CTORS: %[[THIS1:.*]] = bitcast %[[CLASS:.*]]* %[[THIS0]] to i8* -// CHECK-CTORS: %[[THIS2:.*]] = call i8* @llvm.invariant.group.barrier.p0i8(i8* %[[THIS1]]) +// CHECK-CTORS: %[[THIS2:.*]] = call i8* @llvm.launder.invariant.group.p0i8(i8* %[[THIS1]]) // CHECK-CTORS: %[[THIS3:.*]] = bitcast i8* %[[THIS2]] to %[[CLASS]]* // CHECK-CTORS: %[[THIS4:.*]] = bitcast %[[CLASS]]* %[[THIS3]] to %[[BASE_CLASS:.*]]* // CHECK-CTORS: call void @_ZN12DynamicBase1C2Ev(%[[BASE_CLASS]]* %[[THIS4]]) -// CHECK-CTORS: call i8* @llvm.invariant.group.barrier.p0i8( +// CHECK-CTORS: call i8* @llvm.launder.invariant.group.p0i8( // CHECK-CTORS: call void @_ZN12DynamicBase2C2Ev( -// CHECK-CTORS-NOT: @llvm.invariant.group.barrier.p0i8 +// CHECK-CTORS-NOT: @llvm.launder.invariant.group.p0i8 // CHECK-CTORS: %[[THIS10:.*]] = bitcast %struct.DynamicDerivedMultiple* %[[THIS0]] to i32 (...)*** @@ -177,7 +177,7 @@ struct DynamicFromStatic; // CHECK-CTORS-LABEL: define linkonce_odr void @_ZN17DynamicFromStaticC2Ev( -// CHECK-CTORS-NOT: @llvm.invariant.group.barrier.p0i8( +// CHECK-CTORS-NOT: @llvm.launder.invariant.group.p0i8( // CHECK-CTORS-LABEL: {{^}}} struct A { @@ -206,15 +206,15 @@ void UnionsBarriers(U *u) { // CHECK-NEW: call void @_Z9changeToBP1U( changeToB(u); - // CHECK-NEW: call i8* @llvm.invariant.group.barrier.p0i8(i8* + // CHECK-NEW: call i8* @llvm.launder.invariant.group.p0i8(i8* // CHECK-NEW: call void @_Z2g2P1A(%struct.A* g2(&u->b); // CHECK-NEW: call void @_Z9changeToAP1U(%union.U* changeToA(u); - // CHECK-NEW: call i8* @llvm.invariant.group.barrier.p0i8(i8* + // CHECK-NEW: call i8* @llvm.launder.invariant.group.p0i8(i8* // call void @_Z2g2P1A(%struct.A* %a) g2(&u->a); - // CHECK-NEW-NOT: call i8* @llvm.invariant.group.barrier.p0i8(i8* + // CHECK-NEW-NOT: call i8* @llvm.launder.invariant.group.p0i8(i8* } struct HoldingVirtuals { @@ -233,10 +233,10 @@ // CHECK-NEW-LABEL: noBarriers void noBarriers(NoVptrs &noVptrs) { - // CHECK-NEW-NOT: call i8* @llvm.invariant.group.barrier.p0i8(i8* + // CHECK-NEW-NOT: call i8* @llvm.launder.invariant.group.p0i8(i8* // CHECK-NEW: 42 noVptrs.a += 42; - // CHECK-NEW-NOT: call i8* @llvm.invariant.group.barrier.p0i8(i8* + // CHECK-NEW-NOT: call i8* @llvm.launder.invariant.group.p0i8(i8* // CHECK-NEW: call void @_Z4takeR12AnotherEmpty( take(noVptrs.empty); } @@ -249,10 +249,10 @@ // CHECK-NEW-LABEL: define void @_Z15UnionsBarriers2R2U2 void UnionsBarriers2(U2 &u) { - // CHECK-NEW-NOT: call i8* @llvm.invariant.group.barrier.p0i8(i8* + // CHECK-NEW-NOT: call i8* @llvm.launder.invariant.group.p0i8(i8* // CHECK-NEW: 42 u.z += 42; - // CHECK-NEW: call i8* @llvm.invariant.group.barrier.p0i8(i8* + // CHECK-NEW: call i8* @llvm.launder.invariant.group.p0i8(i8* // CHECK-NEW: call void @_Z4takeR15HoldingVirtuals( take(u.h); } @@ -279,24 +279,24 @@ void take(VirtualInheritance &); void UnionsBarrier3(U3 &u) { - // CHECK-NEW-NOT: call i8* @llvm.invariant.group.barrier.p0i8(i8* + // CHECK-NEW-NOT: call i8* @llvm.launder.invariant.group.p0i8(i8* // CHECK-NEW: 42 u.z += 42; - // CHECK-NEW: call i8* @llvm.invariant.group.barrier.p0i8(i8* + // CHECK-NEW: call i8* @llvm.launder.invariant.group.p0i8(i8* // CHECK-NEW: call void @_Z4takeR13VirtualInBase( take(u.v1); - // CHECK-NEW: call i8* @llvm.invariant.group.barrier.p0i8(i8* + // CHECK-NEW: call i8* @llvm.launder.invariant.group.p0i8(i8* // CHECK-NEW: call void @_Z4takeR13VirtualInBase( take(u.v2); - // CHECK-NEW: call i8* @llvm.invariant.group.barrier.p0i8(i8* + // CHECK-NEW: call i8* @llvm.launder.invariant.group.p0i8(i8* // CHECK-NEW: call void @_Z4takeR18VirtualInheritance( take(u.v3); } /** DTORS **/ // CHECK-DTORS-LABEL: define linkonce_odr void @_ZN10StaticBaseD2Ev( -// CHECK-DTORS-NOT: call i8* @llvm.invariant.group.barrier.p0i8( +// CHECK-DTORS-NOT: call i8* @llvm.launder.invariant.group.p0i8( // CHECK-DTORS-LABEL: {{^}}} @@ -305,22 +305,22 @@ // CHECK-DTORS-LABEL: {{^}}} // CHECK-DTORS-LABEL: define linkonce_odr void @_ZN17DynamicFromStaticD2Ev -// CHECK-DTORS-NOT: call i8* @llvm.invariant.group.barrier.p0i8( +// CHECK-DTORS-NOT: call i8* @llvm.launder.invariant.group.p0i8( // CHECK-DTORS-LABEL: {{^}}} // CHECK-DTORS-LABEL: define linkonce_odr void @_ZN22DynamicDerivedMultipleD2Ev( // CHECK-DTORS-LABEL: define linkonce_odr void @_ZN12DynamicBase2D2Ev( -// CHECK-DTORS: call i8* @llvm.invariant.group.barrier.p0i8( +// CHECK-DTORS: call i8* @llvm.launder.invariant.group.p0i8( // CHECK-DTORS-LABEL: {{^}}} // CHECK-DTORS-LABEL: define linkonce_odr void @_ZN12DynamicBase1D2Ev -// CHECK-DTORS: call i8* @llvm.invariant.group.barrier.p0i8( +// CHECK-DTORS: call i8* @llvm.launder.invariant.group.p0i8( // CHECK-DTORS-LABEL: {{^}}} // CHECK-DTORS-LABEL: define linkonce_odr void @_ZN14DynamicDerivedD2Ev -// CHECK-DTORS-NOT: call i8* @llvm.invariant.group.barrier.p0i8( +// CHECK-DTORS-NOT: call i8* @llvm.launder.invariant.group.p0i8( // CHECK-DTORS-LABEL: {{^}}} Index: llvm/docs/LangRef.rst =================================================================== --- llvm/docs/LangRef.rst +++ llvm/docs/LangRef.rst @@ -5312,7 +5312,7 @@ The existence of the ``invariant.group`` metadata on the instruction tells the optimizer that every ``load`` and ``store`` to the same pointer operand within the same invariant group can be assumed to load or store the same -value (but see the ``llvm.invariant.group.barrier`` intrinsic which affects +value (but see the ``llvm.launder.invariant.group`` intrinsic which affects when two pointers are considered the same). Pointers returned by bitcast or getelementptr with only zero indices are considered the same. @@ -5337,13 +5337,13 @@ store i8 %unknownValue, i8* %ptr, !invariant.group !0 ; Can assume that %unknownValue == 42 call void @foo(i8* %ptr) - %newPtr2 = call i8* @llvm.invariant.group.barrier(i8* %ptr) - %d = load i8, i8* %newPtr2, !invariant.group !0 ; Can't step through invariant.group.barrier to get value of %ptr + %newPtr2 = call i8* @llvm.launder.invariant.group(i8* %ptr) + %d = load i8, i8* %newPtr2, !invariant.group !0 ; Can't step through launder.invariant.group to get value of %ptr ... declare void @foo(i8*) declare i8* @getPointer(i8*) - declare i8* @llvm.invariant.group.barrier(i8*) + declare i8* @llvm.launder.invariant.group(i8*) !0 = !{!"magic ptr"} !1 = !{!"other ptr"} @@ -12901,7 +12901,7 @@ This intrinsic indicates that the memory is mutable again. -'``llvm.invariant.group.barrier``' Intrinsic +'``llvm.launder.invariant.group``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Syntax: @@ -12912,12 +12912,12 @@ :: - declare i8* @llvm.invariant.group.barrier.p0i8(i8* ) + declare i8* @llvm.launder.invariant.group.p0i8(i8* ) Overview: """"""""" -The '``llvm.invariant.group.barrier``' intrinsic can be used when an invariant +The '``llvm.launder.invariant.group``' intrinsic can be used when an invariant established by invariant.group metadata no longer holds, to obtain a new pointer value that does not carry the invariant information. @@ -12925,7 +12925,7 @@ Arguments: """""""""" -The ``llvm.invariant.group.barrier`` takes only one argument, which is +The ``llvm.launder.invariant.group`` takes only one argument, which is the pointer to the memory for which the ``invariant.group`` no longer holds. Semantics: @@ -12933,6 +12933,7 @@ Returns another pointer that aliases its argument but which is considered different for the purposes of ``load``/``store`` ``invariant.group`` metadata. +It does not read any accessible memory and the execution can be speculated. .. _constrainedfp: Index: llvm/include/llvm/IR/IRBuilder.h =================================================================== --- llvm/include/llvm/IR/IRBuilder.h +++ llvm/include/llvm/IR/IRBuilder.h @@ -1957,21 +1957,22 @@ Name); } - /// \brief Create an invariant.group.barrier intrinsic call, that stops - /// optimizer to propagate equality using invariant.group metadata. - /// If Ptr type is different from pointer to i8, it's casted to pointer to i8 - /// in the same address space before call and casted back to Ptr type after - /// call. + /// \brief Create an launder.invariant.group intrinsic call, that returns + /// another pointer that aliases its argument but which is considered + /// different for the purposes of ``load``/``store`` ``invariant.group`` + /// metadata. If Ptr type is different from pointer to i8, it's casted to + /// pointer to i8 in the same address space before call and casted back to Ptr + /// type after call. Value *CreateInvariantGroupBarrier(Value *Ptr) { assert(isa(Ptr->getType()) && - "invariant.group.barrier only applies to pointers."); + "launder.invariant.group only applies to pointers."); auto *PtrType = Ptr->getType(); auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace()); if (PtrType != Int8PtrTy) Ptr = CreateBitCast(Ptr, Int8PtrTy); Module *M = BB->getParent()->getParent(); Function *FnInvariantGroupBarrier = Intrinsic::getDeclaration( - M, Intrinsic::invariant_group_barrier, {Int8PtrTy}); + M, Intrinsic::launder_invariant_group, {Int8PtrTy}); assert(FnInvariantGroupBarrier->getReturnType() == Int8PtrTy && FnInvariantGroupBarrier->getFunctionType()->getParamType(0) == Index: llvm/include/llvm/IR/Intrinsics.td =================================================================== --- llvm/include/llvm/IR/Intrinsics.td +++ llvm/include/llvm/IR/Intrinsics.td @@ -706,7 +706,7 @@ llvm_anyptr_ty], [IntrArgMemOnly, NoCapture<2>]>; -// invariant.group.barrier can't be marked with 'readnone' (IntrNoMem), +// launder.invariant.group can't be marked with 'readnone' (IntrNoMem), // because it would cause CSE of two barriers with the same argument. // Inaccessiblememonly says that the barrier doesn't read the argument, // but it changes state not accessible to this module. This way @@ -716,9 +716,9 @@ // CSE 2 barriers with store in between. // The argument also can't be marked with 'returned' attribute, because // it would remove barrier. -def int_invariant_group_barrier : Intrinsic<[llvm_anyptr_ty], +def int_launder_invariant_group : Intrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>], - [IntrInaccessibleMemOnly]>; + [IntrInaccessibleMemOnly, IntrSpeculatable]>; //===------------------------ Stackmap Intrinsics -------------------------===// // Index: llvm/include/llvm/IR/Value.h =================================================================== --- llvm/include/llvm/IR/Value.h +++ llvm/include/llvm/IR/Value.h @@ -514,10 +514,10 @@ /// Returns the original uncasted value. If this is called on a non-pointer /// value, it returns 'this'. This function should be used only in /// Alias analysis. - const Value *stripPointerCastsAndBarriers() const; - Value *stripPointerCastsAndBarriers() { + const Value *stripPointerCastsAndInvariantGroups() const; + Value *stripPointerCastsAndInvariantGroups() { return const_cast( - static_cast(this)->stripPointerCastsAndBarriers()); + static_cast(this)->stripPointerCastsAndInvariantGroups()); } /// \brief Strip off pointer casts and all-zero GEPs. Index: llvm/lib/Analysis/BasicAliasAnalysis.cpp =================================================================== --- llvm/lib/Analysis/BasicAliasAnalysis.cpp +++ llvm/lib/Analysis/BasicAliasAnalysis.cpp @@ -985,8 +985,8 @@ const GEPOperator *GEP2, uint64_t V2Size, const DataLayout &DL) { - assert(GEP1->getPointerOperand()->stripPointerCastsAndBarriers() == - GEP2->getPointerOperand()->stripPointerCastsAndBarriers() && + assert(GEP1->getPointerOperand()->stripPointerCastsAndInvariantGroups() == + GEP2->getPointerOperand()->stripPointerCastsAndInvariantGroups() && GEP1->getPointerOperandType() == GEP2->getPointerOperandType() && "Expected GEPs with the same pointer operand"); @@ -1264,8 +1264,8 @@ // If we know the two GEPs are based off of the exact same pointer (and not // just the same underlying object), see if that tells us anything about // the resulting pointers. - if (GEP1->getPointerOperand()->stripPointerCastsAndBarriers() == - GEP2->getPointerOperand()->stripPointerCastsAndBarriers() && + if (GEP1->getPointerOperand()->stripPointerCastsAndInvariantGroups() == + GEP2->getPointerOperand()->stripPointerCastsAndInvariantGroups() && GEP1->getPointerOperandType() == GEP2->getPointerOperandType()) { AliasResult R = aliasSameBasePointerGEPs(GEP1, V1Size, GEP2, V2Size, DL); // If we couldn't find anything interesting, don't abandon just yet. @@ -1578,8 +1578,8 @@ return NoAlias; // Strip off any casts if they exist. - V1 = V1->stripPointerCastsAndBarriers(); - V2 = V2->stripPointerCastsAndBarriers(); + V1 = V1->stripPointerCastsAndInvariantGroups(); + V2 = V2->stripPointerCastsAndInvariantGroups(); // If V1 or V2 is undef, the result is NoAlias because we can always pick a // value for undef that aliases nothing in the program. Index: llvm/lib/Analysis/MemorySSA.cpp =================================================================== --- llvm/lib/Analysis/MemorySSA.cpp +++ llvm/lib/Analysis/MemorySSA.cpp @@ -351,9 +351,6 @@ const Instruction *I) { // If the memory can't be changed, then loads of the memory can't be // clobbered. - // - // FIXME: We should handle invariant groups, as well. It's a bit harder, - // because we need to pay close attention to invariant group barriers. return isa(I) && (I->getMetadata(LLVMContext::MD_invariant_load) || AA.pointsToConstantMemory(cast(I)-> getPointerOperand())); Index: llvm/lib/CodeGen/CodeGenPrepare.cpp =================================================================== --- llvm/lib/CodeGen/CodeGenPrepare.cpp +++ llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -1667,7 +1667,7 @@ InsertedInsts.insert(ExtVal); return true; } - case Intrinsic::invariant_group_barrier: + case Intrinsic::launder_invariant_group: II->replaceAllUsesWith(II->getArgOperand(0)); II->eraseFromParent(); return true; Index: llvm/lib/CodeGen/SelectionDAG/FastISel.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/FastISel.cpp +++ llvm/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -1410,7 +1410,7 @@ updateValueMap(II, ResultReg); return true; } - case Intrinsic::invariant_group_barrier: + case Intrinsic::launder_invariant_group: case Intrinsic::expect: { unsigned ResultReg = getRegForValue(II->getArgOperand(0)); if (!ResultReg) Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -5760,7 +5760,7 @@ } case Intrinsic::annotation: case Intrinsic::ptr_annotation: - case Intrinsic::invariant_group_barrier: + case Intrinsic::launder_invariant_group: // Drop the intrinsic, but forward the value setValue(&I, getValue(I.getOperand(0))); return nullptr; Index: llvm/lib/IR/Value.cpp =================================================================== --- llvm/lib/IR/Value.cpp +++ llvm/lib/IR/Value.cpp @@ -499,7 +499,7 @@ enum PointerStripKind { PSK_ZeroIndices, PSK_ZeroIndicesAndAliases, - PSK_ZeroIndicesAndAliasesAndBarriers, + PSK_ZeroIndicesAndAliasesAndInvariantGroups, PSK_InBoundsConstantIndices, PSK_InBounds }; @@ -518,7 +518,7 @@ if (auto *GEP = dyn_cast(V)) { switch (StripKind) { case PSK_ZeroIndicesAndAliases: - case PSK_ZeroIndicesAndAliasesAndBarriers: + case PSK_ZeroIndicesAndAliasesAndInvariantGroups: case PSK_ZeroIndices: if (!GEP->hasAllZeroIndices()) return V; @@ -546,11 +546,11 @@ V = RV; continue; } - // The result of invariant.group.barrier must alias it's argument, + // The result of launder.invariant.group must alias it's argument, // but it can't be marked with returned attribute, that's why it needs // special case. - if (StripKind == PSK_ZeroIndicesAndAliasesAndBarriers && - CS.getIntrinsicID() == Intrinsic::invariant_group_barrier) { + if (StripKind == PSK_ZeroIndicesAndAliasesAndInvariantGroups && + CS.getIntrinsicID() == Intrinsic::launder_invariant_group) { V = CS.getArgOperand(0); continue; } @@ -576,8 +576,8 @@ return stripPointerCastsAndOffsets(this); } -const Value *Value::stripPointerCastsAndBarriers() const { - return stripPointerCastsAndOffsets( +const Value *Value::stripPointerCastsAndInvariantGroups() const { + return stripPointerCastsAndOffsets( this); } Index: llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp =================================================================== --- llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp +++ llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp @@ -454,7 +454,7 @@ case Intrinsic::lifetime_end: case Intrinsic::invariant_start: case Intrinsic::invariant_end: - case Intrinsic::invariant_group_barrier: + case Intrinsic::launder_invariant_group: case Intrinsic::objectsize: return true; default: @@ -878,7 +878,7 @@ } case Intrinsic::invariant_start: case Intrinsic::invariant_end: - case Intrinsic::invariant_group_barrier: + case Intrinsic::launder_invariant_group: Intr->eraseFromParent(); // FIXME: I think the invariant marker should still theoretically apply, // but the intrinsics need to be changed to accept pointers with any Index: llvm/test/Analysis/MemorySSA/invariant-groups.ll =================================================================== --- llvm/test/Analysis/MemorySSA/invariant-groups.ll +++ llvm/test/Analysis/MemorySSA/invariant-groups.ll @@ -1,7 +1,7 @@ ; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s ; ; Currently, MemorySSA doesn't support invariant groups. So, we should ignore -; invariant.group.barrier intrinsics entirely. We'll need to pay attention to +; launder.invariant.group intrinsics entirely. We'll need to pay attention to ; them when/if we decide to support invariant groups. @g = external global i32 @@ -17,8 +17,8 @@ %1 = bitcast i32* %a to i8* ; CHECK: 3 = MemoryDef(2) -; CHECK-NEXT: %a8 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %1) - %a8 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %1) +; CHECK-NEXT: %a8 = call i8* @llvm.launder.invariant.group.p0i8(i8* %1) + %a8 = call i8* @llvm.launder.invariant.group.p0i8(i8* %1) %a32 = bitcast i8* %a8 to i32* ; This have to be MemoryUse(2), because we can't skip the barrier based on @@ -36,8 +36,8 @@ %1 = bitcast i32* %a to i8* ; CHECK: 2 = MemoryDef(1) -; CHECK-NEXT: %a8 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %1) - %a8 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %1) +; CHECK-NEXT: %a8 = call i8* @llvm.launder.invariant.group.p0i8(i8* %1) + %a8 = call i8* @llvm.launder.invariant.group.p0i8(i8* %1) %a32 = bitcast i8* %a8 to i32* ; We can skip the barrier only if the "skip" is not based on !invariant.group. @@ -55,8 +55,8 @@ %1 = bitcast i32* %a to i8* ; CHECK: 1 = MemoryDef(liveOnEntry) -; CHECK-NEXT: %a8 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %1) - %a8 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %1) +; CHECK-NEXT: %a8 = call i8* @llvm.launder.invariant.group.p0i8(i8* %1) + %a8 = call i8* @llvm.launder.invariant.group.p0i8(i8* %1) %a32 = bitcast i8* %a8 to i32* ; We can skip the barrier only if the "skip" is not based on !invariant.group. @@ -86,8 +86,8 @@ store i32 1, i32* @g, align 4 %1 = bitcast i32* %a to i8* ; CHECK: 3 = MemoryDef(2) -; CHECK-NEXT: %a8 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %1) - %a8 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %1) +; CHECK-NEXT: %a8 = call i8* @llvm.launder.invariant.group.p0i8(i8* %1) + %a8 = call i8* @llvm.launder.invariant.group.p0i8(i8* %1) %a32 = bitcast i8* %a8 to i32* ; CHECK: MemoryUse(2) @@ -145,8 +145,8 @@ call void @clobber8(i8* %p) ; CHECK: 3 = MemoryDef(2) -; CHECK-NEXT: %after = call i8* @llvm.invariant.group.barrier.p0i8(i8* %p) - %after = call i8* @llvm.invariant.group.barrier.p0i8(i8* %p) +; CHECK-NEXT: %after = call i8* @llvm.launder.invariant.group.p0i8(i8* %p) + %after = call i8* @llvm.launder.invariant.group.p0i8(i8* %p) br i1 undef, label %Loop.Body, label %Loop.End Loop.Body: @@ -192,8 +192,8 @@ call void @clobber8(i8* %p) ; CHECK: 3 = MemoryDef(2) -; CHECK-NEXT: %after = call i8* @llvm.invariant.group.barrier.p0i8(i8* %p) - %after = call i8* @llvm.invariant.group.barrier.p0i8(i8* %p) +; CHECK-NEXT: %after = call i8* @llvm.launder.invariant.group.p0i8(i8* %p) + %after = call i8* @llvm.launder.invariant.group.p0i8(i8* %p) br i1 undef, label %Loop.Body, label %Loop.End Loop.Body: @@ -253,8 +253,8 @@ ; CHECK-NEXT: call void @clobber call void @clobber8(i8* %p) ; CHECK: 3 = MemoryDef(2) -; CHECK-NEXT: %after = call i8* @llvm.invariant.group.barrier.p0i8(i8* %p) - %after = call i8* @llvm.invariant.group.barrier.p0i8(i8* %p) +; CHECK-NEXT: %after = call i8* @llvm.launder.invariant.group.p0i8(i8* %p) + %after = call i8* @llvm.launder.invariant.group.p0i8(i8* %p) br i1 undef, label %Loop.Pre, label %Loop.End Loop.Pre: @@ -302,12 +302,12 @@ ; CHECK-NEXT: store i8 42, i8* %ptr, !invariant.group !0 store i8 42, i8* %ptr, !invariant.group !0 ; CHECK: 2 = MemoryDef(1) -; CHECK-NEXT: call i8* @llvm.invariant.group.barrier - %ptr2 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %ptr) +; CHECK-NEXT: call i8* @llvm.launder.invariant.group + %ptr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr) ; FIXME: This one could be CSEd. ; CHECK: 3 = MemoryDef(2) -; CHECK: call i8* @llvm.invariant.group.barrier - %ptr3 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %ptr) +; CHECK: call i8* @llvm.launder.invariant.group + %ptr3 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr) ; CHECK: 4 = MemoryDef(3) ; CHECK-NEXT: call void @clobber8(i8* %ptr) call void @clobber8(i8* %ptr) @@ -331,13 +331,13 @@ ; CHECK-NEXT: store i8 42, i8* %ptr, !invariant.group !0 store i8 42, i8* %ptr, !invariant.group !0 ; CHECK: 2 = MemoryDef(1) -; CHECK-NEXT: call i8* @llvm.invariant.group.barrier - %ptr2 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %ptr) +; CHECK-NEXT: call i8* @llvm.launder.invariant.group + %ptr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr) ; CHECK: 3 = MemoryDef(2) store i8 43, i8* %ptr ; CHECK: 4 = MemoryDef(3) -; CHECK-NEXT: call i8* @llvm.invariant.group.barrier - %ptr3 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %ptr) +; CHECK-NEXT: call i8* @llvm.launder.invariant.group + %ptr3 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr) ; CHECK: 5 = MemoryDef(4) ; CHECK-NEXT: call void @clobber8(i8* %ptr) call void @clobber8(i8* %ptr) @@ -354,7 +354,7 @@ } -declare i8* @llvm.invariant.group.barrier.p0i8(i8*) +declare i8* @llvm.launder.invariant.group.p0i8(i8*) declare void @clobber(i32*) declare void @clobber8(i8*) declare void @use(i8* readonly) Index: llvm/test/CodeGen/AArch64/GlobalISel/unknown-intrinsic.ll =================================================================== --- llvm/test/CodeGen/AArch64/GlobalISel/unknown-intrinsic.ll +++ llvm/test/CodeGen/AArch64/GlobalISel/unknown-intrinsic.ll @@ -1,10 +1,10 @@ ; RUN: llc -O0 -mtriple=arm64 < %s -declare i8* @llvm.invariant.group.barrier(i8*) +declare i8* @llvm.launder.invariant.group(i8*) define i8* @barrier(i8* %p) { -; CHECK: bl llvm.invariant.group.barrier - %q = call i8* @llvm.invariant.group.barrier(i8* %p) +; CHECK: bl llvm.launder.invariant.group + %q = call i8* @llvm.launder.invariant.group(i8* %p) ret i8* %q } Index: llvm/test/CodeGen/AMDGPU/promote-alloca-invariant-markers.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/promote-alloca-invariant-markers.ll +++ llvm/test/CodeGen/AMDGPU/promote-alloca-invariant-markers.ll @@ -3,7 +3,7 @@ declare {}* @llvm.invariant.start.p5i8(i64, i8 addrspace(5)* nocapture) #0 declare void @llvm.invariant.end.p5i8({}*, i64, i8 addrspace(5)* nocapture) #0 -declare i8 addrspace(5)* @llvm.invariant.group.barrier.p5i8(i8 addrspace(5)*) #1 +declare i8 addrspace(5)* @llvm.launder.invariant.group.p5i8(i8 addrspace(5)*) #1 ; GCN-LABEL: {{^}}use_invariant_promotable_lds: ; GCN: buffer_load_dword @@ -17,7 +17,7 @@ store i32 %tmp3, i32 addrspace(5)* %tmp %tmp4 = call {}* @llvm.invariant.start.p5i8(i64 4, i8 addrspace(5)* %tmp1) #0 call void @llvm.invariant.end.p5i8({}* %tmp4, i64 4, i8 addrspace(5)* %tmp1) #0 - %tmp5 = call i8 addrspace(5)* @llvm.invariant.group.barrier.p5i8(i8 addrspace(5)* %tmp1) #1 + %tmp5 = call i8 addrspace(5)* @llvm.launder.invariant.group.p5i8(i8 addrspace(5)* %tmp1) #1 ret void } Index: llvm/test/CodeGen/Generic/intrinsics.ll =================================================================== --- llvm/test/CodeGen/Generic/intrinsics.ll +++ llvm/test/CodeGen/Generic/intrinsics.ll @@ -39,10 +39,10 @@ ret double %I } -declare i8* @llvm.invariant.group.barrier(i8*) +declare i8* @llvm.launder.invariant.group(i8*) define i8* @barrier(i8* %p) { - %q = call i8* @llvm.invariant.group.barrier(i8* %p) + %q = call i8* @llvm.launder.invariant.group(i8* %p) ret i8* %q } Index: llvm/test/Other/Inputs/invariant.group.barrier.ll =================================================================== --- llvm/test/Other/Inputs/invariant.group.barrier.ll +++ /dev/null @@ -1,15 +0,0 @@ -; RUN: opt -S -gvn < %s | FileCheck %s -; RUN: opt -S -newgvn < %s | FileCheck %s -; RUN: opt -S -O3 < %s | FileCheck %s - -; This test check if optimizer is not proving equality based on mustalias -; CHECK-LABEL: define void @dontProveEquality(i8* %a) -define void @dontProveEquality(i8* %a) { - %b = call i8* @llvm.invariant.group.barrier(i8* %a) - %r = i1 icmp eq i8* %b, i8* %a -;CHECK: call void @use(%r) - call void @use(%r) -} - -declare void @use(i1) -declare i8* @llvm.invariant.group.barrier(i8 *) Index: llvm/test/Other/launder.invariant.group.ll =================================================================== --- llvm/test/Other/launder.invariant.group.ll +++ llvm/test/Other/launder.invariant.group.ll @@ -4,7 +4,7 @@ ; RUN: opt -S -O3 < %s | FileCheck %s ; These tests checks if passes with CSE functionality can do CSE on -; invariant.group.barrier, that is prohibited if there is a memory clobber +; launder.invariant.group, that is prohibited if there is a memory clobber ; between barriers call. ; CHECK-LABEL: define i8 @optimizable() @@ -12,11 +12,11 @@ entry: %ptr = alloca i8 store i8 42, i8* %ptr, !invariant.group !0 -; CHECK: call i8* @llvm.invariant.group.barrier.p0i8 - %ptr2 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %ptr) +; CHECK: call i8* @llvm.launder.invariant.group.p0i8 + %ptr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr) ; FIXME: This one could be CSE -; CHECK: call i8* @llvm.invariant.group.barrier - %ptr3 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %ptr) +; CHECK: call i8* @llvm.launder.invariant.group + %ptr3 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr) ; CHECK: call void @clobber(i8* {{.*}}%ptr) call void @clobber(i8* %ptr) @@ -35,11 +35,11 @@ entry: %ptr = alloca i8 store i8 42, i8* %ptr, !invariant.group !0 -; CHECK: call i8* @llvm.invariant.group.barrier.p0i8 - %ptr2 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %ptr) +; CHECK: call i8* @llvm.launder.invariant.group.p0i8 + %ptr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr) call void @clobber(i8* %ptr) -; CHECK: call i8* @llvm.invariant.group.barrier.p0i8 - %ptr3 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %ptr) +; CHECK: call i8* @llvm.launder.invariant.group.p0i8 + %ptr3 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr) ; CHECK: call void @clobber(i8* {{.*}}%ptr) call void @clobber(i8* %ptr) ; CHECK: call void @use(i8* {{.*}}%ptr2) @@ -56,11 +56,11 @@ define i8 @unoptimizable2() { %ptr = alloca i8 store i8 42, i8* %ptr, !invariant.group !0 -; CHECK: call i8* @llvm.invariant.group.barrier - %ptr2 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %ptr) +; CHECK: call i8* @llvm.launder.invariant.group + %ptr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr) store i8 43, i8* %ptr -; CHECK: call i8* @llvm.invariant.group.barrier - %ptr3 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %ptr) +; CHECK: call i8* @llvm.launder.invariant.group + %ptr3 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr) ; CHECK: call void @clobber(i8* {{.*}}%ptr) call void @clobber(i8* %ptr) ; CHECK: call void @use(i8* {{.*}}%ptr2) @@ -75,9 +75,9 @@ declare void @use(i8* readonly) declare void @clobber(i8*) -; CHECK: Function Attrs: inaccessiblememonly nounwind{{$}} -; CHECK-NEXT: declare i8* @llvm.invariant.group.barrier.p0i8(i8*) -declare i8* @llvm.invariant.group.barrier.p0i8(i8*) +; CHECK: Function Attrs: inaccessiblememonly nounwind speculatable{{$}} +; CHECK-NEXT: declare i8* @llvm.launder.invariant.group.p0i8(i8*) +declare i8* @llvm.launder.invariant.group.p0i8(i8*) !0 = !{} Index: llvm/test/Transforms/CodeGenPrepare/invariant.group.ll =================================================================== --- llvm/test/Transforms/CodeGenPrepare/invariant.group.ll +++ llvm/test/Transforms/CodeGenPrepare/invariant.group.ll @@ -6,10 +6,10 @@ define void @foo() { enter: ; CHECK-NOT: !invariant.group - ; CHECK-NOT: @llvm.invariant.group.barrier.p0i8( + ; CHECK-NOT: @llvm.launder.invariant.group.p0i8( ; CHECK: %val = load i8, i8* @tmp, !tbaa %val = load i8, i8* @tmp, !invariant.group !0, !tbaa !{!1, !1, i64 0} - %ptr = call i8* @llvm.invariant.group.barrier.p0i8(i8* @tmp) + %ptr = call i8* @llvm.launder.invariant.group.p0i8(i8* @tmp) ; CHECK: store i8 42, i8* @tmp store i8 42, i8* %ptr, !invariant.group !0 @@ -18,7 +18,7 @@ } ; CHECK-LABEL: } -declare i8* @llvm.invariant.group.barrier.p0i8(i8*) +declare i8* @llvm.launder.invariant.group.p0i8(i8*) !0 = !{!"something"} !1 = !{!"x", !0} Index: llvm/test/Transforms/GVN/invariant.group.ll =================================================================== --- llvm/test/Transforms/GVN/invariant.group.ll +++ llvm/test/Transforms/GVN/invariant.group.ll @@ -25,7 +25,7 @@ entry: %ptr = alloca i8 store i8 42, i8* %ptr, !invariant.group !0 - %ptr2 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %ptr) + %ptr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr) %a = load i8, i8* %ptr, !invariant.group !0 call void @foo(i8* %ptr2); call to use %ptr2 @@ -242,7 +242,7 @@ entry: %ptr = alloca i8 store i8 42, i8* %ptr, !invariant.group !0 - %ptr2 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %ptr) + %ptr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr) ; CHECK-NOT: load %a = load i8, i8* %ptr2, !invariant.group !0 @@ -314,7 +314,7 @@ ; CHECK: store i8 %unknownValue, i8* %ptr, !invariant.group !0 store i8 %unknownValue, i8* %ptr, !invariant.group !0 - %newPtr2 = call i8* @llvm.invariant.group.barrier.p0i8(i8* %ptr) + %newPtr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr) ; CHECK-NOT: load %d = load i8, i8* %newPtr2, !invariant.group !0 ; CHECK: ret i8 %unknownValue @@ -441,7 +441,7 @@ declare void @_ZN1AC1Ev(%struct.A*) declare void @fooBit(i1*, i1) -declare i8* @llvm.invariant.group.barrier.p0i8(i8*) +declare i8* @llvm.launder.invariant.group.p0i8(i8*) ; Function Attrs: nounwind declare void @llvm.assume(i1 %cmp.vtables) #0 Index: llvm/test/Transforms/GlobalOpt/invariant.group.barrier.ll =================================================================== --- llvm/test/Transforms/GlobalOpt/invariant.group.barrier.ll +++ llvm/test/Transforms/GlobalOpt/invariant.group.barrier.ll @@ -33,7 +33,7 @@ store i32 %val, i32* %valptr %0 = bitcast i32* %valptr to i8* - %barr = call i8* @llvm.invariant.group.barrier(i8* %0) + %barr = call i8* @llvm.launder.invariant.group(i8* %0) %1 = bitcast i8* %barr to i32* %val2 = load i32, i32* %1 @@ -41,7 +41,7 @@ ret void } -; We can't step through invariant.group.barrier here, because that would change +; We can't step through launder.invariant.group here, because that would change ; this load in @usage_of_globals() ; val = load i32, i32* %ptrVal, !invariant.group !0 ; into @@ -54,7 +54,7 @@ store i32 13, i32* @tmp3, !invariant.group !0 %0 = bitcast i32* @tmp3 to i8* - %barr = call i8* @llvm.invariant.group.barrier(i8* %0) + %barr = call i8* @llvm.launder.invariant.group(i8* %0) %1 = bitcast i8* %barr to i32* store i32* %1, i32** @ptrToTmp3 @@ -74,6 +74,6 @@ declare void @changeTmp3ValAndCallBarrierInside() -declare i8* @llvm.invariant.group.barrier(i8*) +declare i8* @llvm.launder.invariant.group(i8*) !0 = !{!"something"} Index: llvm/test/Transforms/NewGVN/invariant.group.ll =================================================================== --- llvm/test/Transforms/NewGVN/invariant.group.ll +++ llvm/test/Transforms/NewGVN/invariant.group.ll @@ -26,7 +26,7 @@ entry: %ptr = alloca i8 store i8 42, i8* %ptr, !invariant.group !0 - %ptr2 = call i8* @llvm.invariant.group.barrier(i8* %ptr) + %ptr2 = call i8* @llvm.launder.invariant.group(i8* %ptr) %a = load i8, i8* %ptr, !invariant.group !0 call void @foo(i8* %ptr2); call to use %ptr2 @@ -243,7 +243,7 @@ entry: %ptr = alloca i8 store i8 42, i8* %ptr, !invariant.group !0 - %ptr2 = call i8* @llvm.invariant.group.barrier(i8* %ptr) + %ptr2 = call i8* @llvm.launder.invariant.group(i8* %ptr) %a = load i8, i8* %ptr2, !invariant.group !0 ; CHECK: ret i8 %a @@ -314,8 +314,8 @@ ; CHECK: store i8 %unknownValue, i8* %ptr, !invariant.group !0 store i8 %unknownValue, i8* %ptr, !invariant.group !0 - %newPtr2 = call i8* @llvm.invariant.group.barrier(i8* %ptr) - %d = load i8, i8* %newPtr2, !invariant.group !0 ; Can't step through invariant.group.barrier to get value of %ptr + %newPtr2 = call i8* @llvm.launder.invariant.group(i8* %ptr) + %d = load i8, i8* %newPtr2, !invariant.group !0 ; Can't step through launder.invariant.group to get value of %ptr ; CHECK: ret i8 %d ret i8 %d } @@ -441,7 +441,7 @@ declare void @_ZN1AC1Ev(%struct.A*) declare void @fooBit(i1*, i1) -declare i8* @llvm.invariant.group.barrier(i8*) +declare i8* @llvm.launder.invariant.group(i8*) ; Function Attrs: nounwind declare void @llvm.assume(i1 %cmp.vtables) #0