diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h --- a/llvm/include/llvm/IR/IRBuilder.h +++ b/llvm/include/llvm/IR/IRBuilder.h @@ -667,6 +667,43 @@ MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr); + /// Create and insert a memcmp between the specified pointers. + /// + /// If the pointers aren't i8*, they will be converted. If a TBAA tag is + /// specified, it will be added to the instruction. Likewise with alias.scope + /// and noalias tags. + CallInst *CreateMemCmp(Value *LHS, MaybeAlign LHSAlign, Value *RHS, + MaybeAlign RHSAlign, uint64_t Size, + bool isVolatile = false, MDNode *TBAATag = nullptr, + MDNode *TBAAStructTag = nullptr, + MDNode *ScopeTag = nullptr, + MDNode *NoAliasTag = nullptr) { + return CreateMemCmp(LHS, LHSAlign, RHS, RHSAlign, getInt64(Size), + isVolatile, TBAATag, TBAAStructTag, ScopeTag, + NoAliasTag); + } + + CallInst *CreateMemCmp(Value *LHS, MaybeAlign LHSAlign, Value *RHS, + MaybeAlign RHSAlign, Value *Size, + bool isVolatile = false, MDNode *TBAATag = nullptr, + MDNode *TBAAStructTag = nullptr, + MDNode *ScopeTag = nullptr, + MDNode *NoAliasTag = nullptr); + + /// Create and insert an element unordered-atomic memcmp between the + /// specified pointers. + /// + /// LHSAlign/RHSAlign are the alignments of the LHS/RHS pointers, respectively. + /// + /// If the pointers aren't i8*, they will be converted. If a TBAA tag is + /// specified, it will be added to the instruction. Likewise with alias.scope + /// and noalias tags. + CallInst *CreateElementUnorderedAtomicMemCmp( + Value *LHS, Align LHSAlign, Value *RHS, Align RHSAlign, Value *Size, + uint32_t ElementSize, MDNode *TBAATag = nullptr, + MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, + MDNode *NoAliasTag = nullptr); + CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, uint64_t Size, bool isVolatile = false, MDNode *TBAATag = nullptr, diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -765,6 +765,98 @@ void setValue(Value *Val) { BaseCL::setArg(ARG_VALUE, Val); } }; +/// Common base class for all binary mem intrinsics (i.e. memcmp). Simply +/// provides common methods. +template class BinaryMemIntrinsicBase : public BaseCL { +private: + enum { ARG_LHS = 0, ARG_RHS = 1 }; + +public: + /// Return the arguments to the instruction. + Value *getRawLHS() const { return BaseCL::getRawArg(ARG_LHS); } + + const Use &getRawLHSUse() const { + return BaseCL::getRawArgUse(ARG_LHS); + } + + Use &getRawLHSUse() { return BaseCL::getRawArgUse(ARG_LHS); } + + /// This is just like getRawLHS, but it strips off any cast + /// instructions that feed it, giving the original input. The returned + /// value is guaranteed to be a pointer. + Value *getLHS() const { return BaseCL::getArg(ARG_LHS); } + + unsigned getLHSAddressSpace() const { + return BaseCL::getArgAddressSpace(ARG_LHS); + } + + /// FIXME: Remove this function once transition to Align is over. + /// Use getLHSAlign() instead. + unsigned getLHSAlignment() const { + return BaseCL::getArgAlignment(ARG_LHS); + } + + MaybeAlign getLHSAlign() const { return BaseCL::getArgAlign(ARG_LHS); } + + void setLHS(Value *Ptr) { BaseCL::setArg(ARG_LHS, Ptr); } + + /// FIXME: Remove this function once transition to Align is over. + /// Use the version that takes MaybeAlign instead of this one. + void setLHSAlignment(unsigned Alignment) { + BaseCL::setArgAlignment(ARG_LHS, Alignment); + } + + void setLHSAlignment(MaybeAlign Alignment) { + BaseCL::setArgAlignment(ARG_LHS, Alignment); + } + + void setLHSAlignment(Align Alignment) { + BaseCL::setArgAlignment(ARG_LHS, Alignment); + } + + /// Return the arguments to the instruction. + Value *getRawRHS() const { return BaseCL::getRawArg(ARG_RHS); } + + const Use &getRawRHSUse() const { + return BaseCL::getRawArgUse(ARG_RHS); + } + + Use &getRawRHSUse() { return BaseCL::getRawArgUse(ARG_RHS); } + + /// This is just like getRawRHS, but it strips off any cast + /// instructions that feed it, giving the original input. The returned + /// value is guaranteed to be a pointer. + Value *getRHS() const { return BaseCL::getArg(ARG_RHS); } + + unsigned getRHSAddressSpace() const { + return BaseCL::getArgAddressSpace(ARG_RHS); + } + + /// FIXME: Remove this function once transition to Align is over. + /// Use getRHSAlign() instead. + unsigned getRHSAlignment() const { + return BaseCL::getArgAlignment(ARG_RHS); + } + + MaybeAlign getRHSAlign() const { return BaseCL::getArgAlign(ARG_RHS); } + + void setRHS(Value *Ptr) { BaseCL::setArg(ARG_RHS, Ptr); } + + /// FIXME: Remove this function once transition to Align is over. + /// Use the version that takes MaybeAlign instead of this one. + void setRHSAlignment(unsigned Alignment) { + BaseCL::setArgAlignment(ARG_RHS, Alignment); + } + + void setRHSAlignment(MaybeAlign Alignment) { + BaseCL::setArgAlignment(ARG_RHS, Alignment); + } + + void setRHSAlignment(Align Alignment) { + BaseCL::setArgAlignment(ARG_RHS, Alignment); + } +}; + // The common base class for the atomic memset/memmove/memcpy intrinsics // i.e. llvm.element.unordered.atomic.memset/memcpy/memmove class AtomicMemIntrinsic : public MemIntrinsicBase { @@ -789,6 +881,7 @@ case Intrinsic::memcpy_element_unordered_atomic: case Intrinsic::memmove_element_unordered_atomic: case Intrinsic::memset_element_unordered_atomic: + case Intrinsic::memcmp_element_unordered_atomic: return true; default: return false; @@ -799,6 +892,17 @@ } }; +class AtomicBinaryMemIntrinsic + : public BinaryMemIntrinsicBase { +public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::memcmp_element_unordered_atomic; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } +}; + // The common base class for all memory write intrinsics (memcpy/memset/memmove) template class MemWriteIntrinsicBase : public BaseCL { private: @@ -918,6 +1022,16 @@ } }; +class AtomicMemCmpInst : public AtomicBinaryMemIntrinsic { +public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::memcmp_element_unordered_atomic; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } +}; + /// This is the common base class for memset/memcpy/memmove. class MemIntrinsic : public MemIntrinsicBase { private: @@ -939,6 +1053,7 @@ case Intrinsic::memmove: case Intrinsic::memset: case Intrinsic::memcpy_inline: + case Intrinsic::memcmp: return true; default: return false; @@ -949,6 +1064,17 @@ } }; +class BinaryMemIntrinsic : public BinaryMemIntrinsicBase { +public: + // Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::memcmp; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } +}; + class MemWriteIntrinsic : public MemWriteIntrinsicBase { public: // Methods for support type inquiry through isa, cast, and dyn_cast: @@ -1024,6 +1150,17 @@ } }; +/// This class wraps the llvm.memcmp intrinsic. +class MemCmpInst : public BinaryMemIntrinsic { +public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::memcmp; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } +}; + /// This class wraps the llvm.memcpy.inline intrinsic. class MemCpyInlineInst : public MemCpyInst { public: @@ -1039,10 +1176,10 @@ } }; -// The common base class for any memset/memmove/memcpy intrinsics; +// The common base class for any memset/memmove/memcpy/memcmp intrinsics; // whether they be atomic or non-atomic. -// i.e. llvm.element.unordered.atomic.memset/memcpy/memmove -// and llvm.memset/memcpy/memmove +// i.e. llvm.element.unordered.atomic.memset/memcpy/memmove/memcmp +// and llvm.memset/memcpy/memmove/memcmp class AnyMemIntrinsic : public MemIntrinsicBase { public: bool isVolatile() const { @@ -1061,6 +1198,24 @@ case Intrinsic::memcpy_element_unordered_atomic: case Intrinsic::memmove_element_unordered_atomic: case Intrinsic::memset_element_unordered_atomic: + case Intrinsic::memcmp: + case Intrinsic::memcmp_element_unordered_atomic: + return true; + default: + return false; + } + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } +}; + +class AnyBinaryMemIntrinsic : public BinaryMemIntrinsicBase { +public: + static bool classof(const IntrinsicInst *I) { + switch (I->getIntrinsicID()) { + case Intrinsic::memcmp: + case Intrinsic::memcmp_element_unordered_atomic: return true; default: return false; diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -639,6 +639,13 @@ NoCapture>, WriteOnly>, ImmArg>]>; +def int_memcmp : Intrinsic<[llvm_i1_ty], + [llvm_anyptr_ty, llvm_anyptr_ty, llvm_anyint_ty, llvm_i1_ty], + [IntrReadMem, IntrArgMemOnly, IntrWillReturn, IntrNoFree, + NoCapture>, NoCapture>, + ReadOnly>, ReadOnly>, + ImmArg>]>; + // FIXME: Add version of these floating point intrinsics which allow non-default // rounding modes and FP exception handling. @@ -1687,6 +1694,14 @@ NoCapture>, WriteOnly>, ImmArg>]>; +// @llvm.memcmp.element.unordered.atomic.*(lhs, rhs, length, elementsize) +def int_memcmp_element_unordered_atomic : Intrinsic<[llvm_i1_ty], + [llvm_anyptr_ty, llvm_anyptr_ty, llvm_anyint_ty, llvm_i32_ty], + [IntrReadMem, IntrArgMemOnly, IntrWillReturn, IntrNoSync, + NoCapture>, NoCapture>, + ReadOnly>, ReadOnly>, + ImmArg>]>; + //===------------------------ Reduction Intrinsics ------------------------===// // let IntrProperties = [IntrNoMem] in { diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp --- a/llvm/lib/IR/IRBuilder.cpp +++ b/llvm/lib/IR/IRBuilder.cpp @@ -203,6 +203,44 @@ return CI; } +CallInst *IRBuilderBase::CreateMemCmp(Value *LHS, MaybeAlign LHSAlign, + Value *RHS, MaybeAlign RHSAlign, + Value *Size, bool isVolatile, + MDNode *TBAATag, MDNode *TBAAStructTag, + MDNode *ScopeTag, MDNode *NoAliasTag) { + LHS = getCastedInt8PtrValue(LHS); + RHS = getCastedInt8PtrValue(RHS); + + Value *Ops[] = {LHS, RHS, Size, getInt1(isVolatile)}; + Type *Tys[] = {LHS->getType(), RHS->getType(), Size->getType()}; + Module *M = BB->getParent()->getParent(); + Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcmp, Tys); + + CallInst *CI = createCallHelper(TheFn, Ops, this); + + auto *MWI = cast(CI); + if (LHSAlign) + MWI->setLHSAlignment(*LHSAlign); + if (RHSAlign) + MWI->setRHSAlignment(*RHSAlign); + + // Set the TBAA info if present. + if (TBAATag) + CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); + + // Set the TBAA Struct info if present. + if (TBAAStructTag) + CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag); + + if (ScopeTag) + CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); + + if (NoAliasTag) + CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); + + return CI; +} + CallInst *IRBuilderBase::CreateMemCpyInline( Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, Value *Size, bool IsVolatile, MDNode *TBAATag, MDNode *TBAAStructTag, @@ -282,6 +320,49 @@ return CI; } +CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCmp( + Value *LHS, Align LHSAlign, Value *RHS, Align RHSAlign, Value *Size, + uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag, + MDNode *ScopeTag, MDNode *NoAliasTag) { + assert(LHSAlign >= ElementSize && + "Pointer alignment must be at least element size"); + assert(RHSAlign >= ElementSize && + "Pointer alignment must be at least element size"); + LHS = getCastedInt8PtrValue(LHS); + RHS = getCastedInt8PtrValue(RHS); + + Value *Ops[] = {LHS, RHS, Size, getInt32(ElementSize)}; + Type *Tys[] = {LHS->getType(), RHS->getType(), Size->getType()}; + Module *M = BB->getParent()->getParent(); + Function *TheFn = Intrinsic::getDeclaration( + M, Intrinsic::memcmp_element_unordered_atomic, Tys); + + dbgs() << "TheFn = " << *TheFn; + + CallInst *CI = createCallHelper(TheFn, Ops, this); + + // Set the alignment of the pointer args. + auto *AMCI = cast(CI); + AMCI->setLHSAlignment(LHSAlign); + AMCI->setRHSAlignment(RHSAlign); + + // Set the TBAA info if present. + if (TBAATag) + CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); + + // Set the TBAA Struct info if present. + if (TBAAStructTag) + CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag); + + if (ScopeTag) + CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); + + if (NoAliasTag) + CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); + + return CI; +} + CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, Value *Size, bool isVolatile, diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -4670,16 +4670,26 @@ case Intrinsic::memcpy: case Intrinsic::memcpy_inline: case Intrinsic::memmove: + case Intrinsic::memcmp: case Intrinsic::memset: { - const auto *MWI = cast(&Call); auto IsValidAlignment = [&](unsigned Alignment) -> bool { return Alignment == 0 || isPowerOf2_32(Alignment); }; - Assert(IsValidAlignment(MWI->getDestAlignment()), - "alignment of arg 0 of memory intrinsic must be 0 or a power of 2", - Call); - if (const auto *MTI = dyn_cast(MWI)) { - Assert(IsValidAlignment(MTI->getSourceAlignment()), + if (const auto *MWI = dyn_cast(&Call)) { + Assert(IsValidAlignment(MWI->getDestAlignment()), + "alignment of arg 0 of memory intrinsic must be 0 or a power of 2", + Call); + if (const auto *MTI = dyn_cast(MWI)) { + Assert( + IsValidAlignment(MTI->getSourceAlignment()), + "alignment of arg 1 of memory intrinsic must be 0 or a power of 2", + Call); + } + } else if (const auto *BMI = dyn_cast(&Call)) { + Assert(IsValidAlignment(BMI->getLHSAlignment()), + "alignment of arg 0 of memory intrinsic must be 0 or a power of 2", + Call); + Assert(IsValidAlignment(BMI->getRHSAlignment()), "alignment of arg 1 of memory intrinsic must be 0 or a power of 2", Call); } @@ -4687,11 +4697,12 @@ } case Intrinsic::memcpy_element_unordered_atomic: case Intrinsic::memmove_element_unordered_atomic: + case Intrinsic::memcmp_element_unordered_atomic: case Intrinsic::memset_element_unordered_atomic: { - const auto *AMWI = cast(&Call); + const auto *AMI = cast(&Call); ConstantInt *ElementSizeCI = - cast(AMWI->getRawElementSizeInBytes()); + cast(AMI->getRawElementSizeInBytes()); const APInt &ElementSizeVal = ElementSizeCI->getValue(); Assert(ElementSizeVal.isPowerOf2(), "element size of the element-wise atomic memory intrinsic " @@ -4701,13 +4712,23 @@ auto IsValidAlignment = [&](uint64_t Alignment) { return isPowerOf2_64(Alignment) && ElementSizeVal.ule(Alignment); }; - uint64_t DstAlignment = AMWI->getDestAlignment(); - Assert(IsValidAlignment(DstAlignment), - "incorrect alignment of the destination argument", Call); - if (const auto *AMT = dyn_cast(AMWI)) { - uint64_t SrcAlignment = AMT->getSourceAlignment(); - Assert(IsValidAlignment(SrcAlignment), - "incorrect alignment of the source argument", Call); + + if (const auto *AMWI = dyn_cast(AMI)) { + uint64_t DstAlignment = AMWI->getDestAlignment(); + Assert(IsValidAlignment(DstAlignment), + "incorrect alignment of the destination argument", Call); + if (const auto *AMT = dyn_cast(AMWI)) { + uint64_t SrcAlignment = AMT->getSourceAlignment(); + Assert(IsValidAlignment(SrcAlignment), + "incorrect alignment of the source argument", Call); + } + } else if (const auto *ABMI = dyn_cast(AMI)) { + uint64_t LHSAlignment = ABMI->getLHSAlignment(); + Assert(IsValidAlignment(LHSAlignment), + "incorrect alignment of the LHS argument", Call); + uint64_t RHSAlignment = ABMI->getRHSAlignment(); + Assert(IsValidAlignment(RHSAlignment), + "incorrect alignment of the RHS argument", Call); } break; } diff --git a/llvm/test/Verifier/element-wise-atomic-memory-intrinsics.ll b/llvm/test/Verifier/element-wise-atomic-memory-intrinsics.ll --- a/llvm/test/Verifier/element-wise-atomic-memory-intrinsics.ll +++ b/llvm/test/Verifier/element-wise-atomic-memory-intrinsics.ll @@ -72,4 +72,29 @@ } declare void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* nocapture, i8, i32, i32) nounwind +define void @test_memcmp(i8* %P, i8* %Q, i32 %A, i32 %E) { + ; CHECK: immarg operand has non-immediate parameter + ; CHECK: i32 %E + ; CHECK-NEXT: call i1 @llvm.memcmp.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %P, i8* align 4 %Q, i32 1, i32 %E) + call i1 @llvm.memcmp.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %P, i8* align 4 %Q, i32 1, i32 %E) + + ; CHECK: element size of the element-wise atomic memory intrinsic must be a power of 2 + call i1 @llvm.memcmp.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %P, i8* align 4 %Q, i32 1, i32 3) + + call i1 @llvm.memcmp.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %P, i8* align 4 %Q, i32 7, i32 4) + + ; CHECK: incorrect alignment of the LHS argument + call i1 @llvm.memcmp.element.unordered.atomic.p0i8.p0i8.i32(i8* %P, i8* align 4 %Q, i32 1, i32 1) + ; CHECK: incorrect alignment of the LHS argument + call i1 @llvm.memcmp.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %P, i8* align 4 %Q, i32 4, i32 4) + + ; CHECK: incorrect alignment of the RHS argument + call i1 @llvm.memcmp.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %P, i8* %Q, i32 1, i32 1) + ; CHECK: incorrect alignment of the RHS argument + call i1 @llvm.memcmp.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %P, i8* align 1 %Q, i32 4, i32 4) + + ret void +} +declare i1 @llvm.memcmp.element.unordered.atomic.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i32) nounwind + ; CHECK: input module is broken!