diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -29,6 +29,7 @@ #include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/BranchProbability.h" #include "llvm/Support/InstructionCost.h" +#include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h" #include #include #include @@ -884,6 +885,9 @@ MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const; + // Return InterestingMemoryOperand of Intrinsic \p II. + InterestingMemoryOperand getInterstingMemoryOperand(IntrinsicInst *II) const; + /// Should the Select Optimization pass be enabled and ran. bool enableSelectOptimize() const; @@ -1785,6 +1789,8 @@ virtual bool enableAggressiveInterleaving(bool LoopHasReductions) = 0; virtual MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const = 0; + virtual InterestingMemoryOperand + getInterstingMemoryOperand(IntrinsicInst *II) const = 0; virtual bool enableSelectOptimize() = 0; virtual bool enableInterleavedAccessVectorization() = 0; virtual bool enableMaskedInterleavedAccessVectorization() = 0; @@ -2285,6 +2291,12 @@ bool IsZeroCmp) const override { return Impl.enableMemCmpExpansion(OptSize, IsZeroCmp); } + + InterestingMemoryOperand + getInterstingMemoryOperand(IntrinsicInst *II) const override { + return Impl.getInterstingMemoryOperand(II); + } + bool enableInterleavedAccessVectorization() override { return Impl.enableInterleavedAccessVectorization(); } diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -359,6 +359,10 @@ return {}; } + InterestingMemoryOperand getInterstingMemoryOperand(IntrinsicInst *II) const { + return InterestingMemoryOperand(); + } + bool enableSelectOptimize() const { return true; } bool enableInterleavedAccessVectorization() const { return false; } diff --git a/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h b/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h --- a/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h +++ b/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h @@ -24,7 +24,7 @@ class InterestingMemoryOperand { public: - Use *PtrUse; + Use *PtrUse = nullptr; bool IsWrite; Type *OpType; TypeSize TypeStoreSize = TypeSize::Fixed(0); @@ -34,6 +34,7 @@ // The EVL Value, if we're looking at a vp intrinsic. Value *MaybeEVL; + InterestingMemoryOperand() = default; InterestingMemoryOperand(Instruction *I, unsigned OperandNo, bool IsWrite, class Type *OpType, MaybeAlign Alignment, Value *MaybeMask = nullptr, @@ -46,8 +47,8 @@ } Instruction *getInsn() { return cast(PtrUse->getUser()); } - Value *getPtr() { return PtrUse->get(); } + operator bool() { return PtrUse != nullptr; } }; // Get AddressSanitizer parameters. diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -571,6 +571,11 @@ return TTIImpl->enableMemCmpExpansion(OptSize, IsZeroCmp); } +InterestingMemoryOperand +TargetTransformInfo::getInterstingMemoryOperand(IntrinsicInst *II) const { + return TTIImpl->getInterstingMemoryOperand(II); +} + bool TargetTransformInfo::enableSelectOptimize() const { return TTIImpl->enableSelectOptimize(); } diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h @@ -22,6 +22,7 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/CodeGen/BasicTTIImpl.h" #include "llvm/IR/Function.h" +#include "llvm/IR/IntrinsicsRISCV.h" #include namespace llvm { @@ -60,6 +61,7 @@ : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)), TLI(ST->getTargetLowering()) {} + InterestingMemoryOperand getInterstingMemoryOperand(IntrinsicInst *II) const; /// Return the cost of materializing an immediate for a value operand of /// a store instruction. InstructionCost getStoreImmCost(Type *VecTy, TTI::OperandValueInfo OpInfo, diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -34,6 +34,31 @@ "SLP vectorizer. Defaults to 1 which disables SLP."), cl::init(1), cl::Hidden); +InterestingMemoryOperand +RISCVTTIImpl::getInterstingMemoryOperand(IntrinsicInst *II) const { + const DataLayout &DL = getDataLayout(); + unsigned IntNo = II->getIntrinsicID(); + + switch (IntNo) { + case Intrinsic::riscv_vle: + case Intrinsic::riscv_vle_mask: + case Intrinsic::riscv_vse: + case Intrinsic::riscv_vse_mask: { + bool IsWrite = II->getType()->isVoidTy(); + Type *Ty = IsWrite ? II->getArgOperand(0)->getType() : II->getType(); + MaybeAlign Alignment = II->getArgOperand(1)->getPointerAlignment(DL); + const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo); + Value *Mask = ConstantInt::get(Ty->getWithNewBitWidth(1), 1); + if (IntNo == Intrinsic::riscv_vle_mask || + IntNo == Intrinsic::riscv_vse_mask) + Mask = II->getArgOperand(RVVIInfo->VLOperand - 1); + Value *EVL = II->getArgOperand(RVVIInfo->VLOperand); + return InterestingMemoryOperand(II, 1, IsWrite, Ty, Alignment, Mask, EVL); + } + } + return InterestingMemoryOperand(); +} + InstructionCost RISCVTTIImpl::getLMULCost(MVT VT) { // TODO: Here assume reciprocal throughput is 1 for LMUL_1, it is // implementation-defined. diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp --- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -29,6 +29,7 @@ #include "llvm/Analysis/MemoryBuiltins.h" #include "llvm/Analysis/StackSafetyAnalysis.h" #include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/BinaryFormat/MachO.h" #include "llvm/Demangle/Demangle.h" @@ -677,7 +678,8 @@ bool ignoreAccess(Instruction *Inst, Value *Ptr); void getInterestingMemoryOperands( - Instruction *I, SmallVectorImpl &Interesting); + Instruction *I, SmallVectorImpl &Interesting, + const TargetTransformInfo *TTI); void instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis, InterestingMemoryOperand &O, bool UseCalls, @@ -703,7 +705,8 @@ void instrumentMemIntrinsic(MemIntrinsic *MI); Value *memToShadow(Value *Shadow, IRBuilder<> &IRB); bool suppressInstrumentationSiteForDebug(int &Instrumented); - bool instrumentFunction(Function &F, const TargetLibraryInfo *TLI); + bool instrumentFunction(Function &F, const TargetLibraryInfo *TLI, + const TargetTransformInfo *TTI); bool maybeInsertAsanInitAtFunctionEntry(Function &F); bool maybeInsertDynamicShadowAtFunctionEntry(Function &F); void markEscapedLocalAllocas(Function &F); @@ -1157,7 +1160,8 @@ Options.Recover, Options.UseAfterScope, Options.UseAfterReturn); const TargetLibraryInfo &TLI = FAM.getResult(F); - Modified |= FunctionSanitizer.instrumentFunction(F, &TLI); + const TargetTransformInfo &TTI = FAM.getResult(F); + Modified |= FunctionSanitizer.instrumentFunction(F, &TLI, &TTI); } Modified |= ModuleSanitizer.instrumentModule(M); if (!Modified) @@ -1294,11 +1298,19 @@ } void AddressSanitizer::getInterestingMemoryOperands( - Instruction *I, SmallVectorImpl &Interesting) { + Instruction *I, SmallVectorImpl &Interesting, + const TargetTransformInfo *TTI) { // Do not instrument the load fetching the dynamic shadow address. if (LocalDynamicShadow == I) return; + if (auto *II = dyn_cast(I)) { + if (InterestingMemoryOperand IMO = TTI->getInterstingMemoryOperand(II)) { + Interesting.push_back(IMO); + return; + } + } + if (LoadInst *LI = dyn_cast(I)) { if (!ClInstrumentReads || ignoreAccess(I, LI->getPointerOperand())) return; @@ -2715,7 +2727,8 @@ } bool AddressSanitizer::instrumentFunction(Function &F, - const TargetLibraryInfo *TLI) { + const TargetLibraryInfo *TLI, + const TargetTransformInfo *TTI) { if (F.empty()) return false; if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage) return false; @@ -2768,7 +2781,7 @@ if (Inst.hasMetadata(LLVMContext::MD_nosanitize)) continue; SmallVector InterestingOperands; - getInterestingMemoryOperands(&Inst, InterestingOperands); + getInterestingMemoryOperands(&Inst, InterestingOperands, TTI); if (!InterestingOperands.empty()) { for (auto &Operand : InterestingOperands) { diff --git a/llvm/test/Instrumentation/AddressSanitizer/RISCV/asan-rvv-intrinsics.ll b/llvm/test/Instrumentation/AddressSanitizer/RISCV/asan-rvv-intrinsics.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Instrumentation/AddressSanitizer/RISCV/asan-rvv-intrinsics.ll @@ -0,0 +1,168 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -mtriple=riscv64 -mattr=+v -passes=asan \ +; RUN: -asan-instrumentation-with-call-threshold=0 -S | FileCheck %s + +declare @llvm.riscv.vle.nxv1i32( + , + *, + i64) +define @intrinsic_vle_v_nxv1i32_nxv1i32(* %0, i64 %1) sanitize_address { +; CHECK-LABEL: @intrinsic_vle_v_nxv1i32_nxv1i32( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i64 [[TMP1:%.*]], 0 +; CHECK-NEXT: br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP11:%.*]] +; CHECK: 3: +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 [[TMP4]]) +; CHECK-NEXT: br label [[DOTSPLIT:%.*]] +; CHECK: .split: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[TMP3]] ], [ [[IV_NEXT:%.*]], [[TMP10:%.*]] ] +; CHECK-NEXT: [[TMP6:%.*]] = extractelement shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i64 [[IV]] +; CHECK-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP10]] +; CHECK: 7: +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr , ptr [[TMP0:%.*]], i64 0, i64 [[IV]] +; CHECK-NEXT: [[TMP9:%.*]] = ptrtoint ptr [[TMP8]] to i64 +; CHECK-NEXT: call void @__asan_loadN(i64 [[TMP9]], i64 4) +; CHECK-NEXT: br label [[TMP10]] +; CHECK: 10: +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[IV_CHECK:%.*]] = icmp eq i64 [[IV_NEXT]], [[TMP5]] +; CHECK-NEXT: br i1 [[IV_CHECK]], label [[DOTSPLIT_SPLIT:%.*]], label [[DOTSPLIT]] +; CHECK: .split.split: +; CHECK-NEXT: br label [[TMP11]] +; CHECK: 11: +; CHECK-NEXT: [[A:%.*]] = call @llvm.riscv.vle.nxv1i32.i64( undef, ptr [[TMP0]], i64 [[TMP1]]) +; CHECK-NEXT: ret [[A]] +; +entry: + %a = call @llvm.riscv.vle.nxv1i32( + undef, + * %0, + i64 %1) + ret %a +} + +declare @llvm.riscv.vle.mask.nxv1i32( + , + *, + , + i64, + i64) +define @intrinsic_vle_mask_v_nxv1i32_nxv1i32( %0, * %1, %2, i64 %3) sanitize_address { +; CHECK-LABEL: @intrinsic_vle_mask_v_nxv1i32_nxv1i32( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP4:%.*]] = icmp ne i64 [[TMP3:%.*]], 0 +; CHECK-NEXT: br i1 [[TMP4]], label [[TMP5:%.*]], label [[TMP13:%.*]] +; CHECK: 5: +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP3]], i64 [[TMP6]]) +; CHECK-NEXT: br label [[DOTSPLIT:%.*]] +; CHECK: .split: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[TMP5]] ], [ [[IV_NEXT:%.*]], [[TMP12:%.*]] ] +; CHECK-NEXT: [[TMP8:%.*]] = extractelement [[TMP2:%.*]], i64 [[IV]] +; CHECK-NEXT: br i1 [[TMP8]], label [[TMP9:%.*]], label [[TMP12]] +; CHECK: 9: +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr , ptr [[TMP1:%.*]], i64 0, i64 [[IV]] +; CHECK-NEXT: [[TMP11:%.*]] = ptrtoint ptr [[TMP10]] to i64 +; CHECK-NEXT: call void @__asan_loadN(i64 [[TMP11]], i64 4) +; CHECK-NEXT: br label [[TMP12]] +; CHECK: 12: +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[IV_CHECK:%.*]] = icmp eq i64 [[IV_NEXT]], [[TMP7]] +; CHECK-NEXT: br i1 [[IV_CHECK]], label [[DOTSPLIT_SPLIT:%.*]], label [[DOTSPLIT]] +; CHECK: .split.split: +; CHECK-NEXT: br label [[TMP13]] +; CHECK: 13: +; CHECK-NEXT: [[A:%.*]] = call @llvm.riscv.vle.mask.nxv1i32.i64( [[TMP0:%.*]], ptr [[TMP1]], [[TMP2]], i64 [[TMP3]], i64 1) +; CHECK-NEXT: ret [[A]] +; +entry: + %a = call @llvm.riscv.vle.mask.nxv1i32( + %0, + * %1, + %2, + i64 %3, i64 1) + ret %a +} + +declare void @llvm.riscv.vse.nxv1i32( + , + *, + i64) +define void @intrinsic_vse_v_nxv1i32_nxv1i32( %0, * %1, i64 %2) sanitize_address { +; CHECK-LABEL: @intrinsic_vse_v_nxv1i32_nxv1i32( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP3:%.*]] = icmp ne i64 [[TMP2:%.*]], 0 +; CHECK-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP12:%.*]] +; CHECK: 4: +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP2]], i64 [[TMP5]]) +; CHECK-NEXT: br label [[DOTSPLIT:%.*]] +; CHECK: .split: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[TMP4]] ], [ [[IV_NEXT:%.*]], [[TMP11:%.*]] ] +; CHECK-NEXT: [[TMP7:%.*]] = extractelement shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i64 [[IV]] +; CHECK-NEXT: br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP11]] +; CHECK: 8: +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr , ptr [[TMP1:%.*]], i64 0, i64 [[IV]] +; CHECK-NEXT: [[TMP10:%.*]] = ptrtoint ptr [[TMP9]] to i64 +; CHECK-NEXT: call void @__asan_storeN(i64 [[TMP10]], i64 4) +; CHECK-NEXT: br label [[TMP11]] +; CHECK: 11: +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[IV_CHECK:%.*]] = icmp eq i64 [[IV_NEXT]], [[TMP6]] +; CHECK-NEXT: br i1 [[IV_CHECK]], label [[DOTSPLIT_SPLIT:%.*]], label [[DOTSPLIT]] +; CHECK: .split.split: +; CHECK-NEXT: br label [[TMP12]] +; CHECK: 12: +; CHECK-NEXT: call void @llvm.riscv.vse.nxv1i32.i64( [[TMP0:%.*]], ptr [[TMP1]], i64 [[TMP2]]) +; CHECK-NEXT: ret void +; +entry: + call void @llvm.riscv.vse.nxv1i32( + %0, + * %1, + i64 %2) + ret void +} + +declare void @llvm.riscv.vse.mask.nxv1i32( + , + *, + , + i64) +define void @intrinsic_vse_mask_v_nxv1i32_nxv1i32( %0, * %1, %2, i64 %3) sanitize_address { +; CHECK-LABEL: @intrinsic_vse_mask_v_nxv1i32_nxv1i32( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP4:%.*]] = icmp ne i64 [[TMP3:%.*]], 0 +; CHECK-NEXT: br i1 [[TMP4]], label [[TMP5:%.*]], label [[TMP13:%.*]] +; CHECK: 5: +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP3]], i64 [[TMP6]]) +; CHECK-NEXT: br label [[DOTSPLIT:%.*]] +; CHECK: .split: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[TMP5]] ], [ [[IV_NEXT:%.*]], [[TMP12:%.*]] ] +; CHECK-NEXT: [[TMP8:%.*]] = extractelement [[TMP2:%.*]], i64 [[IV]] +; CHECK-NEXT: br i1 [[TMP8]], label [[TMP9:%.*]], label [[TMP12]] +; CHECK: 9: +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr , ptr [[TMP1:%.*]], i64 0, i64 [[IV]] +; CHECK-NEXT: [[TMP11:%.*]] = ptrtoint ptr [[TMP10]] to i64 +; CHECK-NEXT: call void @__asan_storeN(i64 [[TMP11]], i64 4) +; CHECK-NEXT: br label [[TMP12]] +; CHECK: 12: +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[IV_CHECK:%.*]] = icmp eq i64 [[IV_NEXT]], [[TMP7]] +; CHECK-NEXT: br i1 [[IV_CHECK]], label [[DOTSPLIT_SPLIT:%.*]], label [[DOTSPLIT]] +; CHECK: .split.split: +; CHECK-NEXT: br label [[TMP13]] +; CHECK: 13: +; CHECK-NEXT: call void @llvm.riscv.vse.mask.nxv1i32.i64( [[TMP0:%.*]], ptr [[TMP1]], [[TMP2]], i64 [[TMP3]]) +; CHECK-NEXT: ret void +; +entry: + call void @llvm.riscv.vse.mask.nxv1i32( + %0, + * %1, + %2, + i64 %3) + ret void +}