diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp --- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -1050,7 +1050,7 @@ ValueMap ShadowMap, OriginMap; std::unique_ptr VAHelper; const TargetLibraryInfo *TLI; - BasicBlock *ActualFnStart; + Instruction *ActualFnStart; // The following flags disable parts of MSan instrumentation based on // exclusion list contents and command-line options. @@ -1084,9 +1084,9 @@ MS.initializeCallbacks(*F.getParent()); if (MS.CompileKernel) - ActualFnStart = insertKmsanPrologue(F); + ActualFnStart = insertKmsanPrologue(F)->getFirstNonPHI(); else - ActualFnStart = &F.getEntryBlock(); + ActualFnStart = F.getEntryBlock().getFirstNonPHI(); LLVM_DEBUG(if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '" @@ -1291,7 +1291,7 @@ // Iterate all BBs in depth-first order and create shadow instructions // for all instructions (where applicable). // For PHI nodes we create dummy shadow PHIs which will be finalized later. - for (BasicBlock *BB : depth_first(ActualFnStart)) + for (BasicBlock *BB : depth_first(ActualFnStart->getParent())) visit(*BB); // Finalize PHI nodes. @@ -1658,7 +1658,7 @@ if (*ShadowPtr) return *ShadowPtr; Function *F = A->getParent(); - IRBuilder<> EntryIRB(ActualFnStart->getFirstNonPHI()); + IRBuilder<> EntryIRB(ActualFnStart); unsigned ArgOffset = 0; const DataLayout &DL = F->getParent()->getDataLayout(); for (auto &FArg : F->args()) { @@ -1838,8 +1838,13 @@ // ------------------- Visitors. using InstVisitor::visit; void visit(Instruction &I) { - if (!I.getMetadata("nosanitize")) - InstVisitor::visit(I); + if (I.getMetadata("nosanitize")) + return; + // Don't want to visit if we're in the zone before ActualFnStart + if (I.getParent() == ActualFnStart->getParent() && + I.comesBefore(ActualFnStart)) + return; + InstVisitor::visit(I); } /// Instrument LoadInst @@ -4185,7 +4190,7 @@ if (!VAStartInstrumentationList.empty()) { // If there is a va_start in this function, make a backup copy of // va_arg_tls somewhere in the function entry block. - IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); + IRBuilder<> IRB(MSV.ActualFnStart); VAArgOverflowSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); Value *CopySize = @@ -4331,7 +4336,7 @@ void finalizeInstrumentation() override { assert(!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"); - IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); + IRBuilder<> IRB(MSV.ActualFnStart); VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize); @@ -4524,7 +4529,7 @@ if (!VAStartInstrumentationList.empty()) { // If there is a va_start in this function, make a backup copy of // va_arg_tls somewhere in the function entry block. - IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); + IRBuilder<> IRB(MSV.ActualFnStart); VAArgOverflowSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); Value *CopySize = @@ -4769,7 +4774,7 @@ void finalizeInstrumentation() override { assert(!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"); - IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); + IRBuilder<> IRB(MSV.ActualFnStart); VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize); @@ -5088,7 +5093,7 @@ if (!VAStartInstrumentationList.empty()) { // If there is a va_start in this function, make a backup copy of // va_arg_tls somewhere in the function entry block. - IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); + IRBuilder<> IRB(MSV.ActualFnStart); VAArgOverflowSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); Value *CopySize = diff --git a/llvm/test/Instrumentation/MemorySanitizer/array_types.ll b/llvm/test/Instrumentation/MemorySanitizer/array_types.ll --- a/llvm/test/Instrumentation/MemorySanitizer/array_types.ll +++ b/llvm/test/Instrumentation/MemorySanitizer/array_types.ll @@ -17,9 +17,9 @@ } ; CHECK-LABEL: @InsertValue( -; CHECK-DAG: [[Sy:%.*]] = load i32, i32* {{.*}}@__msan_param_tls to i64), i64 8) to i32*) ; CHECK-DAG: [[Sx:%.*]] = load i32, i32* {{.*}}@__msan_param_tls to i32*) ; CHECK: [[A:%.*]] = insertvalue [2 x i32] [i32 -1, i32 -1], i32 [[Sx]], 0 +; CHECK-DAG: [[Sy:%.*]] = load i32, i32* {{.*}}@__msan_param_tls to i64), i64 8) to i32*) ; CHECK: [[B:%.*]] = insertvalue [2 x i32] [[A]], i32 [[Sy]], 1 ; CHECK: store [2 x i32] [[B]], [2 x i32]* {{.*}}@__msan_retval_tls ; CHECK: ret [2 x i32] @@ -33,9 +33,9 @@ } ; CHECK-LABEL: @InsertValueDouble( -; CHECK-DAG: [[Sy:%.*]] = load i64, i64* {{.*}}@__msan_param_tls to i64), i64 8) to i64*) ; CHECK-DAG: [[Sx:%.*]] = load i64, i64* getelementptr {{.*}}@__msan_param_tls, i32 0, i32 0 ; CHECK: [[A:%.*]] = insertvalue [2 x i64] [i64 -1, i64 -1], i64 [[Sx]], 0 +; CHECK-DAG: [[Sy:%.*]] = load i64, i64* {{.*}}@__msan_param_tls to i64), i64 8) to i64*) ; CHECK: [[B:%.*]] = insertvalue [2 x i64] [[A]], i64 [[Sy]], 1 ; CHECK: store [2 x i64] [[B]], [2 x i64]* {{.*}}@__msan_retval_tls ; CHECK: ret [2 x double] diff --git a/llvm/test/Instrumentation/MemorySanitizer/clmul.ll b/llvm/test/Instrumentation/MemorySanitizer/clmul.ll --- a/llvm/test/Instrumentation/MemorySanitizer/clmul.ll +++ b/llvm/test/Instrumentation/MemorySanitizer/clmul.ll @@ -19,10 +19,10 @@ ; CHECK-LABEL: @clmul00 ; CHECK: %[[S0:.*]] = load <2 x i64>, <2 x i64>* {{.*}}@__msan_param_tls +; CHECK: %[[SHUF0:.*]] = shufflevector <2 x i64> %[[S0]], <2 x i64> undef, <2 x i32> zeroinitializer ; CHECK: %[[S1:.*]] = load <2 x i64>, <2 x i64>* {{.*}}@__msan_param_tls ; CHECK: %[[SHUF1:.*]] = shufflevector <2 x i64> %[[S1]], <2 x i64> undef, <2 x i32> zeroinitializer -; CHECK: %[[SHUF0:.*]] = shufflevector <2 x i64> %[[S0]], <2 x i64> undef, <2 x i32> zeroinitializer -; CHECK: %[[SRET:.*]] = or <2 x i64> %[[SHUF1]], %[[SHUF0]] +; CHECK: %[[SRET:.*]] = or <2 x i64> %[[SHUF0]], %[[SHUF1]] ; CHECK: store <2 x i64> %[[SRET]], <2 x i64>* {{.*}}@__msan_retval_tls define <2 x i64> @clmul10(<2 x i64> %a, <2 x i64> %b) sanitize_memory { @@ -33,10 +33,10 @@ ; CHECK-LABEL: @clmul10 ; CHECK: %[[S0:.*]] = load <2 x i64>, <2 x i64>* {{.*}}@__msan_param_tls +; CHECK: %[[SHUF0:.*]] = shufflevector <2 x i64> %[[S0]], <2 x i64> undef, <2 x i32> zeroinitializer ; CHECK: %[[S1:.*]] = load <2 x i64>, <2 x i64>* {{.*}}@__msan_param_tls -; CHECK: %[[SHUF1:.*]] = shufflevector <2 x i64> %[[S1]], <2 x i64> undef, <2 x i32> zeroinitializer -; CHECK: %[[SHUF0:.*]] = shufflevector <2 x i64> %[[S0]], <2 x i64> undef, <2 x i32> -; CHECK: %[[SRET:.*]] = or <2 x i64> %[[SHUF1]], %[[SHUF0]] +; CHECK: %[[SHUF1:.*]] = shufflevector <2 x i64> %[[S1]], <2 x i64> undef, <2 x i32> +; CHECK: %[[SRET:.*]] = or <2 x i64> %[[SHUF0]], %[[SHUF1]] ; CHECK: store <2 x i64> %[[SRET]], <2 x i64>* {{.*}}@__msan_retval_tls define <4 x i64> @clmul11_256(<4 x i64> %a, <4 x i64> %b) sanitize_memory { @@ -47,10 +47,10 @@ ; CHECK-LABEL: @clmul11_256 ; CHECK: %[[S0:.*]] = load <4 x i64>, <4 x i64>* {{.*}}@__msan_param_tls +; CHECK: %[[SHUF0:.*]] = shufflevector <4 x i64> %[[S0]], <4 x i64> undef, <4 x i32> ; CHECK: %[[S1:.*]] = load <4 x i64>, <4 x i64>* {{.*}}@__msan_param_tls ; CHECK: %[[SHUF1:.*]] = shufflevector <4 x i64> %[[S1]], <4 x i64> undef, <4 x i32> -; CHECK: %[[SHUF0:.*]] = shufflevector <4 x i64> %[[S0]], <4 x i64> undef, <4 x i32> -; CHECK: %[[SRET:.*]] = or <4 x i64> %[[SHUF1]], %[[SHUF0]] +; CHECK: %[[SRET:.*]] = or <4 x i64> %[[SHUF0]], %[[SHUF1]] ; CHECK: store <4 x i64> %[[SRET]], <4 x i64>* {{.*}}@__msan_retval_tls define <8 x i64> @clmul01_512(<8 x i64> %a, <8 x i64> %b) sanitize_memory { @@ -61,11 +61,11 @@ ; CHECK-LABEL: @clmul01_512 ; CHECK: %[[S0:.*]] = load <8 x i64>, <8 x i64>* {{.*}}@__msan_param_tls +; CHECK: %[[SHUF0:.*]] = shufflevector <8 x i64> %[[S0]], <8 x i64> undef, <8 x i32> ; CHECK: %[[S1:.*]] = load <8 x i64>, <8 x i64>* {{.*}}@__msan_param_tls -; CHECK: %[[SHUF1:.*]] = shufflevector <8 x i64> %[[S1]], <8 x i64> undef, <8 x i32> -; CHECK: %[[SHUF0:.*]] = shufflevector <8 x i64> %[[S0]], <8 x i64> undef, <8 x i32> -; CHECK: %[[SRET:.*]] = or <8 x i64> %[[SHUF1]], %[[SHUF0]] -; ORIGIN: %[[FLAT:.*]] = bitcast <8 x i64> %[[SHUF0]] to i512 +; CHECK: %[[SHUF1:.*]] = shufflevector <8 x i64> %[[S1]], <8 x i64> undef, <8 x i32> +; CHECK: %[[SRET:.*]] = or <8 x i64> %[[SHUF0]], %[[SHUF1]] +; ORIGIN: %[[FLAT:.*]] = bitcast <8 x i64> %[[SHUF1]] to i512 ; ORIGIN: %[[I:.*]] = icmp ne i512 %[[FLAT]], 0 ; ORIGIN: %[[O:.*]] = select i1 %[[I]], ; CHECK: store <8 x i64> %[[SRET]], <8 x i64>* {{.*}}@__msan_retval_tls diff --git a/llvm/test/Instrumentation/MemorySanitizer/masked-store-load.ll b/llvm/test/Instrumentation/MemorySanitizer/masked-store-load.ll --- a/llvm/test/Instrumentation/MemorySanitizer/masked-store-load.ll +++ b/llvm/test/Instrumentation/MemorySanitizer/masked-store-load.ll @@ -49,8 +49,8 @@ ; CHECK: ret void ; ADDR-LABEL: @Store( -; ADDR: %[[MASKSHADOW:.*]] = load <4 x i1>, {{.*}}@__msan_param_tls to i64), i64 40) ; ADDR: %[[ADDRSHADOW:.*]] = load i64, {{.*}}[100 x i64]* @__msan_param_tls, i32 0, i32 0) +; ADDR: %[[MASKSHADOW:.*]] = load <4 x i1>, {{.*}}@__msan_param_tls to i64), i64 40) ; ADDR: %[[ADDRBAD:.*]] = icmp ne i64 %[[ADDRSHADOW]], 0 ; ADDR: br i1 %[[ADDRBAD]], label {{.*}}, label {{.*}} @@ -72,14 +72,14 @@ } ; CHECK-LABEL: @Load( -; CHECK: %[[A:.*]] = load <4 x i64>, {{.*}}@__msan_param_tls to i64), i64 8) -; CHECK-ORIGIN: %[[O:.*]] = load i32, {{.*}}@__msan_param_origin_tls to i64), i64 8) ; CHECK: %[[B:.*]] = ptrtoint <4 x double>* %p to i64 ; CHECK: %[[C:.*]] = xor i64 %[[B]], 87960930222080 ; CHECK: %[[D:.*]] = inttoptr i64 %[[C]] to <4 x i64>* ; CHECK-ORIGIN: %[[E:.*]] = add i64 %[[C]], 17592186044416 ; CHECK-ORIGIN: %[[F:.*]] = and i64 %[[E]], -4 ; CHECK-ORIGIN: %[[G:.*]] = inttoptr i64 %[[F]] to i32* +; CHECK: %[[A:.*]] = load <4 x i64>, {{.*}}@__msan_param_tls to i64), i64 8) +; CHECK-ORIGIN: %[[O:.*]] = load i32, {{.*}}@__msan_param_origin_tls to i64), i64 8) ; CHECK: %[[E:.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0v4i64(<4 x i64>* %[[D]], i32 1, <4 x i1> %mask, <4 x i64> %[[A]]) ; CHECK-ORIGIN: %[[H:.*]] = load i32, i32* %[[G]] ; CHECK-ORIGIN: %[[O2:.*]] = select i1 %{{.*}}, i32 %[[O]], i32 %[[H]] @@ -89,8 +89,8 @@ ; CHECK: ret <4 x double> %[[X]] ; ADDR-LABEL: @Load( -; ADDR: %[[MASKSHADOW:.*]] = load <4 x i1>, {{.*}}@__msan_param_tls to i64), i64 40) ; ADDR: %[[ADDRSHADOW:.*]] = load i64, {{.*}}[100 x i64]* @__msan_param_tls, i32 0, i32 0) +; ADDR: %[[MASKSHADOW:.*]] = load <4 x i1>, {{.*}}@__msan_param_tls to i64), i64 40) ; ADDR: %[[ADDRBAD:.*]] = icmp ne i64 %[[ADDRSHADOW]], 0 ; ADDR: br i1 %[[ADDRBAD]], label {{.*}}, label {{.*}} diff --git a/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll b/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll --- a/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll +++ b/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll @@ -436,7 +436,7 @@ ; CHECK-LABEL: @FDiv ; CHECK: %[[SA:.*]] = load i32,{{.*}}@__msan_param_tls ; CHECK: %[[SB:.*]] = load i32,{{.*}}@__msan_param_tls -; CHECK: %[[SC:.*]] = or i32 %[[SB]], %[[SA]] +; CHECK: %[[SC:.*]] = or i32 %[[SA]], %[[SB]] ; CHECK: = fdiv float ; CHECK: store i32 %[[SC]], i32* {{.*}}@__msan_retval_tls ; CHECK: ret float diff --git a/llvm/test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll b/llvm/test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll --- a/llvm/test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll +++ b/llvm/test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll @@ -44,14 +44,14 @@ ; CHECK: @__msan_get_context_state() ; CHECK: [[PARAM_SHADOW:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 0 ; CHECK: entry.split: -; CHECK: [[BASE2:%[0-9]+]] = ptrtoint {{.*}} [[PARAM_SHADOW]] ; CHECK: [[BASE:%[0-9]+]] = ptrtoint {{.*}} [[PARAM_SHADOW]] -; CHECK: [[SHADOW:%[a-z0-9_]+]] = inttoptr {{.*}} [[BASE]] +; CHECK: [[SHADOW_PTR:%[a-z0-9_]+]] = inttoptr {{.*}} [[BASE]] +; CHECK: [[SHADOW:%[a-z0-9]+]] = load i64, i64* [[SHADOW_PTR]] ; Load the shadow of %p and check it -; CHECK: load i64, i64* [[SHADOW]] -; CHECK: icmp +; CHECK: icmp ne i64 [[SHADOW]] ; CHECK: br i1 ; CHECK: {{^[0-9]+}}: +; CHECK: [[BASE2:%[0-9]+]] = ptrtoint {{.*}} [[PARAM_SHADOW]] ; CHECK: @__msan_metadata_ptr_for_store_1(i8* %p) ; CHECK: store i8 ; If the new shadow is non-zero, jump to __msan_chain_origin() diff --git a/llvm/test/Instrumentation/MemorySanitizer/pr32842.ll b/llvm/test/Instrumentation/MemorySanitizer/pr32842.ll --- a/llvm/test/Instrumentation/MemorySanitizer/pr32842.ll +++ b/llvm/test/Instrumentation/MemorySanitizer/pr32842.ll @@ -13,7 +13,7 @@ ; CHECK: [[X:[^ ]+]] = load{{.*}}__msan_param_tls{{.*}} ; CHECK: [[Y:[^ ]+]] = load{{.*}}__msan_param_tls{{.*}} -; CHECK: [[OR:[^ ]+]] = or i32 [[Y]], [[X]] +; CHECK: [[OR:[^ ]+]] = or i32 [[X]], [[Y]] ; Make sure the shadow of the (x < y) comparison isn't truncated to i1. ; CHECK-NOT: trunc i32 [[OR]] to i1