[MSAN] Convert ActualFnStart to be a particular Instruction *, not BB

[guiand]

Summary

This allows us to add addtional instrumentation before the function start,
without splitting the first BB. Removes the BB splitting for KMSAN's prologue.

[eugenis]

llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp

1085–1086

Could you also change insertKmsanPrologue to get rid of BB splitting?
I'm thinking set ActualFnStart = F.getEntryBlock().getFirstNonPHI() in advance, then simply insert all the KMSan stuff into IRBuilder<>(ActualFnStart).

[guiand]

Remove BB splitting for KMSAN

[eugenis]

LGTM

[guiand]

Move removeUnreachableBlocks before inserting the prologue. This makes sure there's no issue with the ActualFnStart instruction being deleted before the visitor loop.

[eugenis]

LGTM++

llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp

 /// non-zero.
 struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   Function &F;
   MemorySanitizer &MS;
   SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
   ValueMap<Value*, Value*> ShadowMap, OriginMap;
   std::unique_ptr<VarArgHelper> VAHelper;
   const TargetLibraryInfo *TLI;
-  BasicBlock *ActualFnStart;
+  Instruction *ActualFnStart;
 
   // The following flags disable parts of MSan instrumentation based on
   // exclusion list contents and command-line options.
   bool InsertChecks;
   bool PropagateShadow;
   bool PoisonStack;
   bool PoisonUndef;
 
MemorySanitizerVisitor(Function &F, MemorySanitizer &MS,
                          const TargetLibraryInfo &TLI)
       : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)), TLI(&TLI) {
     bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeMemory);
     InsertChecks = SanitizeFunction;
     PropagateShadow = SanitizeFunction;
     PoisonStack = SanitizeFunction && ClPoisonStack;
     PoisonUndef = SanitizeFunction && ClPoisonUndef;
 
     MS.initializeCallbacks(*F.getParent());
-    if (MS.CompileKernel)
-      ActualFnStart = insertKmsanPrologue(F);
-    else
-      ActualFnStart = &F.getEntryBlock();
+    ActualFnStart = F.getEntryBlock().getFirstNonPHI();
+
+    if (MS.CompileKernel) {
+      IRBuilder<> IRB(ActualFnStart);
+      insertKmsanPrologue(IRB);
+    }
 
     LLVM_DEBUG(if (!InsertChecks) dbgs()
                << "MemorySanitizer is not inserting checks into '"
                << F.getName() << "'\n");
   }
 
   Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
     if (MS.TrackOrigins <= 1) return V;
for (const auto &ShadowData : InstrumentationList) {
       Instruction *OrigIns = ShadowData.OrigIns;
       Value *Shadow = ShadowData.Shadow;
       Value *Origin = ShadowData.Origin;
       materializeOneCheck(OrigIns, Shadow, Origin, InstrumentWithCalls);
     }
     LLVM_DEBUG(dbgs() << "DONE:\n" << F);
   }
 
-  BasicBlock *insertKmsanPrologue(Function &F) {
-    BasicBlock *ret =
-        SplitBlock(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHI());
-    IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
+  void insertKmsanPrologue(IRBuilder<> &IRB) {
     Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {});
     Constant *Zero = IRB.getInt32(0);
     MS.ParamTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
                                 {Zero, IRB.getInt32(0)}, "param_shadow");
     MS.RetvalTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
                                  {Zero, IRB.getInt32(1)}, "retval_shadow");
     MS.VAArgTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
                                 {Zero, IRB.getInt32(2)}, "va_arg_shadow");
     MS.VAArgOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
                                       {Zero, IRB.getInt32(3)}, "va_arg_origin");
     MS.VAArgOverflowSizeTLS =
         IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
                       {Zero, IRB.getInt32(4)}, "va_arg_overflow_size");
     MS.ParamOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
                                       {Zero, IRB.getInt32(5)}, "param_origin");
     MS.RetvalOriginTLS =
         IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
                       {Zero, IRB.getInt32(6)}, "retval_origin");
-    return ret;
   }
 
   /// Add MemorySanitizer instrumentation to a function.
   bool runOnFunction() {
+    BasicBlock *StartBB = ActualFnStart->getParent();
     // In the presence of unreachable blocks, we may see Phi nodes with
     // incoming nodes from such blocks. Since InstVisitor skips unreachable
     // blocks, such nodes will not have any shadow value associated with them.
     // It's easier to remove unreachable blocks than deal with missing shadow.
     removeUnreachableBlocks(F);
 
     // Iterate all BBs in depth-first order and create shadow instructions
     // for all instructions (where applicable).
     // For PHI nodes we create dummy shadow PHIs which will be finalized later.
-    for (BasicBlock *BB : depth_first(ActualFnStart))
+    for (BasicBlock *BB : depth_first(StartBB))
       visit(*BB);
 
     // Finalize PHI nodes.
     for (PHINode *PN : ShadowPHINodes) {
       PHINode *PNS = cast<PHINode>(getShadow(PN));
       PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
       size_t NumValues = PN->getNumIncomingValues();
       for (size_t v = 0; v < NumValues; v++) {
if (UndefValue *U = dyn_cast<UndefValue>(V)) {
       return AllOnes;
     }
     if (Argument *A = dyn_cast<Argument>(V)) {
       // For arguments we compute the shadow on demand and store it in the map.
       Value **ShadowPtr = &ShadowMap[V];
       if (*ShadowPtr)
         return *ShadowPtr;
       Function *F = A->getParent();
-      IRBuilder<> EntryIRB(ActualFnStart->getFirstNonPHI());
+      IRBuilder<> EntryIRB(ActualFnStart);
       unsigned ArgOffset = 0;
       const DataLayout &DL = F->getParent()->getDataLayout();
       for (auto &FArg : F->args()) {
         if (!FArg.getType()->isSized()) {
           LLVM_DEBUG(dbgs() << "Arg is not sized\n");
           continue;
         }
 
switch (a) {
         return AtomicOrdering::SequentiallyConsistent;
     }
     llvm_unreachable("Unknown ordering");
   }
 
   // ------------------- Visitors.
   using InstVisitor<MemorySanitizerVisitor>::visit;
   void visit(Instruction &I) {
-    if (!I.getMetadata("nosanitize"))
-      InstVisitor<MemorySanitizerVisitor>::visit(I);
+    if (I.getMetadata("nosanitize"))
+      return;
+    // Don't want to visit if we're in the zone before ActualFnStart
+    if (I.getParent() == ActualFnStart->getParent() &&
+        I.comesBefore(ActualFnStart))
+      return;
   }
 
   /// Instrument LoadInst
   ///
   /// Loads the corresponding shadow and (optionally) origin.
   /// Optionally, checks that the load address is fully defined.
   void visitLoadInst(LoadInst &I) {
     assert(I.getType()->isSized() && "Load type must have size");
struct VarArgAMD64Helper : public VarArgHelper {
   }
 
   void finalizeInstrumentation() override {
     assert(!VAArgOverflowSize && !VAArgTLSCopy &&
            "finalizeInstrumentation called twice");
     if (!VAStartInstrumentationList.empty()) {
       // If there is a va_start in this function, make a backup copy of
       // va_arg_tls somewhere in the function entry block.
-      IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
+      IRBuilder<> IRB(MSV.ActualFnStart);
       VAArgOverflowSize =
           IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
       Value *CopySize =
         IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset),
                       VAArgOverflowSize);
       VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
       IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
       if (MS.TrackOrigins) {
std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
         VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
     IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
                      /* size */ 8, Alignment, false);
   }
 
   void finalizeInstrumentation() override {
     assert(!VAArgSize && !VAArgTLSCopy &&
            "finalizeInstrumentation called twice");
-    IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
+    IRBuilder<> IRB(MSV.ActualFnStart);
     VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
     Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
                                     VAArgSize);
 
     if (!VAStartInstrumentationList.empty()) {
       // If there is a va_start in this function, make a backup copy of
       // va_arg_tls somewhere in the function entry block.
       VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
struct VarArgAArch64Helper : public VarArgHelper {
   }
 
   void finalizeInstrumentation() override {
     assert(!VAArgOverflowSize && !VAArgTLSCopy &&
            "finalizeInstrumentation called twice");
     if (!VAStartInstrumentationList.empty()) {
       // If there is a va_start in this function, make a backup copy of
       // va_arg_tls somewhere in the function entry block.
-      IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
+      IRBuilder<> IRB(MSV.ActualFnStart);
       VAArgOverflowSize =
           IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
       Value *CopySize =
         IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset),
                       VAArgOverflowSize);
       VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
       IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
     }
void visitVACopyInst(VACopyInst &I) override {
     // FIXME: magic ABI constants.
     IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
                      /* size */ 8, Alignment, false);
   }
 
   void finalizeInstrumentation() override {
     assert(!VAArgSize && !VAArgTLSCopy &&
            "finalizeInstrumentation called twice");
-    IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
+    IRBuilder<> IRB(MSV.ActualFnStart);
     VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
     Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
                                     VAArgSize);
 
     if (!VAStartInstrumentationList.empty()) {
       // If there is a va_start in this function, make a backup copy of
       // va_arg_tls somewhere in the function entry block.
       VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
struct VarArgSystemZHelper : public VarArgHelper {
   }
 
   void finalizeInstrumentation() override {
     assert(!VAArgOverflowSize && !VAArgTLSCopy &&
            "finalizeInstrumentation called twice");
     if (!VAStartInstrumentationList.empty()) {
       // If there is a va_start in this function, make a backup copy of
       // va_arg_tls somewhere in the function entry block.
-      IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
+      IRBuilder<> IRB(MSV.ActualFnStart);
       VAArgOverflowSize =
           IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
       Value *CopySize =
           IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, SystemZOverflowOffset),
                         VAArgOverflowSize);
       VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
       IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
       if (MS.TrackOrigins) {

llvm/test/Instrumentation/MemorySanitizer/array_types.ll

 define [2 x i32] @InsertValue(i32 %x, i32 %y) sanitize_memory {
 entry:
   %a = insertvalue [2 x i32] undef, i32 %x, 0
   %b = insertvalue [2 x i32] %a, i32 %y, 1
   ret [2 x i32] %b
 }
 
 ; CHECK-LABEL: @InsertValue(
-; CHECK-DAG: [[Sy:%.*]] = load i32, i32* {{.*}}@__msan_param_tls to i64), i64 8) to i32*)
 ; CHECK-DAG: [[Sx:%.*]] = load i32, i32* {{.*}}@__msan_param_tls to i32*)
 ; CHECK: [[A:%.*]] = insertvalue [2 x i32] [i32 -1, i32 -1], i32 [[Sx]], 0
+; CHECK-DAG: [[Sy:%.*]] = load i32, i32* {{.*}}@__msan_param_tls to i64), i64 8) to i32*)
 ; CHECK: [[B:%.*]] = insertvalue [2 x i32] [[A]], i32 [[Sy]], 1
 ; CHECK: store [2 x i32] [[B]], [2 x i32]* {{.*}}@__msan_retval_tls
 ; CHECK: ret [2 x i32]
 
 
 define [2 x double] @InsertValueDouble(double %x, double %y) sanitize_memory {
 entry:
   %a = insertvalue [2 x double] undef, double %x, 0
   %b = insertvalue [2 x double] %a, double %y, 1
   ret [2 x double] %b
 }
 
 ; CHECK-LABEL: @InsertValueDouble(
-; CHECK-DAG: [[Sy:%.*]] = load i64, i64* {{.*}}@__msan_param_tls to i64), i64 8) to i64*)
 ; CHECK-DAG: [[Sx:%.*]] = load i64, i64* getelementptr {{.*}}@__msan_param_tls, i32 0, i32 0
 ; CHECK: [[A:%.*]] = insertvalue [2 x i64] [i64 -1, i64 -1], i64 [[Sx]], 0
+; CHECK-DAG: [[Sy:%.*]] = load i64, i64* {{.*}}@__msan_param_tls to i64), i64 8) to i64*)
 ; CHECK: [[B:%.*]] = insertvalue [2 x i64] [[A]], i64 [[Sy]], 1
 ; CHECK: store [2 x i64] [[B]], [2 x i64]* {{.*}}@__msan_retval_tls
 ; CHECK: ret [2 x double]
 
 
 define i32 @ExtractValue([2 x i32] %a) sanitize_memory {
 entry:
   %x = extractvalue [2 x i32] %a, 1

llvm/test/Instrumentation/MemorySanitizer/clmul.ll

 define <2 x i64> @clmul00(<2 x i64> %a, <2 x i64> %b) sanitize_memory {
 entry:
   %0 = tail call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %a, <2 x i64> %b, i8 0)
   ret <2 x i64> %0
 }
 
 ; CHECK-LABEL: @clmul00
 ; CHECK: %[[S0:.*]] = load <2 x i64>, <2 x i64>* {{.*}}@__msan_param_tls
+; CHECK: %[[SHUF0:.*]] = shufflevector <2 x i64> %[[S0]], <2 x i64> undef, <2 x i32> zeroinitializer
 ; CHECK: %[[S1:.*]] = load <2 x i64>, <2 x i64>* {{.*}}@__msan_param_tls
 ; CHECK: %[[SHUF1:.*]] = shufflevector <2 x i64> %[[S1]], <2 x i64> undef, <2 x i32> zeroinitializer
-; CHECK: %[[SHUF0:.*]] = shufflevector <2 x i64> %[[S0]], <2 x i64> undef, <2 x i32> zeroinitializer
-; CHECK: %[[SRET:.*]] = or <2 x i64> %[[SHUF1]], %[[SHUF0]]
+; CHECK: %[[SRET:.*]] = or <2 x i64> %[[SHUF0]], %[[SHUF1]]
 ; CHECK: store <2 x i64> %[[SRET]], <2 x i64>* {{.*}}@__msan_retval_tls
 
 define <2 x i64> @clmul10(<2 x i64> %a, <2 x i64> %b) sanitize_memory {
 entry:
   %0 = tail call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %a, <2 x i64> %b, i8 16)
   ret <2 x i64> %0
 }
 
 ; CHECK-LABEL: @clmul10
 ; CHECK: %[[S0:.*]] = load <2 x i64>, <2 x i64>* {{.*}}@__msan_param_tls
+; CHECK: %[[SHUF0:.*]] = shufflevector <2 x i64> %[[S0]], <2 x i64> undef, <2 x i32> zeroinitializer
 ; CHECK: %[[S1:.*]] = load <2 x i64>, <2 x i64>* {{.*}}@__msan_param_tls
-; CHECK: %[[SHUF1:.*]] = shufflevector <2 x i64> %[[S1]], <2 x i64> undef, <2 x i32> zeroinitializer
-; CHECK: %[[SHUF0:.*]] = shufflevector <2 x i64> %[[S0]], <2 x i64> undef, <2 x i32> <i32 1, i32 1>
-; CHECK: %[[SRET:.*]] = or <2 x i64> %[[SHUF1]], %[[SHUF0]]
+; CHECK: %[[SHUF1:.*]] = shufflevector <2 x i64> %[[S1]], <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; CHECK: %[[SRET:.*]] = or <2 x i64> %[[SHUF0]], %[[SHUF1]]
 ; CHECK: store <2 x i64> %[[SRET]], <2 x i64>* {{.*}}@__msan_retval_tls
 
 define <4 x i64> @clmul11_256(<4 x i64> %a, <4 x i64> %b) sanitize_memory {
 entry:
   %0 = tail call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> %a, <4 x i64> %b, i8 17)
   ret <4 x i64> %0
 }
 
 ; CHECK-LABEL: @clmul11_256
 ; CHECK: %[[S0:.*]] = load <4 x i64>, <4 x i64>* {{.*}}@__msan_param_tls
+; CHECK: %[[SHUF0:.*]] = shufflevector <4 x i64> %[[S0]], <4 x i64> undef, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
 ; CHECK: %[[S1:.*]] = load <4 x i64>, <4 x i64>* {{.*}}@__msan_param_tls
 ; CHECK: %[[SHUF1:.*]] = shufflevector <4 x i64> %[[S1]], <4 x i64> undef, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
-; CHECK: %[[SHUF0:.*]] = shufflevector <4 x i64> %[[S0]], <4 x i64> undef, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
-; CHECK: %[[SRET:.*]] = or <4 x i64> %[[SHUF1]], %[[SHUF0]]
+; CHECK: %[[SRET:.*]] = or <4 x i64> %[[SHUF0]], %[[SHUF1]]
 ; CHECK: store <4 x i64> %[[SRET]], <4 x i64>* {{.*}}@__msan_retval_tls
 
 define <8 x i64> @clmul01_512(<8 x i64> %a, <8 x i64> %b) sanitize_memory {
 entry:
   %0 = tail call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> %a, <8 x i64> %b, i8 16)
   ret <8 x i64> %0
 }
 
 ; CHECK-LABEL: @clmul01_512
 ; CHECK: %[[S0:.*]] = load <8 x i64>, <8 x i64>* {{.*}}@__msan_param_tls
+; CHECK: %[[SHUF0:.*]] = shufflevector <8 x i64> %[[S0]], <8 x i64> undef, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
 ; CHECK: %[[S1:.*]] = load <8 x i64>, <8 x i64>* {{.*}}@__msan_param_tls
-; CHECK: %[[SHUF1:.*]] = shufflevector <8 x i64> %[[S1]], <8 x i64> undef, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
-; CHECK: %[[SHUF0:.*]] = shufflevector <8 x i64> %[[S0]], <8 x i64> undef, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
-; CHECK: %[[SRET:.*]] = or <8 x i64> %[[SHUF1]], %[[SHUF0]]
-; ORIGIN: %[[FLAT:.*]] = bitcast <8 x i64> %[[SHUF0]] to i512
+; CHECK: %[[SHUF1:.*]] = shufflevector <8 x i64> %[[S1]], <8 x i64> undef, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
+; CHECK: %[[SRET:.*]] = or <8 x i64> %[[SHUF0]], %[[SHUF1]]
+; ORIGIN: %[[FLAT:.*]] = bitcast <8 x i64> %[[SHUF1]] to i512
 ; ORIGIN: %[[I:.*]] = icmp ne i512 %[[FLAT]], 0
 ; ORIGIN: %[[O:.*]] = select i1 %[[I]],
 ; CHECK: store <8 x i64> %[[SRET]], <8 x i64>* {{.*}}@__msan_retval_tls
 ; ORIGIN: store i32 %[[O]], i32* @__msan_retval_origin_tls

llvm/test/Instrumentation/MemorySanitizer/masked-store-load.ll

 ; CHECK-ORIGIN: getelementptr i32, i32* %[[G]], i32 6
 ; CHECK-ORIGIN: store i32 %[[O]], i32* {{.*}}, align 4
 ; CHECK-ORIGIN: getelementptr i32, i32* %[[G]], i32 7
 ; CHECK-ORIGIN: store i32 %[[O]], i32* {{.*}}, align 4
 ; CHECK: tail call void @llvm.masked.store.v4i64.p0v4i64(<4 x i64> %v, <4 x i64>* %p, i32 1, <4 x i1> %mask)
 ; CHECK: ret void
 
 ; ADDR-LABEL: @Store(
-; ADDR: %[[MASKSHADOW:.*]] = load <4 x i1>, {{.*}}@__msan_param_tls to i64), i64 40)
 ; ADDR: %[[ADDRSHADOW:.*]] = load i64, {{.*}}[100 x i64]* @__msan_param_tls, i32 0, i32 0)
+; ADDR: %[[MASKSHADOW:.*]] = load <4 x i1>, {{.*}}@__msan_param_tls to i64), i64 40)
 
 ; ADDR: %[[ADDRBAD:.*]] = icmp ne i64 %[[ADDRSHADOW]], 0
 ; ADDR: br i1 %[[ADDRBAD]], label {{.*}}, label {{.*}}
 ; ADDR: call void @__msan_warning_with_origin_noreturn(i32 0)
 
 ; ADDR: %[[MASKSHADOWFLAT:.*]] = bitcast <4 x i1> %[[MASKSHADOW]] to i4
 ; ADDR: %[[MASKBAD:.*]] = icmp ne i4 %[[MASKSHADOWFLAT]], 0
 ; ADDR: br i1 %[[MASKBAD]], label {{.*}}, label {{.*}}
 ; ADDR: call void @__msan_warning_with_origin_noreturn(i32 0)
 
 ; ADDR: tail call void @llvm.masked.store.v4i64.p0v4i64(<4 x i64> %v, <4 x i64>* %p, i32 1, <4 x i1> %mask)
 ; ADDR: ret void
 
 
 define <4 x double> @Load(<4 x double>* %p, <4 x double> %v, <4 x i1> %mask) sanitize_memory {
 entry:
   %x = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* %p, i32 1, <4 x i1> %mask, <4 x double> %v)
   ret <4 x double> %x
 }
 
 ; CHECK-LABEL: @Load(
-; CHECK: %[[A:.*]] = load <4 x i64>, {{.*}}@__msan_param_tls to i64), i64 8)
-; CHECK-ORIGIN: %[[O:.*]] = load i32, {{.*}}@__msan_param_origin_tls to i64), i64 8)
 ; CHECK: %[[B:.*]] = ptrtoint <4 x double>* %p to i64
 ; CHECK: %[[C:.*]] = xor i64 %[[B]], 87960930222080
 ; CHECK: %[[D:.*]] = inttoptr i64 %[[C]] to <4 x i64>*
 ; CHECK-ORIGIN: %[[E:.*]] = add i64 %[[C]], 17592186044416
 ; CHECK-ORIGIN: %[[F:.*]] = and i64 %[[E]], -4
 ; CHECK-ORIGIN: %[[G:.*]] = inttoptr i64 %[[F]] to i32*
+; CHECK: %[[A:.*]] = load <4 x i64>, {{.*}}@__msan_param_tls to i64), i64 8)
+; CHECK-ORIGIN: %[[O:.*]] = load i32, {{.*}}@__msan_param_origin_tls to i64), i64 8)
 ; CHECK: %[[E:.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0v4i64(<4 x i64>* %[[D]], i32 1, <4 x i1> %mask, <4 x i64> %[[A]])
 ; CHECK-ORIGIN: %[[H:.*]] = load i32, i32* %[[G]]
 ; CHECK-ORIGIN: %[[O2:.*]] = select i1 %{{.*}}, i32 %[[O]], i32 %[[H]]
 ; CHECK: %[[X:.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* %p, i32 1, <4 x i1> %mask, <4 x double> %v)
 ; CHECK: store <4 x i64> %[[E]], {{.*}}@__msan_retval_tls
 ; CHECK-ORIGIN: store i32 %[[O2]], i32* @__msan_retval_origin_tls
 ; CHECK: ret <4 x double> %[[X]]
 
 ; ADDR-LABEL: @Load(
-; ADDR: %[[MASKSHADOW:.*]] = load <4 x i1>, {{.*}}@__msan_param_tls to i64), i64 40)
 ; ADDR: %[[ADDRSHADOW:.*]] = load i64, {{.*}}[100 x i64]* @__msan_param_tls, i32 0, i32 0)
+; ADDR: %[[MASKSHADOW:.*]] = load <4 x i1>, {{.*}}@__msan_param_tls to i64), i64 40)
 
 ; ADDR: %[[ADDRBAD:.*]] = icmp ne i64 %[[ADDRSHADOW]], 0
 ; ADDR: br i1 %[[ADDRBAD]], label {{.*}}, label {{.*}}
 ; ADDR: call void @__msan_warning_with_origin_noreturn(i32 0)
 
 ; ADDR: %[[MASKSHADOWFLAT:.*]] = bitcast <4 x i1> %[[MASKSHADOW]] to i4
 ; ADDR: %[[MASKBAD:.*]] = icmp ne i4 %[[MASKSHADOWFLAT]], 0
 ; ADDR: br i1 %[[MASKBAD]], label {{.*}}, label {{.*}}

llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll

 entry:
   %c = fdiv float %a, %b
   ret float %c
 }
 
 ; CHECK-LABEL: @FDiv
 ; CHECK: %[[SA:.*]] = load i32,{{.*}}@__msan_param_tls
 ; CHECK: %[[SB:.*]] = load i32,{{.*}}@__msan_param_tls
-; CHECK: %[[SC:.*]] = or i32 %[[SB]], %[[SA]]
+; CHECK: %[[SC:.*]] = or i32 %[[SA]], %[[SB]]
 ; CHECK: = fdiv float
 ; CHECK: store i32 %[[SC]], i32* {{.*}}@__msan_retval_tls
 ; CHECK: ret float
 
 ; Check that fneg simply propagates shadow.
 
 define float @FNeg(float %a) nounwind uwtable readnone sanitize_memory {
 entry:

llvm/test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll

 ; %va_arg_origin:
 ; CHECK: getelementptr {{.*}} i32 0, i32 3
 ; %va_arg_overflow_size:
 ; CHECK: getelementptr {{.*}} i32 0, i32 4
 ; %param_origin:
 ; CHECK: getelementptr {{.*}} i32 0, i32 5
 ; %retval_origin:
 ; CHECK: getelementptr {{.*}} i32 0, i32 6
-; CHECK: entry.split:
 
 ; Check instrumentation of stores
 
 define void @Store1(i8* nocapture %p, i8 %x) nounwind uwtable sanitize_memory {
 entry:
   store i8 %x, i8* %p
   ret void
 }
 
 ; CHECK-LABEL: @Store1
 ; CHECK: entry:
 ; CHECK: @__msan_get_context_state()
 ; CHECK: [[PARAM_SHADOW:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 0
-; CHECK: entry.split:
-; CHECK: [[BASE2:%[0-9]+]] = ptrtoint {{.*}} [[PARAM_SHADOW]]
 ; CHECK: [[BASE:%[0-9]+]] = ptrtoint {{.*}} [[PARAM_SHADOW]]
-; CHECK: [[SHADOW:%[a-z0-9_]+]] = inttoptr {{.*}} [[BASE]]
+; CHECK: [[SHADOW_PTR:%[a-z0-9_]+]] = inttoptr {{.*}} [[BASE]]
+; CHECK: [[SHADOW:%[a-z0-9]+]] = load i64, i64* [[SHADOW_PTR]]
 ; Load the shadow of %p and check it
-; CHECK: load i64, i64* [[SHADOW]]
-; CHECK: icmp
+; CHECK: icmp ne i64 [[SHADOW]]
 ; CHECK: br i1
 ; CHECK: {{^[0-9]+}}:
+; CHECK: [[BASE2:%[0-9]+]] = ptrtoint {{.*}} [[PARAM_SHADOW]]
 ; CHECK: @__msan_metadata_ptr_for_store_1(i8* %p)
 ; CHECK: store i8
 ; If the new shadow is non-zero, jump to __msan_chain_origin()
 ; CHECK: icmp
 ; CHECK: br i1
 ; CHECK: {{^[0-9]+}}:
 ; CHECK: @__msan_chain_origin
 ; Storing origin here:
 ; CHECK: store i32
 ; CHECK: br label
 ; CHECK: {{^[0-9]+}}:
 ; CHECK: store i8
 ; CHECK: ret void
 
 define void @Store2(i16* nocapture %p, i16 %x) nounwind uwtable sanitize_memory {
 entry:
   store i16 %x, i16* %p
   ret void
 }
 
 ; CHECK-LABEL: @Store2
 ; CHECK: entry:
 ; CHECK: @__msan_get_context_state()
 ; CHECK: [[PARAM_SHADOW:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 0
-; CHECK: entry.split:
 ; CHECK: ptrtoint {{.*}} [[PARAM_SHADOW]]
 ; Load the shadow of %p and check it
 ; CHECK: load i64
 ; CHECK: icmp
 ; CHECK: br i1
 ; CHECK: {{^[0-9]+}}:
 ; CHECK: [[REG:%[0-9]+]] = bitcast i16* %p to i8*
 ; CHECK: @__msan_metadata_ptr_for_store_2(i8* [[REG]])
entry:
   store i32 %x, i32* %p
   ret void
 }
 
 ; CHECK-LABEL: @Store4
 ; CHECK: entry:
 ; CHECK: @__msan_get_context_state()
 ; CHECK: [[PARAM_SHADOW:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 0
-; CHECK: entry.split:
 ; CHECK: ptrtoint {{.*}} [[PARAM_SHADOW]]
 ; Load the shadow of %p and check it
 ; CHECK: load i32
 ; CHECK: icmp
 ; CHECK: br i1
 ; CHECK: {{^[0-9]+}}:
 ; CHECK: [[REG:%[0-9]+]] = bitcast i32* %p to i8*
 ; CHECK: @__msan_metadata_ptr_for_store_4(i8* [[REG]])
entry:
   store i64 %x, i64* %p
   ret void
 }
 
 ; CHECK-LABEL: @Store8
 ; CHECK: entry:
 ; CHECK: @__msan_get_context_state()
 ; CHECK: [[PARAM_SHADOW:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 0
-; CHECK: entry.split:
 ; CHECK: ptrtoint {{.*}} [[PARAM_SHADOW]]
 ; Load the shadow of %p and check it
 ; CHECK: load i64
 ; CHECK: icmp
 ; CHECK: br i1
 ; CHECK: {{^[0-9]+}}:
 ; CHECK: [[REG:%[0-9]+]] = bitcast i64* %p to i8*
 ; CHECK: @__msan_metadata_ptr_for_store_8(i8* [[REG]])
entry:
   store i128 %x, i128* %p
   ret void
 }
 
 ; CHECK-LABEL: @Store16
 ; CHECK: entry:
 ; CHECK: @__msan_get_context_state()
 ; CHECK: [[PARAM_SHADOW:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 0
-; CHECK: entry.split:
 ; CHECK: ptrtoint {{.*}} [[PARAM_SHADOW]]
 ; Load the shadow of %p and check it
 ; CHECK: load i64
 ; CHECK: icmp
 ; CHECK: br i1
 ; CHECK: {{^[0-9]+}}:
 ; CHECK: [[REG:%[0-9]+]] = bitcast i128* %p to i8*
 ; CHECK: @__msan_metadata_ptr_for_store_n(i8* [[REG]], i64 16)
entry:
   %0 = load i8, i8* %p
   ret i8 %0
 }
 
 ; CHECK-LABEL: @Load1
 ; CHECK: entry:
 ; CHECK: @__msan_get_context_state()
 ; CHECK: [[PARAM_SHADOW:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 0
-; CHECK: entry.split:
 ; CHECK: ptrtoint {{.*}} [[PARAM_SHADOW]]
 ; Load the shadow of %p and check it
 ; CHECK: load i64
 ; CHECK: icmp
 ; CHECK: br i1
 ; CHECK: {{^[0-9]+}}:
 ; Load the value from %p. This is done before accessing the shadow
 ; to ease atomic handling.
entry:
   %0 = load i16, i16* %p
   ret i16 %0
 }
 
 ; CHECK-LABEL: @Load2
 ; CHECK: entry:
 ; CHECK: @__msan_get_context_state()
 ; CHECK: [[PARAM_SHADOW:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 0
-; CHECK: entry.split:
 ; CHECK: ptrtoint {{.*}} [[PARAM_SHADOW]]
 ; Load the shadow of %p and check it
 ; CHECK: load i64
 ; CHECK: icmp
 ; CHECK: br i1
 ; CHECK: {{^[0-9]+}}:
 ; Load the value from %p. This is done before accessing the shadow
 ; to ease atomic handling.
entry:
   %0 = load i32, i32* %p
   ret i32 %0
 }
 
 ; CHECK-LABEL: @Load4
 ; CHECK: entry:
 ; CHECK: @__msan_get_context_state()
 ; CHECK: [[PARAM_SHADOW:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 0
-; CHECK: entry.split:
 ; CHECK: ptrtoint {{.*}} [[PARAM_SHADOW]]
 ; Load the shadow of %p and check it
 ; CHECK: load i64
 ; CHECK: icmp
 ; CHECK: br i1
 ; CHECK: {{^[0-9]+}}:
 ; Load the value from %p. This is done before accessing the shadow
 ; to ease atomic handling.
entry:
   %0 = load i64, i64* %p
   ret i64 %0
 }
 
 ; CHECK-LABEL: @Load8
 ; CHECK: entry:
 ; CHECK: @__msan_get_context_state()
 ; CHECK: [[PARAM_SHADOW:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 0
-; CHECK: entry.split:
 ; CHECK: ptrtoint {{.*}} [[PARAM_SHADOW]]
 ; Load the shadow of %p and check it
 ; CHECK: load i64
 ; CHECK: icmp
 ; CHECK: br i1
 ; CHECK: {{^[0-9]+}}:
 ; Load the value from %p. This is done before accessing the shadow
 ; to ease atomic handling.
entry:
   %0 = load i128, i128* %p
   ret i128 %0
 }
 
 ; CHECK-LABEL: @Load16
 ; CHECK: entry:
 ; CHECK: @__msan_get_context_state()
 ; CHECK: [[PARAM_SHADOW:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 0
-; CHECK: entry.split:
 ; CHECK: ptrtoint {{.*}} [[PARAM_SHADOW]]
 ; Load the shadow of %p and check it
 ; CHECK: load i64
 ; CHECK: icmp
 ; CHECK: br i1
 ; CHECK: {{^[0-9]+}}:
 ; Load the value from %p. This is done before accessing the shadow
 ; to ease atomic handling.

 attributes #0 = { "target-features"="+fxsr,+x87,-sse" }
 
 ; CHECK-LABEL: @VarArgFn
 ; CHECK: @__msan_get_context_state()
 ; CHECK: [[VA_ARG_SHADOW:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 2
 ; CHECK: [[VA_ARG_ORIGIN:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 3
 ; CHECK: [[VA_ARG_OVERFLOW_SIZE:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 4
 
-; CHECK: entry.split:
 ; CHECK: [[OSIZE:%[0-9]+]] = load i64, i64* [[VA_ARG_OVERFLOW_SIZE]]
 ; Register save area is 48 bytes for non-SSE builds.
 ; CHECK: [[SIZE:%[0-9]+]] = add i64 48, [[OSIZE]]
 ; CHECK: [[SHADOWS:%[0-9]+]] = alloca i8, i64 [[SIZE]]
 ; CHECK: [[VA_ARG_SHADOW]]
 ; CHECK: call void @llvm.memcpy{{.*}}(i8* align 8 [[SHADOWS]], {{.*}}, i64 [[SIZE]]
 ; CHECK: [[ORIGINS:%[0-9]+]] = alloca i8, i64 [[SIZE]]
 ; CHECK: [[VA_ARG_ORIGIN]]

 ; CHECK-LABEL: @VarArgCaller
 
 ; CHECK: entry:
 ; CHECK: @__msan_get_context_state()
 ; CHECK: [[PARAM_SHADOW:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 0
 ; CHECK: [[VA_ARG_SHADOW:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 2
 ; CHECK: [[VA_ARG_OVERFLOW_SIZE:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 4
 
-; CHECK: entry.split:
 ; CHECK: [[PARAM_SI:%[_a-z0-9]+]] = ptrtoint {{.*}} [[PARAM_SHADOW]]
 ; CHECK: [[ARG1_S:%[_a-z0-9]+]] = inttoptr i64 [[PARAM_SI]] to i64*
 ; First argument is initialized
 ; CHECK: store i64 0, i64* [[ARG1_S]]
 
 ; Dangling cast of va_arg_shadow[0], unused because the first argument is fixed.
 ; CHECK: [[VA_CAST0:%[_a-z0-9]+]] = ptrtoint {{.*}} [[VA_ARG_SHADOW]] to i64
 

llvm/test/Instrumentation/MemorySanitizer/pr32842.ll

 ; Regression test for https://bugs.llvm.org/show_bug.cgi?id=32842
 ;
 ; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -S | FileCheck %s
 ;target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define zeroext i1 @_Z1fii(i32 %x, i32 %y) sanitize_memory {
 entry:
   %cmp = icmp slt i32 %x, %y
   ret i1 %cmp
 }
 
 ; CHECK:      [[X:[^ ]+]] = load{{.*}}__msan_param_tls{{.*}}
 ; CHECK:      [[Y:[^ ]+]] = load{{.*}}__msan_param_tls{{.*}}
-; CHECK:      [[OR:[^ ]+]] = or i32 [[Y]], [[X]]
+; CHECK:      [[OR:[^ ]+]] = or i32 [[X]], [[Y]]
 
 ; Make sure the shadow of the (x < y) comparison isn't truncated to i1.
 ; CHECK-NOT:  trunc i32 [[OR]] to i1
 ; CHECK:      [[CMP:[^ ]+]] = icmp ne i32 [[OR]], 0
 ; CHECK:      store i1 [[CMP]],{{.*}}__msan_retval_tls

llvm/test/Instrumentation/MemorySanitizer/store-origin.ll

 !22 = !DILocation(line: 3, scope: !4)
 
 
 ; CHECK-LABEL: @Store
 
 ; CHECK-MSAN: load {{.*}} @__msan_param_tls
 ; CHECK-MSAN: [[ORIGIN:%[0-9a-z]+]] = load {{.*}} @__msan_param_origin_tls
 
-; CHECK-KMSAN-LABEL: entry.split:
 ; CHECK-KMSAN: %param_shadow
 ; CHECK-KMSAN: load i32, i32*
 ; CHECK-KMSAN: %param_origin
 ; CHECK-KMSAN: [[ORIGIN:%[0-9a-z]+]] = load i32, i32*
 
 ; CHECK: store {{.*}}!dbg ![[DBG:[0-9]+]]
 ; CHECK: icmp
 ; CHECK: br i1