Index: include/llvm/Transforms/Instrumentation.h
===================================================================
--- include/llvm/Transforms/Instrumentation.h
+++ include/llvm/Transforms/Instrumentation.h
@@ -127,7 +127,8 @@
 
 // Insert MemorySanitizer instrumentation (detection of uninitialized reads)
 FunctionPass *createMemorySanitizerPass(int TrackOrigins = 0,
-                                        bool Recover = false);
+                                        bool Recover = false,
+                                        bool EnableKmsan = false);
 
 // Insert ThreadSanitizer (race detection) instrumentation
 FunctionPass *createThreadSanitizerPass();
Index: lib/Transforms/Instrumentation/MemorySanitizer.cpp
===================================================================
--- lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -89,7 +89,28 @@
 /// implementation ignores the load aspect of CAS/RMW, always returning a clean
 /// value. It implements the store part as a simple atomic store by storing a
 /// clean shadow.
-//
+///
+///                  KernelMemorySanitizer (KMSAN) implementation.
+///
+/// The major differences between KMSAN and MSan instrumentation are:
+///  - KMSAN implies msan-track-origins=2, msan-keep-going=true;
+///  - KMSAN allocates shadows and origins for each page separately, so there
+///    are no explicit accesses to shadow and origin memory.
+///    Shadow and origin values for a particular X-byte memory location
+///    (X=1,2,4,8) are accessed via pointers obtained via the
+///      __msan_metadata_ptr_for_load_X(ptr)
+///      __msan_metadata_ptr_for_store_X(ptr)
+///    functions. The corresponding functions check that the X-byte accesses
+///    are possible and returns the pointers to shadow and origin memory.
+///      __msan_metadata_ptr_for_load_n
+///      __msan_metadata_ptr_for_store_X
+///  - TLS variables are stored in a single struct in per-task storage. A call
+///    to a function __msan_get_context_state() returning a pointer to that
+///    struct is inserted into every instrumented function before the entry block;
+///  - __msan_warning() now becomes __msan_warning_32(uptr origin)
+///
+///  KernelMemorySanitizer only supports X86_64 at the moment.
+///
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/APInt.h"
@@ -199,6 +220,11 @@
        cl::desc("exact handling of relational integer ICmp"),
        cl::Hidden, cl::init(false));
 
+static cl::opt<bool>
+    ClEnableKmsan("msan-kernel",
+                  cl::desc("Enable KernelMemorySanitizer instrumentation"),
+                  cl::Hidden, cl::init(false));
+
 // This flag controls whether we check the shadow of the address
 // operand of load or store. Such bugs are very rare, since load from
 // a garbage address typically results in SEGV, but still happen
@@ -357,11 +383,12 @@
   // Pass identification, replacement for typeid.
   static char ID; 
 
-  MemorySanitizer(int TrackOrigins = 0, bool Recover = false)
-      : FunctionPass(ID),
-        TrackOrigins(std::max(TrackOrigins, (int)ClTrackOrigins)),
-        Recover(Recover || ClKeepGoing) {}
-
+  MemorySanitizer(int TrackOrigins = 0, bool Recover = false,
+                  bool EnableKmsan = false)
+      : FunctionPass(ID), CompileKernel(EnableKmsan || ClEnableKmsan),
+        TrackOrigins(
+            CompileKernel ? 2 : std::max(TrackOrigins, (int)ClTrackOrigins)),
+        Recover(Recover || ClKeepGoing || CompileKernel), WarningFn(nullptr) {}
   StringRef getPassName() const override { return "MemorySanitizer"; }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
@@ -379,6 +406,11 @@
   friend struct VarArgPowerPC64Helper;
 
   void initializeCallbacks(Module &M);
+  void createKernelApi(Module &M);
+  void createUserspaceApi(Module &M);
+
+  /// \brief True if we're compiling the Linux kernel.
+  bool CompileKernel;
 
   /// \brief Track origins (allocation points) of uninitialized values.
   int TrackOrigins;
@@ -389,32 +421,38 @@
   Type *OriginTy;
 
   /// \brief Thread-local shadow storage for function parameters.
-  GlobalVariable *ParamTLS;
+  Value *ParamTLS;
 
   /// \brief Thread-local origin storage for function parameters.
-  GlobalVariable *ParamOriginTLS;
+  Value *ParamOriginTLS;
 
   /// \brief Thread-local shadow storage for function return value.
-  GlobalVariable *RetvalTLS;
+  Value *RetvalTLS;
 
   /// \brief Thread-local origin storage for function return value.
-  GlobalVariable *RetvalOriginTLS;
+  Value *RetvalOriginTLS;
 
   /// \brief Thread-local shadow storage for in-register va_arg function
   /// parameters (x86_64-specific).
-  GlobalVariable *VAArgTLS;
+  Value *VAArgTLS;
+
+  // \brief Thread-local shadow storage for in-register va_arg function
+  // parameters (x86_64-specific, KMSAN only).
+  Value *VAArgOriginTLS;
 
   /// \brief Thread-local shadow storage for va_arg overflow area
   /// (x86_64-specific).
-  GlobalVariable *VAArgOverflowSizeTLS;
+  Value *VAArgOverflowSizeTLS;
 
   /// \brief Thread-local space used to pass origin value to the UMR reporting
   /// function.
-  GlobalVariable *OriginTLS;
+  Value *OriginTLS;
 
   /// \brief The run-time callback to print a warning.
   Value *WarningFn = nullptr;
 
+  // \brief KMSAN-specific error callback that takes the origin.
+  Value *MsanWarning32Fn;
   // These arrays are indexed by log2(AccessSize).
   Value *MaybeWarningFn[kNumberOfAccessSizes];
   Value *MaybeStoreOriginFn[kNumberOfAccessSizes];
@@ -423,6 +461,14 @@
   /// allocation.
   Value *MsanSetAllocaOrigin4Fn;
 
+  Value *getKmsanShadowOriginAccessFn(bool isStore, int size);
+
+  Value *MsanPoisonAllocaFn;
+  Value *MsanUnpoisonFn;
+  Value *MsanLoadArgShadowFn, *MsanLoadArgOriginFn;
+  Value *MsanStoreArgShadowFn;
+  Value *MsanStoreArgShadowOriginFn;
+
   /// \brief Run-time helper that poisons stack on function entry.
   Value *MsanPoisonStackFn;
 
@@ -433,6 +479,14 @@
   /// \brief MSan runtime replacements for memmove, memcpy and memset.
   Value *MemmoveFn, *MemcpyFn, *MemsetFn;
 
+  /// \brief KMSAN callback for task-local function argument shadow.
+  Value *GetContextStateFn;
+
+  /// \brief function returning a pair of shadow/origin pointers.
+  Value *MsanMetadataPtrForLoadN, *MsanMetadataPtrForStoreN;
+  Value *MsanMetadataPtrForLoad_1_8[4];
+  Value *MsanMetadataPtrForStore_1_8[4];
+
   /// \brief Memory map parameters used in application-to-shadow calculation.
   const MemoryMapParams *MapParams;
 
@@ -459,8 +513,9 @@
     MemorySanitizer, "msan",
     "MemorySanitizer: detects uninitialized reads.", false, false)
 
-FunctionPass *llvm::createMemorySanitizerPass(int TrackOrigins, bool Recover) {
-  return new MemorySanitizer(TrackOrigins, Recover);
+FunctionPass *llvm::createMemorySanitizerPass(int TrackOrigins, bool Recover,
+                                              bool CompileKernel) {
+  return new MemorySanitizer(TrackOrigins, Recover, CompileKernel);
 }
 
 /// \brief Create a non-const global initialized with the given string.
@@ -475,13 +530,94 @@
                             GlobalValue::PrivateLinkage, StrConst, "");
 }
 
-/// \brief Insert extern declaration of runtime-provided functions and globals.
-void MemorySanitizer::initializeCallbacks(Module &M) {
-  // Only do this once.
-  if (WarningFn)
-    return;
+/// \brief Create KMSAN API callbacks.
+void MemorySanitizer::createKernelApi(Module &M) {
+  IRBuilder<> IRB(*C);
+
+  // These will be initialized in insertKmsanPrologue().
+  RetvalTLS = nullptr;
+  RetvalOriginTLS = nullptr;
+  ParamTLS = nullptr;
+  ParamOriginTLS = nullptr;
+  VAArgTLS = nullptr;
+  VAArgOriginTLS = nullptr;
+  VAArgOverflowSizeTLS = nullptr;
+  // OriginTLS is unused in the kernel.
+  OriginTLS = nullptr;
+
+  // Like __msan_warning(), but takes an origin.
+  MsanWarning32Fn = M.getOrInsertFunction("__msan_warning_32", IRB.getVoidTy(),
+                                          IRB.getInt32Ty());
+  // Requests the per-task context state (kmsan_context_state*) from the
+  // runtime library.
+  GetContextStateFn = M.getOrInsertFunction(
+      "__msan_get_context_state",
+      PointerType::get(
+          StructType::get(ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
+                          ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8),
+                          ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
+                          ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),  /* va_arg_origin */
+                          IRB.getInt64Ty(),
+                          ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy,
+                          OriginTy),
+          0));
+
+  Type *RetTy = StructType::get(
+      PointerType::get(IRB.getInt8Ty(), 0),
+      PointerType::get(IRB.getInt32Ty(), 0));
+
+  for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) {
+    std::string name_load =
+        "__msan_metadata_ptr_for_load_" + std::to_string(size);
+    std::string name_store =
+        "__msan_metadata_ptr_for_store_" + std::to_string(size);
+    MsanMetadataPtrForLoad_1_8[ind] = M.getOrInsertFunction(
+        name_load, RetTy,
+        PointerType::get(IRB.getInt8Ty(), 0));
+    MsanMetadataPtrForStore_1_8[ind] = M.getOrInsertFunction(
+        name_store, RetTy,
+        PointerType::get(IRB.getInt8Ty(), 0));
+  }
+
+  MsanMetadataPtrForLoadN = M.getOrInsertFunction(
+      "__msan_metadata_ptr_for_load_n", RetTy,
+      PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
+  MsanMetadataPtrForStoreN = M.getOrInsertFunction(
+      "__msan_metadata_ptr_for_store_n", RetTy,
+      PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
+
+  // Functions for poisoning and unpoisoning memory.
+  MsanPoisonAllocaFn = M.getOrInsertFunction(
+      "__msan_poison_alloca", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy,
+      IRB.getInt8PtrTy(), IntptrTy);
+
+  MsanUnpoisonFn = M.getOrInsertFunction("__msan_unpoison", IRB.getVoidTy(),
+                                         IRB.getInt8PtrTy(), IntptrTy);
+
+  // Functions for loading the shadow for a given memory range into an app
+  // memory buffer and storing it back to the shadow memory.
+  MsanLoadArgShadowFn =
+      M.getOrInsertFunction("__msan_load_arg_shadow", IRB.getVoidTy(),
+                            IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
+  MsanLoadArgOriginFn =
+      M.getOrInsertFunction("__msan_load_arg_origin", IRB.getVoidTy(),
+                            IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
+
+  MsanStoreArgShadowFn =
+      M.getOrInsertFunction("__msan_store_arg_shadow", IRB.getVoidTy(),
+                            IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
+
+  // Functions for loading the origin for a given memory range into an app
+  // memory buffer and storing it back to the origin memory.
+  MsanStoreArgShadowOriginFn =
+      M.getOrInsertFunction("__msan_store_arg_shadow_origin", IRB.getVoidTy(),
+                            IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
 
+}
+
+void MemorySanitizer::createUserspaceApi(Module &M) {
   IRBuilder<> IRB(*C);
+
   // Create the callback.
   // FIXME: this function should have "Cold" calling conv,
   // which is not yet implemented.
@@ -489,6 +625,39 @@
                                     : "__msan_warning_noreturn";
   WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
 
+  // Create the global TLS variables.
+  RetvalTLS = new GlobalVariable(
+      M, ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), false,
+      GlobalVariable::ExternalLinkage, nullptr, "__msan_retval_tls", nullptr,
+      GlobalVariable::InitialExecTLSModel);
+
+  RetvalOriginTLS = new GlobalVariable(
+      M, OriginTy, false, GlobalVariable::ExternalLinkage, nullptr,
+      "__msan_retval_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
+
+  ParamTLS = new GlobalVariable(
+      M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
+      GlobalVariable::ExternalLinkage, nullptr, "__msan_param_tls", nullptr,
+      GlobalVariable::InitialExecTLSModel);
+
+  ParamOriginTLS = new GlobalVariable(
+      M, ArrayType::get(OriginTy, kParamTLSSize / 4), false,
+      GlobalVariable::ExternalLinkage, nullptr, "__msan_param_origin_tls",
+      nullptr, GlobalVariable::InitialExecTLSModel);
+
+  VAArgTLS = new GlobalVariable(
+      M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
+      GlobalVariable::ExternalLinkage, nullptr, "__msan_va_arg_tls", nullptr,
+      GlobalVariable::InitialExecTLSModel);
+  VAArgOverflowSizeTLS = new GlobalVariable(
+      M, IRB.getInt64Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
+      "__msan_va_arg_overflow_size_tls", nullptr,
+      GlobalVariable::InitialExecTLSModel);
+  OriginTLS = new GlobalVariable(
+      M, IRB.getInt32Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
+      "__msan_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
+
+  // Create the _maybe_ functions.
   for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
        AccessSizeIndex++) {
     unsigned AccessSize = 1 << AccessSizeIndex;
@@ -503,12 +672,23 @@
         IRB.getInt8PtrTy(), IRB.getInt32Ty());
   }
 
+  // Functions for stack instrumentation.
   MsanSetAllocaOrigin4Fn = M.getOrInsertFunction(
     "__msan_set_alloca_origin4", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy,
     IRB.getInt8PtrTy(), IntptrTy);
   MsanPoisonStackFn =
       M.getOrInsertFunction("__msan_poison_stack", IRB.getVoidTy(),
                             IRB.getInt8PtrTy(), IntptrTy);
+}
+
+/// \brief Insert extern declaration of runtime-provided functions and globals.
+void MemorySanitizer::initializeCallbacks(Module &M) {
+  // Only do this once.
+  static bool CallbacksInitialized = false;
+  if (CallbacksInitialized)
+    return;
+
+  IRBuilder<> IRB(*C);
   MsanChainOriginFn = M.getOrInsertFunction(
     "__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty());
   MemmoveFn = M.getOrInsertFunction(
@@ -521,40 +701,34 @@
     "__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(),
     IntptrTy);
 
-  // Create globals.
-  RetvalTLS = new GlobalVariable(
-    M, ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), false,
-    GlobalVariable::ExternalLinkage, nullptr, "__msan_retval_tls", nullptr,
-    GlobalVariable::InitialExecTLSModel);
-  RetvalOriginTLS = new GlobalVariable(
-    M, OriginTy, false, GlobalVariable::ExternalLinkage, nullptr,
-    "__msan_retval_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
-
-  ParamTLS = new GlobalVariable(
-    M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
-    GlobalVariable::ExternalLinkage, nullptr, "__msan_param_tls", nullptr,
-    GlobalVariable::InitialExecTLSModel);
-  ParamOriginTLS = new GlobalVariable(
-    M, ArrayType::get(OriginTy, kParamTLSSize / 4), false,
-    GlobalVariable::ExternalLinkage, nullptr, "__msan_param_origin_tls",
-    nullptr, GlobalVariable::InitialExecTLSModel);
-
-  VAArgTLS = new GlobalVariable(
-    M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
-    GlobalVariable::ExternalLinkage, nullptr, "__msan_va_arg_tls", nullptr,
-    GlobalVariable::InitialExecTLSModel);
-  VAArgOverflowSizeTLS = new GlobalVariable(
-    M, IRB.getInt64Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
-    "__msan_va_arg_overflow_size_tls", nullptr,
-    GlobalVariable::InitialExecTLSModel);
-  OriginTLS = new GlobalVariable(
-    M, IRB.getInt32Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
-    "__msan_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
+  if (!CompileKernel) {
+    createUserspaceApi(M);
+  } else {
+    createKernelApi(M);
+  }
 
   // We insert an empty inline asm after __msan_report* to avoid callback merge.
   EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
                             StringRef(""), StringRef(""),
                             /*hasSideEffects=*/true);
+  CallbacksInitialized = true;
+}
+
+Value *MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore, int size) {
+  Value **Fns =
+      isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
+  switch (size) {
+  case 1:
+    return Fns[0];
+  case 2:
+    return Fns[1];
+  case 4:
+    return Fns[2];
+  case 8:
+    return Fns[3];
+  default:
+    return nullptr;
+  }
 }
 
 /// \brief Module-level initialization.
@@ -693,6 +867,7 @@
   ValueMap<Value*, Value*> ShadowMap, OriginMap;
   std::unique_ptr<VarArgHelper> VAHelper;
   const TargetLibraryInfo *TLI;
+  BasicBlock *ActualFnStart;
 
   // The following flags disable parts of MSan instrumentation based on
   // blacklist contents and command-line options.
@@ -776,14 +951,14 @@
     }
   }
 
-  void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
+  void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin, Value *MaybeOriginPtr,
                    unsigned Alignment, bool AsCall) {
     const DataLayout &DL = F.getParent()->getDataLayout();
     unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
     unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
     if (Shadow->getType()->isAggregateType()) {
       paintOrigin(IRB, updateOrigin(Origin, IRB),
-                  getOriginPtr(Addr, IRB, Alignment), StoreSize,
+                  MaybeOriginPtr ? MaybeOriginPtr : getOriginPtr(Addr, IRB, Alignment), StoreSize,
                   OriginAlignment);
     } else {
       Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
@@ -791,7 +966,7 @@
       if (ConstantShadow) {
         if (ClCheckConstantShadow && !ConstantShadow->isZeroValue())
           paintOrigin(IRB, updateOrigin(Origin, IRB),
-                      getOriginPtr(Addr, IRB, Alignment), StoreSize,
+                      MaybeOriginPtr ? MaybeOriginPtr : getOriginPtr(Addr, IRB, Alignment), StoreSize,
                       OriginAlignment);
         return;
       }
@@ -813,34 +988,94 @@
             Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
         IRBuilder<> IRBNew(CheckTerm);
         paintOrigin(IRBNew, updateOrigin(Origin, IRBNew),
-                    getOriginPtr(Addr, IRBNew, Alignment), StoreSize,
+                    MaybeOriginPtr ? MaybeOriginPtr : getOriginPtr(Addr, IRBNew, Alignment), StoreSize,
                     OriginAlignment);
       }
     }
   }
 
+  void setShadowOriginForStoreKmsanPtrs(Instruction *I, Value *Addr,
+                                        Value *Shadow, Value *Origin, unsigned Alignment) {
+    IRBuilder<> IRB(I);
+    Type *ShadowTy = Shadow->getType();
+    int BitWidth = VectorOrPrimitiveTypeSizeInBits(ShadowTy);
+    int Size = BitWidth / 8;
+    unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
+    ///unsigned OriginAlignment = 4; // TODO(glider)
+    // Make sure Size is at least 1 if the operand is i1.
+    if (Size * 8 < BitWidth)
+      Size++;
+
+    Value *SizeVal = ConstantInt::get(IRB.getInt64Ty(), Size);
+    Value *AddrCast = IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0));
+    Instruction *ShadowOriginPtrs;
+    Value *Getter = MS.getKmsanShadowOriginAccessFn(/*isStore*/ true, Size);
+    if (Getter) {
+      ShadowOriginPtrs = IRB.CreateCall(Getter, AddrCast);
+    } else {
+      ShadowOriginPtrs = IRB.CreateCall(MS.MsanMetadataPtrForStoreN, {AddrCast, SizeVal});
+    }
+    Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0);
+    ShadowPtr = IRB.CreatePointerCast(ShadowPtr, PointerType::get(ShadowTy, 0));
+    IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment);
+    Value *OriginPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 1);
+    storeOrigin(IRB, Addr, Shadow, Origin, OriginPtr, /*StoreAlignment*/OriginAlignment, /*InstrumentWithCalls*/false);
+    if (ClCheckAccessAddress)
+      insertShadowCheck(Addr, ShadowOriginPtrs);
+
+    ///paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, Size, OriginAlignment);
+    ///IRB.CreateStore(Origin, OriginPtr);
+  }
+
+  void storeShadowOrigin(Instruction *I, Value *Addr,
+      Value *Shadow, Value *Origin, bool InstrumentWithCalls, int StoreAlignment) {
+    if (MS.CompileKernel) {
+      setShadowOriginForStoreKmsanPtrs(I, Addr, Shadow, Origin, StoreAlignment);
+    } else {
+      IRBuilder<> IRB(I);
+      Value *ShadowPtr = getShadowPtr(Addr, Shadow->getType(), IRB);
+
+      StoreInst *NewI =
+          IRB.CreateAlignedStore(Shadow, ShadowPtr, StoreAlignment);
+      DEBUG(dbgs() << "  STORE: " << *NewI << "\n");
+      if (ClCheckAccessAddress)
+        insertShadowCheck(Addr, NewI);
+
+      // TODO(glider): shouldn't the alignment be 4 here?
+      if (MS.TrackOrigins && !I->isAtomic())
+        storeOrigin(IRB, Addr, Shadow, Origin, /*MaybeOriginPtr*/nullptr, StoreAlignment, InstrumentWithCalls);
+    }
+  }
+
   void materializeStores(bool InstrumentWithCalls) {
     for (StoreInst *SI : StoreList) {
       IRBuilder<> IRB(SI);
       Value *Val = SI->getValueOperand();
       Value *Addr = SI->getPointerOperand();
       Value *Shadow = SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
-      Value *ShadowPtr = getShadowPtr(Addr, Shadow->getType(), IRB);
-
-      StoreInst *NewSI =
-          IRB.CreateAlignedStore(Shadow, ShadowPtr, SI->getAlignment());
-      DEBUG(dbgs() << "  STORE: " << *NewSI << "\n");
-
-      if (ClCheckAccessAddress)
-        insertShadowCheck(Addr, NewSI);
+      Value *Origin = MS.TrackOrigins ? getOrigin(Val) : nullptr;
 
+      storeShadowOrigin(SI, Addr, Shadow, Origin, InstrumentWithCalls, SI->getAlignment());
       if (SI->isAtomic())
         SI->setOrdering(addReleaseOrdering(SI->getOrdering()));
+    }
+  }
 
-      if (MS.TrackOrigins && !SI->isAtomic())
-        storeOrigin(IRB, Addr, Shadow, getOrigin(Val), SI->getAlignment(),
-                    InstrumentWithCalls);
+  /// \brief Helper function to insert a warning at IRB's current insert point.
+  void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
+    if (!Origin) Origin = (Value *)IRB.getInt32(0);
+    if (!MS.CompileKernel) {
+      if (MS.TrackOrigins) {
+        IRB.CreateStore(Origin, MS.OriginTLS);
+      }
+      IRB.CreateCall(MS.WarningFn, {});
+    } else {
+      IRB.CreateCall(MS.MsanWarning32Fn, Origin);
     }
+    IRB.CreateCall(MS.EmptyAsm, {});
+    // FIXME: Insert UnreachableInst if !MS.Recover?
+    // This may invalidate some of the following checks and needs to be done
+    // at the very end.
   }
 
   void materializeOneCheck(Instruction *OrigIns, Value *Shadow, Value *Origin,
@@ -853,15 +1088,7 @@
     Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow);
     if (ConstantShadow) {
       if (ClCheckConstantShadow && !ConstantShadow->isZeroValue()) {
-        if (MS.TrackOrigins) {
-          IRB.CreateStore(Origin ? (Value *)Origin : (Value *)IRB.getInt32(0),
-                          MS.OriginTLS);
-        }
-        IRB.CreateCall(MS.WarningFn, {});
-        IRB.CreateCall(MS.EmptyAsm, {});
-        // FIXME: Insert UnreachableInst if !MS.Recover?
-        // This may invalidate some of the following checks and needs to be done
-        // at the very end.
+        insertWarningFn(IRB, Origin);
       }
       return;
     }
@@ -885,12 +1112,7 @@
           /* Unreachable */ !MS.Recover, MS.ColdCallWeights);
 
       IRB.SetInsertPoint(CheckTerm);
-      if (MS.TrackOrigins) {
-        IRB.CreateStore(Origin ? (Value *)Origin : (Value *)IRB.getInt32(0),
-                        MS.OriginTLS);
-      }
-      IRB.CreateCall(MS.WarningFn, {});
-      IRB.CreateCall(MS.EmptyAsm, {});
+      insertWarningFn(IRB, Origin);
       DEBUG(dbgs() << "  CHECK: " << *Cmp << "\n");
     }
   }
@@ -905,9 +1127,30 @@
     DEBUG(dbgs() << "DONE:\n" << F);
   }
 
+  BasicBlock *insertKmsanPrologue(Function &F) {
+    BasicBlock *ret =
+        SplitBlock(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHI());
+    IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
+    Value *ContextState = IRB.CreateCall(MS.GetContextStateFn, {});
+    Constant *Zero = IRB.getInt32(0);
+    MS.ParamTLS = IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(0)}, "param_shadow");
+    MS.RetvalTLS = IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(1)}, "retval_shadow");
+    MS.VAArgTLS = IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(2)}, "va_arg_shadow");
+    MS.VAArgOriginTLS = IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(3)}, "va_arg_origin");
+    MS.VAArgOverflowSizeTLS =
+        IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(4)}, "va_arg_overflow_size");
+    MS.ParamOriginTLS = IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(5)}, "param_origin");
+    MS.RetvalOriginTLS = IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(6)}, "retval_origin");
+    return ret;
+  }
+
   /// \brief Add MemorySanitizer instrumentation to a function.
   bool runOnFunction() {
     MS.initializeCallbacks(*F.getParent());
+    if (MS.CompileKernel)
+      ActualFnStart = insertKmsanPrologue(F);
+    else
+      ActualFnStart = &F.getEntryBlock();
 
     // In the presence of unreachable blocks, we may see Phi nodes with
     // incoming nodes from such blocks. Since InstVisitor skips unreachable
@@ -918,7 +1161,7 @@
     // Iterate all BBs in depth-first order and create shadow instructions
     // for all instructions (where applicable).
     // For PHI nodes we create dummy shadow PHIs which will be finalized later.
-    for (BasicBlock *BB : depth_first(&F.getEntryBlock()))
+    for (BasicBlock *BB : depth_first(ActualFnStart))
       visit(*BB);
 
     // Finalize PHI nodes.
@@ -1059,9 +1302,10 @@
   Value *getShadowPtrForArgument(Value *A, IRBuilder<> &IRB,
                                  int ArgOffset) {
     Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy);
-    Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
+    if (ArgOffset)
+      Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
     return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0),
-                              "_msarg");
+                              "_msarg_s_");
   }
 
   /// \brief Compute the origin address for a given function argument.
@@ -1069,16 +1313,17 @@
                                  int ArgOffset) {
     if (!MS.TrackOrigins) return nullptr;
     Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy);
-    Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
+    if (ArgOffset)
+      Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
     return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
                               "_msarg_o");
   }
 
   /// \brief Compute the shadow address for a retval.
   Value *getShadowPtrForRetval(Value *A, IRBuilder<> &IRB) {
-    Value *Base = IRB.CreatePointerCast(MS.RetvalTLS, MS.IntptrTy);
-    return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0),
-                              "_msret");
+    return IRB.CreatePointerCast(MS.RetvalTLS,
+                                 PointerType::get(getShadowTy(A), 0),
+                                 "_msret");
   }
 
   /// \brief Compute the origin address for a retval.
@@ -1148,6 +1393,50 @@
     return Constant::getNullValue(MS.OriginTy);
   }
 
+  /// \brief Unpoison an aligned byte range.
+  void unpoisonRange(IRBuilder<> &IRB, Value *Addr, unsigned Size, unsigned ArgAlign) {
+    if (!MS.CompileKernel) {
+      IRB.CreateMemSet(
+          getShadowPtr(Addr, IRB.getInt8Ty(), IRB),
+          Constant::getNullValue(IRB.getInt8Ty()), Size, ArgAlign);
+    } else {
+      Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
+      IRB.CreateCall(MS.MsanUnpoisonFn, {Addr, SizeVal});
+    }
+  }
+
+  /// \brief Create a memcpy() call that copies from SrcPtr to Shadow(DstPtr).
+  ///
+  ///  For KMSAN, also copy the origins.
+  Value *createMemcpyToLoadArgShadow(
+      IRBuilder<> &IRB, Value *DstPtr, Value *SrcPtr, Value *Size, int Align) {
+    if (!MS.CompileKernel) {
+      Value *DstShadowPtr =
+          getShadowPtr(DstPtr, IRB.getInt8Ty(), IRB);
+      return IRB.CreateMemCpy(DstShadowPtr, SrcPtr, Size, Align);
+    } else {
+      // The above alignment is unused, because KMSAN runtime doesn't
+      // make any assumptions about it.
+      DstPtr =
+          IRB.CreateIntToPtr(DstPtr, IRB.getInt8PtrTy());
+      return IRB.CreateCall(MS.MsanLoadArgShadowFn, {DstPtr, SrcPtr, Size});
+    }
+  }
+
+  Value *createMemcpyToLoadArgOrigin(
+      IRBuilder<> &IRB, Value *DstPtr, Value *SrcPtr, Value *Size, int Align) {
+    if (!MS.CompileKernel) {
+      // Do nothing, not implemented yet (breakes userspace ABI).
+      return nullptr;
+    } else {
+      // The above alignment is unused, because KMSAN runtime doesn't
+      // make any assumptions about it.
+      DstPtr =
+          IRB.CreateIntToPtr(DstPtr, IRB.getInt8PtrTy());
+      return IRB.CreateCall(MS.MsanLoadArgOriginFn, {DstPtr, SrcPtr, Size});
+    }
+  }
+
   /// \brief Get the shadow value for a given Value.
   ///
   /// This function either returns the value set earlier with setShadow,
@@ -1178,7 +1467,7 @@
       if (*ShadowPtr)
         return *ShadowPtr;
       Function *F = A->getParent();
-      IRBuilder<> EntryIRB(F->getEntryBlock().getFirstNonPHI());
+      IRBuilder<> EntryIRB(ActualFnStart->getFirstNonPHI());
       unsigned ArgOffset = 0;
       const DataLayout &DL = F->getParent()->getDataLayout();
       for (auto &FArg : F->args()) {
@@ -1204,14 +1493,13 @@
             }
             if (Overflow) {
               // ParamTLS overflow.
-              EntryIRB.CreateMemSet(
-                  getShadowPtr(V, EntryIRB.getInt8Ty(), EntryIRB),
-                  Constant::getNullValue(EntryIRB.getInt8Ty()), Size, ArgAlign);
+              unpoisonRange(EntryIRB, V, Size, ArgAlign);
             } else {
               unsigned CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
-              Value *Cpy = EntryIRB.CreateMemCpy(
-                  getShadowPtr(V, EntryIRB.getInt8Ty(), EntryIRB), Base, Size,
-                  CopyAlign);
+              Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
+              Base =
+                  EntryIRB.CreatePointerCast(Base, EntryIRB.getInt8PtrTy());
+              Value *Cpy = createMemcpyToLoadArgShadow(EntryIRB, V, Base, SizeVal, CopyAlign);
               DEBUG(dbgs() << "  ByValCpy: " << *Cpy << "\n");
               (void)Cpy;
             }
@@ -1346,11 +1634,11 @@
       InstVisitor<MemorySanitizerVisitor>::visit(I);
   }
 
-  /// \brief Instrument LoadInst
+  /// \brief Instrument LoadInst in the userspace.
   ///
   /// Loads the corresponding shadow and (optionally) origin.
   /// Optionally, checks that the load address is fully defined.
-  void visitLoadInst(LoadInst &I) {
+  void visitLoadInstUserspace(LoadInst &I) {
     assert(I.getType()->isSized() && "Load type must have size");
     assert(!I.getMetadata("nosanitize"));
     IRBuilder<> IRB(I.getNextNode());
@@ -1364,12 +1652,6 @@
       setShadow(&I, getCleanShadow(&I));
     }
 
-    if (ClCheckAccessAddress)
-      insertShadowCheck(I.getPointerOperand(), &I);
-
-    if (I.isAtomic())
-      I.setOrdering(addAcquireOrdering(I.getOrdering()));
-
     if (MS.TrackOrigins) {
       if (PropagateShadow) {
         unsigned Alignment = I.getAlignment();
@@ -1382,6 +1664,57 @@
     }
   }
 
+  /// \brief Instrument LoadInst in the kernel.
+  void visitLoadInstKmsanPtrs(LoadInst &I) {
+    assert(I.getType()->isSized() && "Load type must have size");
+    IRBuilder<> IRB(I.getNextNode());
+    Type *ShadowTy = getShadowTy(&I);
+    Value *Addr = I.getPointerOperand();
+    if (PropagateShadow && !I.getMetadata("nosanitize")) {
+      int BitWidth = VectorOrPrimitiveTypeSizeInBits(ShadowTy);
+      int Size = BitWidth / 8;
+      // Make sure Size is at least 1 if the operand is i1.
+      if (Size * 8 < BitWidth)
+        Size++;
+      Value *SizeVal = ConstantInt::get(IRB.getInt64Ty(), Size);
+
+      Addr = IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0));
+      Value *ShadowOriginPtrs;
+      Value *Getter = MS.getKmsanShadowOriginAccessFn(/*isStore*/ false, Size);
+      if (Getter) {
+        ShadowOriginPtrs = IRB.CreateCall(Getter, Addr);
+      } else {
+        ShadowOriginPtrs = IRB.CreateCall(MS.MsanMetadataPtrForLoadN, {Addr, SizeVal});
+      }
+      Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0);
+      ShadowPtr = IRB.CreatePointerCast(ShadowPtr, PointerType::get(ShadowTy, 0));
+      Value *Shadow = IRB.CreateAlignedLoad(ShadowPtr, I.getAlignment(), "_msld");
+      Value *OriginPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 1);
+      Value *Origin = IRB.CreateAlignedLoad(OriginPtr, std::max(kMinOriginAlignment, I.getAlignment()));
+
+      setShadow(&I, Shadow);
+      setOrigin(&I, Origin);
+    } else {
+      setShadow(&I, getCleanShadow(&I));
+      setOrigin(&I, getCleanOrigin());
+    }
+  }
+
+
+  void visitLoadInst(LoadInst &I) {
+    if (MS.CompileKernel) {
+      visitLoadInstKmsanPtrs(I);
+    } else {
+      visitLoadInstUserspace(I);
+    }
+
+    if (ClCheckAccessAddress)
+      insertShadowCheck(I.getPointerOperand(), &I);
+
+    if (I.isAtomic())
+      I.setOrdering(addAcquireOrdering(I.getOrdering()));
+  }
+
   /// \brief Instrument StoreInst
   ///
   /// Stores the corresponding shadow and (optionally) origin.
@@ -1395,7 +1728,6 @@
 
     IRBuilder<> IRB(&I);
     Value *Addr = I.getOperand(0);
-    Value *ShadowPtr = getShadowPtr(Addr, I.getType(), IRB);
 
     if (ClCheckAccessAddress)
       insertShadowCheck(Addr, &I);
@@ -1406,8 +1738,8 @@
     if (isa<AtomicCmpXchgInst>(I))
       insertShadowCheck(I.getOperand(1), &I);
 
-    IRB.CreateStore(getCleanShadow(&I), ShadowPtr);
-
+    storeShadowOrigin(&I, Addr, getCleanShadow(&I), getCleanOrigin(),
+                      /*InstrumentWithCalls*/false, /*StoreAlignment*/1);
     setShadow(&I, getCleanShadow(&I));
     setOrigin(&I, getCleanOrigin());
   }
@@ -2016,18 +2348,15 @@
     IRBuilder<> IRB(&I);
     Value* Addr = I.getArgOperand(0);
     Value *Shadow = getShadow(&I, 1);
-    Value *ShadowPtr = getShadowPtr(Addr, Shadow->getType(), IRB);
+    Value *Origin = MS.TrackOrigins ? getOrigin(&I, 1) : nullptr;
 
     // We don't know the pointer alignment (could be unaligned SSE store!).
     // Have to assume to worst case.
-    IRB.CreateAlignedStore(Shadow, ShadowPtr, 1);
+    storeShadowOrigin(&I, Addr, Shadow, Origin, /*InstrumentWithCalls*/false, /*StoreAlignment*/1);
 
     if (ClCheckAccessAddress)
       insertShadowCheck(Addr, &I);
 
-    // FIXME: factor out common code from materializeStores
-    if (MS.TrackOrigins)
-      IRB.CreateStore(getOrigin(&I, 1), getOriginPtr(Addr, IRB, 1));
     return true;
   }
 
@@ -2723,9 +3052,21 @@
         if (ArgOffset + Size > kParamTLSSize) break;
         unsigned ParamAlignment = CS.getParamAlignment(i);
         unsigned Alignment = std::min(ParamAlignment, kShadowTLSAlignment);
-        Store = IRB.CreateMemCpy(ArgShadowBase,
-                                 getShadowPtr(A, Type::getInt8Ty(*MS.C), IRB),
-                                 Size, Alignment);
+        if (!MS.CompileKernel) {
+          Store = IRB.CreateMemCpy(ArgShadowBase,
+                                   getShadowPtr(A, Type::getInt8Ty(*MS.C), IRB),
+                                   Size, Alignment);
+        } else {
+          // Not using the above alignment, as KMSAN runtime doesn't make any
+          // assumptions about it.
+          Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
+          ArgShadowBase =
+              IRB.CreatePointerCast(ArgShadowBase, IRB.getInt8PtrTy());
+          A = IRB.CreatePointerCast(A, IRB.getInt8PtrTy());
+          Value *OriginPtr = getOriginPtrForArgument(A, IRB, ArgOffset);
+          OriginPtr = IRB.CreatePointerCast(OriginPtr, IRB.getInt8PtrTy());
+          IRB.CreateCall(MS.MsanStoreArgShadowOriginFn, {ArgShadowBase, OriginPtr, A, SizeVal});
+        }
       } else {
         Size = DL.getTypeAllocSize(A->getType());
         if (ArgOffset + Size > kParamTLSSize) break;
@@ -2734,11 +3075,13 @@
         Constant *Cst = dyn_cast<Constant>(ArgShadow);
         if (Cst && Cst->isNullValue()) ArgIsInitialized = true;
       }
-      if (MS.TrackOrigins && !ArgIsInitialized)
-        IRB.CreateStore(getOrigin(A),
-                        getOriginPtrForArgument(A, IRB, ArgOffset));
-      (void)Store;
-      assert(Size != 0 && Store != nullptr);
+      if (!MS.CompileKernel) {
+        if (MS.TrackOrigins && !ArgIsInitialized)
+          IRB.CreateStore(getOrigin(A),
+                          getOriginPtrForArgument(A, IRB, ArgOffset));
+        (void)Store;
+        assert(Size != 0 && Store != nullptr);
+      }
       DEBUG(dbgs() << "  Param:" << *Store << "\n");
       ArgOffset += alignTo(Size, 8);
     }
@@ -2830,15 +3173,20 @@
                                   "_msphi_o"));
   }
 
-  void visitAllocaInst(AllocaInst &I) {
-    setShadow(&I, getCleanShadow(&I));
-    setOrigin(&I, getCleanOrigin());
-    IRBuilder<> IRB(I.getNextNode());
-    const DataLayout &DL = F.getParent()->getDataLayout();
-    uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType());
-    Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize);
-    if (I.isArrayAllocation())
-      Len = IRB.CreateMul(Len, I.getArraySize());
+  Value *getLocalVarDescription(AllocaInst &I) {
+    SmallString<2048> StackDescriptionStorage;
+    raw_svector_ostream StackDescription(StackDescriptionStorage);
+    // We create a string with a description of the stack allocation and
+    // pass it into __msan_set_alloca_origin.
+    // It will be printed by the run-time if stack-originated UMR is found.
+    // The first 4 bytes of the string are set to '----' and will be replaced
+    // by __msan_va_arg_overflow_size_tls at the first call.
+    StackDescription << "----" << I.getName() << "@" << F.getName();
+    return createPrivateNonConstGlobalForString(*F.getParent(),
+                                                StackDescription.str());
+  }
+
+  void instrumentAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
     if (PoisonStack && ClPoisonStackWithCall) {
       IRB.CreateCall(MS.MsanPoisonStackFn,
                      {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
@@ -2849,18 +3197,7 @@
     }
 
     if (PoisonStack && MS.TrackOrigins) {
-      SmallString<2048> StackDescriptionStorage;
-      raw_svector_ostream StackDescription(StackDescriptionStorage);
-      // We create a string with a description of the stack allocation and
-      // pass it into __msan_set_alloca_origin.
-      // It will be printed by the run-time if stack-originated UMR is found.
-      // The first 4 bytes of the string are set to '----' and will be replaced
-      // by __msan_va_arg_overflow_size_tls at the first call.
-      StackDescription << "----" << I.getName() << "@" << F.getName();
-      Value *Descr =
-          createPrivateNonConstGlobalForString(*F.getParent(),
-                                               StackDescription.str());
-
+      Value *Descr = getLocalVarDescription(I);
       IRB.CreateCall(MS.MsanSetAllocaOrigin4Fn,
                      {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
                       IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()),
@@ -2868,6 +3205,33 @@
     }
   }
 
+  void instrumentAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
+    Value *Descr = getLocalVarDescription(I);
+    Value *Pc = IRB.CreateCall(
+        Intrinsic::getDeclaration(F.getParent(), Intrinsic::returnaddress),
+        IRB.getInt32(0));
+    IRB.CreateCall(MS.MsanPoisonAllocaFn,
+                   {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
+                    IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()),
+                    IRB.CreatePointerCast(Pc, MS.IntptrTy)});
+  }
+
+  void visitAllocaInst(AllocaInst &I) {
+    setShadow(&I, getCleanShadow(&I));
+    setOrigin(&I, getCleanOrigin());
+    IRBuilder<> IRB(I.getNextNode());
+    const DataLayout &DL = F.getParent()->getDataLayout();
+    uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType());
+    Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize);
+    if (I.isArrayAllocation())
+      Len = IRB.CreateMul(Len, I.getArraySize());
+
+    if (!MS.CompileKernel)
+      instrumentAllocaUserspace(I, IRB, Len);
+    else
+      instrumentAllocaKmsan(I, IRB, Len);
+  }
+
   void visitSelectInst(SelectInst& I) {
     IRBuilder<> IRB(&I);
     // a = select b, c, d
@@ -3016,6 +3380,7 @@
   MemorySanitizer &MS;
   MemorySanitizerVisitor &MSV;
   Value *VAArgTLSCopy = nullptr;
+  Value *VAArgTLSOriginCopy = nullptr;
   Value *VAArgOverflowSize = nullptr;
 
   SmallVector<CallInst*, 16> VAStartInstrumentationList;
@@ -3067,8 +3432,15 @@
         uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
         Value *Base = getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset);
         OverflowOffset += alignTo(ArgSize, 8);
-        IRB.CreateMemCpy(Base, MSV.getShadowPtr(A, IRB.getInt8Ty(), IRB),
-                         ArgSize, kShadowTLSAlignment);
+        if (!MS.CompileKernel) {
+          IRB.CreateMemCpy(Base, MSV.getShadowPtr(A, IRB.getInt8Ty(), IRB),
+                           ArgSize, kShadowTLSAlignment);
+        } else {
+          Value *ArgSizeV = ConstantInt::get(MS.IntptrTy, ArgSize);
+          Base = IRB.CreatePointerCast(Base, IRB.getInt8PtrTy());
+          A = IRB.CreatePointerCast(A, IRB.getInt8PtrTy());
+          IRB.CreateCall(MS.MsanStoreArgShadowFn, {Base, A, ArgSizeV});
+        }
       } else {
         ArgKind AK = classifyArgument(A);
         if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
@@ -3108,36 +3480,31 @@
   Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
                                    int ArgOffset) {
     Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
-    Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
+    if (ArgOffset)
+      Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
     return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
                               "_msarg");
   }
 
-  void visitVAStartInst(VAStartInst &I) override {
-    if (F.getCallingConv() == CallingConv::Win64)
-      return;
+  void unpoisonVAListTagForInst(IntrinsicInst &I) {
     IRBuilder<> IRB(&I);
-    VAStartInstrumentationList.push_back(&I);
     Value *VAListTag = I.getArgOperand(0);
-    Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
-
     // Unpoison the whole __va_list_tag.
     // FIXME: magic ABI constants.
-    IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
-                     /* size */24, /* alignment */8, false);
+    MSV.unpoisonRange(IRB, VAListTag, 24,  8);
   }
 
-  void visitVACopyInst(VACopyInst &I) override {
+  void visitVAStartInst(VAStartInst &I) {
     if (F.getCallingConv() == CallingConv::Win64)
       return;
-    IRBuilder<> IRB(&I);
-    Value *VAListTag = I.getArgOperand(0);
-    Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
+    VAStartInstrumentationList.push_back(&I);
+    unpoisonVAListTagForInst(I);
+  }
 
-    // Unpoison the whole __va_list_tag.
-    // FIXME: magic ABI constants.
-    IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
-                     /* size */24, /* alignment */8, false);
+  void visitVACopyInst(VACopyInst &I) {
+    if (F.getCallingConv() == CallingConv::Win64)
+      return;
+    unpoisonVAListTagForInst(I);
   }
 
   void finalizeInstrumentation() override {
@@ -3146,13 +3513,17 @@
     if (!VAStartInstrumentationList.empty()) {
       // If there is a va_start in this function, make a backup copy of
       // va_arg_tls somewhere in the function entry block.
-      IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
+      IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
       VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
       Value *CopySize =
         IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset),
                       VAArgOverflowSize);
       VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
       IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8);
+      if (MS.CompileKernel) {
+        VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
+        IRB.CreateMemCpy(VAArgTLSOriginCopy, MS.VAArgOriginTLS, CopySize, 8);
+      }
     }
 
     // Instrument va_start.
@@ -3168,10 +3539,12 @@
                         ConstantInt::get(MS.IntptrTy, 16)),
           Type::getInt64PtrTy(*MS.C));
       Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
-      Value *RegSaveAreaShadowPtr =
-        MSV.getShadowPtr(RegSaveAreaPtr, IRB.getInt8Ty(), IRB);
-      IRB.CreateMemCpy(RegSaveAreaShadowPtr, VAArgTLSCopy,
-                       AMD64FpEndOffset, 16);
+      MSV.createMemcpyToLoadArgShadow(IRB, RegSaveAreaPtr, VAArgTLSCopy,
+                          ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), 16);
+      if (MS.CompileKernel) {
+        MSV.createMemcpyToLoadArgOrigin(IRB, RegSaveAreaPtr, VAArgTLSOriginCopy,
+                            ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), 16);
+      }
 
       Value *OverflowArgAreaPtrPtr =
         IRB.CreateIntToPtr(
@@ -3179,11 +3552,9 @@
                         ConstantInt::get(MS.IntptrTy, 8)),
           Type::getInt64PtrTy(*MS.C));
       Value *OverflowArgAreaPtr = IRB.CreateLoad(OverflowArgAreaPtrPtr);
-      Value *OverflowArgAreaShadowPtr =
-        MSV.getShadowPtr(OverflowArgAreaPtr, IRB.getInt8Ty(), IRB);
       Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
                                              AMD64FpEndOffset);
-      IRB.CreateMemCpy(OverflowArgAreaShadowPtr, SrcPtr, VAArgOverflowSize, 16);
+      MSV.createMemcpyToLoadArgShadow(IRB, OverflowArgAreaPtr, SrcPtr, VAArgOverflowSize, 16);
     }
   }
 };
@@ -3260,7 +3631,7 @@
   void finalizeInstrumentation() override {
     assert(!VAArgSize && !VAArgTLSCopy &&
            "finalizeInstrumentation called twice");
-    IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
+    IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
     VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
     Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
                                     VAArgSize);
@@ -3438,7 +3809,7 @@
     if (!VAStartInstrumentationList.empty()) {
       // If there is a va_start in this function, make a backup copy of
       // va_arg_tls somewhere in the function entry block.
-      IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
+      IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
       VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
       Value *CopySize =
         IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset),
@@ -3658,7 +4029,7 @@
   void finalizeInstrumentation() override {
     assert(!VAArgSize && !VAArgTLSCopy &&
            "finalizeInstrumentation called twice");
-    IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
+    IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
     VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
     Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
                                     VAArgSize);
Index: test/Instrumentation/MemorySanitizer/msan_basic.ll
===================================================================
--- test/Instrumentation/MemorySanitizer/msan_basic.ll
+++ test/Instrumentation/MemorySanitizer/msan_basic.ll
@@ -1,955 +0,0 @@
-; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
-; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=1 -S | FileCheck -check-prefix=CHECK -check-prefix=CHECK-ORIGINS %s
-
-target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-; CHECK: @llvm.global_ctors {{.*}} { i32 0, void ()* @msan.module_ctor, i8* null }
-
-; Check the presence and the linkage type of __msan_track_origins and
-; other interface symbols.
-; CHECK-NOT: @__msan_track_origins
-; CHECK-ORIGINS: @__msan_track_origins = weak_odr constant i32 1
-; CHECK-NOT: @__msan_keep_going = weak_odr constant i32 0
-; CHECK: @__msan_retval_tls = external thread_local(initialexec) global [{{.*}}]
-; CHECK: @__msan_retval_origin_tls = external thread_local(initialexec) global i32
-; CHECK: @__msan_param_tls = external thread_local(initialexec) global [{{.*}}]
-; CHECK: @__msan_param_origin_tls = external thread_local(initialexec) global [{{.*}}]
-; CHECK: @__msan_va_arg_tls = external thread_local(initialexec) global [{{.*}}]
-; CHECK: @__msan_va_arg_overflow_size_tls = external thread_local(initialexec) global i64
-; CHECK: @__msan_origin_tls = external thread_local(initialexec) global i32
-
-
-; Check instrumentation of stores
-
-define void @Store(i32* nocapture %p, i32 %x) nounwind uwtable sanitize_memory {
-entry:
-  store i32 %x, i32* %p, align 4
-  ret void
-}
-
-; CHECK-LABEL: @Store
-; CHECK: load {{.*}} @__msan_param_tls
-; CHECK-ORIGINS: load {{.*}} @__msan_param_origin_tls
-; CHECK: store
-; CHECK-ORIGINS: icmp
-; CHECK-ORIGINS: br i1
-; CHECK-ORIGINS: <label>
-; CHECK-ORIGINS: store
-; CHECK-ORIGINS: br label
-; CHECK-ORIGINS: <label>
-; CHECK: store
-; CHECK: ret void
-
-
-; Check instrumentation of aligned stores
-; Shadow store has the same alignment as the original store; origin store
-; does not specify explicit alignment.
-
-define void @AlignedStore(i32* nocapture %p, i32 %x) nounwind uwtable sanitize_memory {
-entry:
-  store i32 %x, i32* %p, align 32
-  ret void
-}
-
-; CHECK-LABEL: @AlignedStore
-; CHECK: load {{.*}} @__msan_param_tls
-; CHECK-ORIGINS: load {{.*}} @__msan_param_origin_tls
-; CHECK: store {{.*}} align 32
-; CHECK-ORIGINS: icmp
-; CHECK-ORIGINS: br i1
-; CHECK-ORIGINS: <label>
-; CHECK-ORIGINS: store {{.*}} align 32
-; CHECK-ORIGINS: br label
-; CHECK-ORIGINS: <label>
-; CHECK: store {{.*}} align 32
-; CHECK: ret void
-
-
-; load followed by cmp: check that we load the shadow and call __msan_warning.
-define void @LoadAndCmp(i32* nocapture %a) nounwind uwtable sanitize_memory {
-entry:
-  %0 = load i32, i32* %a, align 4
-  %tobool = icmp eq i32 %0, 0
-  br i1 %tobool, label %if.end, label %if.then
-
-if.then:                                          ; preds = %entry
-  tail call void (...) @foo() nounwind
-  br label %if.end
-
-if.end:                                           ; preds = %entry, %if.then
-  ret void
-}
-
-declare void @foo(...)
-
-; CHECK-LABEL: @LoadAndCmp
-; CHECK: = load
-; CHECK: = load
-; CHECK: call void @__msan_warning_noreturn()
-; CHECK-NEXT: call void asm sideeffect
-; CHECK-NEXT: unreachable
-; CHECK: ret void
-
-; Check that we store the shadow for the retval.
-define i32 @ReturnInt() nounwind uwtable readnone sanitize_memory {
-entry:
-  ret i32 123
-}
-
-; CHECK-LABEL: @ReturnInt
-; CHECK: store i32 0,{{.*}}__msan_retval_tls
-; CHECK: ret i32
-
-; Check that we get the shadow for the retval.
-define void @CopyRetVal(i32* nocapture %a) nounwind uwtable sanitize_memory {
-entry:
-  %call = tail call i32 @ReturnInt() nounwind
-  store i32 %call, i32* %a, align 4
-  ret void
-}
-
-; CHECK-LABEL: @CopyRetVal
-; CHECK: load{{.*}}__msan_retval_tls
-; CHECK: store
-; CHECK: store
-; CHECK: ret void
-
-
-; Check that we generate PHIs for shadow.
-define void @FuncWithPhi(i32* nocapture %a, i32* %b, i32* nocapture %c) nounwind uwtable sanitize_memory {
-entry:
-  %tobool = icmp eq i32* %b, null
-  br i1 %tobool, label %if.else, label %if.then
-
-  if.then:                                          ; preds = %entry
-  %0 = load i32, i32* %b, align 4
-  br label %if.end
-
-  if.else:                                          ; preds = %entry
-  %1 = load i32, i32* %c, align 4
-  br label %if.end
-
-  if.end:                                           ; preds = %if.else, %if.then
-  %t.0 = phi i32 [ %0, %if.then ], [ %1, %if.else ]
-  store i32 %t.0, i32* %a, align 4
-  ret void
-}
-
-; CHECK-LABEL: @FuncWithPhi
-; CHECK: = phi
-; CHECK-NEXT: = phi
-; CHECK: store
-; CHECK: store
-; CHECK: ret void
-
-; Compute shadow for "x << 10"
-define void @ShlConst(i32* nocapture %x) nounwind uwtable sanitize_memory {
-entry:
-  %0 = load i32, i32* %x, align 4
-  %1 = shl i32 %0, 10
-  store i32 %1, i32* %x, align 4
-  ret void
-}
-
-; CHECK-LABEL: @ShlConst
-; CHECK: = load
-; CHECK: = load
-; CHECK: shl
-; CHECK: shl
-; CHECK: store
-; CHECK: store
-; CHECK: ret void
-
-; Compute shadow for "10 << x": it should have 'sext i1'.
-define void @ShlNonConst(i32* nocapture %x) nounwind uwtable sanitize_memory {
-entry:
-  %0 = load i32, i32* %x, align 4
-  %1 = shl i32 10, %0
-  store i32 %1, i32* %x, align 4
-  ret void
-}
-
-; CHECK-LABEL: @ShlNonConst
-; CHECK: = load
-; CHECK: = load
-; CHECK: = sext i1
-; CHECK: store
-; CHECK: store
-; CHECK: ret void
-
-; SExt
-define void @SExt(i32* nocapture %a, i16* nocapture %b) nounwind uwtable sanitize_memory {
-entry:
-  %0 = load i16, i16* %b, align 2
-  %1 = sext i16 %0 to i32
-  store i32 %1, i32* %a, align 4
-  ret void
-}
-
-; CHECK-LABEL: @SExt
-; CHECK: = load
-; CHECK: = load
-; CHECK: = sext
-; CHECK: = sext
-; CHECK: store
-; CHECK: store
-; CHECK: ret void
-
-
-; memset
-define void @MemSet(i8* nocapture %x) nounwind uwtable sanitize_memory {
-entry:
-  call void @llvm.memset.p0i8.i64(i8* %x, i8 42, i64 10, i32 1, i1 false)
-  ret void
-}
-
-declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i32, i1) nounwind
-
-; CHECK-LABEL: @MemSet
-; CHECK: call i8* @__msan_memset
-; CHECK: ret void
-
-
-; memcpy
-define void @MemCpy(i8* nocapture %x, i8* nocapture %y) nounwind uwtable sanitize_memory {
-entry:
-  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %x, i8* %y, i64 10, i32 1, i1 false)
-  ret void
-}
-
-declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1) nounwind
-
-; CHECK-LABEL: @MemCpy
-; CHECK: call i8* @__msan_memcpy
-; CHECK: ret void
-
-
-; memmove is lowered to a call
-define void @MemMove(i8* nocapture %x, i8* nocapture %y) nounwind uwtable sanitize_memory {
-entry:
-  call void @llvm.memmove.p0i8.p0i8.i64(i8* %x, i8* %y, i64 10, i32 1, i1 false)
-  ret void
-}
-
-declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1) nounwind
-
-; CHECK-LABEL: @MemMove
-; CHECK: call i8* @__msan_memmove
-; CHECK: ret void
-
-;; ------------
-;; Placeholder tests that will fail once element atomic @llvm.mem[cpy|move|set] instrinsics have
-;; been added to the MemIntrinsic class hierarchy. These will act as a reminder to
-;; verify that MSAN handles these intrinsics properly once they have been
-;; added to that class hierarchy.
-declare void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* nocapture writeonly, i8, i64, i32) nounwind
-declare void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32) nounwind
-declare void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32) nounwind
-
-define void @atomic_memcpy(i8* nocapture %x, i8* nocapture %y) nounwind {
-  ; CHECK-LABEL: atomic_memcpy
-  ; CHECK-NEXT: call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %x, i8* align 2 %y, i64 16, i32 1)
-  ; CHECK-NEXT: ret void
-  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %x, i8* align 2 %y, i64 16, i32 1)
-  ret void
-}
-
-define void @atomic_memmove(i8* nocapture %x, i8* nocapture %y) nounwind {
-  ; CHECK-LABEL: atomic_memmove
-  ; CHECK-NEXT: call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %x, i8* align 2 %y, i64 16, i32 1)
-  ; CHECK-NEXT: ret void
-  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %x, i8* align 2 %y, i64 16, i32 1)
-  ret void
-}
-
-define void @atomic_memset(i8* nocapture %x) nounwind {
-  ; CHECK-LABEL: atomic_memset
-  ; CHECK-NEXT: call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 1 %x, i8 88, i64 16, i32 1)
-  ; CHECK-NEXT: ret void
-  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 1 %x, i8 88, i64 16, i32 1)
-  ret void
-}
-
-;; ------------
-
-
-; Check that we propagate shadow for "select"
-
-define i32 @Select(i32 %a, i32 %b, i1 %c) nounwind uwtable readnone sanitize_memory {
-entry:
-  %cond = select i1 %c, i32 %a, i32 %b
-  ret i32 %cond
-}
-
-; CHECK-LABEL: @Select
-; CHECK: select i1
-; CHECK-DAG: or i32
-; CHECK-DAG: xor i32
-; CHECK: or i32
-; CHECK-DAG: select i1
-; CHECK-ORIGINS-DAG: select
-; CHECK-ORIGINS-DAG: select
-; CHECK-DAG: select i1
-; CHECK: store i32{{.*}}@__msan_retval_tls
-; CHECK-ORIGINS: store i32{{.*}}@__msan_retval_origin_tls
-; CHECK: ret i32
-
-
-; Check that we propagate origin for "select" with vector condition.
-; Select condition is flattened to i1, which is then used to select one of the
-; argument origins.
-
-define <8 x i16> @SelectVector(<8 x i16> %a, <8 x i16> %b, <8 x i1> %c) nounwind uwtable readnone sanitize_memory {
-entry:
-  %cond = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
-  ret <8 x i16> %cond
-}
-
-; CHECK-LABEL: @SelectVector
-; CHECK: select <8 x i1>
-; CHECK-DAG: or <8 x i16>
-; CHECK-DAG: xor <8 x i16>
-; CHECK: or <8 x i16>
-; CHECK-DAG: select <8 x i1>
-; CHECK-ORIGINS-DAG: select
-; CHECK-ORIGINS-DAG: select
-; CHECK-DAG: select <8 x i1>
-; CHECK: store <8 x i16>{{.*}}@__msan_retval_tls
-; CHECK-ORIGINS: store i32{{.*}}@__msan_retval_origin_tls
-; CHECK: ret <8 x i16>
-
-
-; Check that we propagate origin for "select" with scalar condition and vector
-; arguments. Select condition shadow is sign-extended to the vector type and
-; mixed into the result shadow.
-
-define <8 x i16> @SelectVector2(<8 x i16> %a, <8 x i16> %b, i1 %c) nounwind uwtable readnone sanitize_memory {
-entry:
-  %cond = select i1 %c, <8 x i16> %a, <8 x i16> %b
-  ret <8 x i16> %cond
-}
-
-; CHECK-LABEL: @SelectVector2
-; CHECK: select i1
-; CHECK-DAG: or <8 x i16>
-; CHECK-DAG: xor <8 x i16>
-; CHECK: or <8 x i16>
-; CHECK-DAG: select i1
-; CHECK-ORIGINS-DAG: select i1
-; CHECK-ORIGINS-DAG: select i1
-; CHECK-DAG: select i1
-; CHECK: ret <8 x i16>
-
-
-define { i64, i64 } @SelectStruct(i1 zeroext %x, { i64, i64 } %a, { i64, i64 } %b) readnone sanitize_memory {
-entry:
-  %c = select i1 %x, { i64, i64 } %a, { i64, i64 } %b
-  ret { i64, i64 } %c
-}
-
-; CHECK-LABEL: @SelectStruct
-; CHECK: select i1 {{.*}}, { i64, i64 }
-; CHECK-NEXT: select i1 {{.*}}, { i64, i64 } { i64 -1, i64 -1 }, { i64, i64 }
-; CHECK-ORIGINS: select i1
-; CHECK-ORIGINS: select i1
-; CHECK-NEXT: select i1 {{.*}}, { i64, i64 }
-; CHECK: ret { i64, i64 }
-
-
-define { i64*, double } @SelectStruct2(i1 zeroext %x, { i64*, double } %a, { i64*, double } %b) readnone sanitize_memory {
-entry:
-  %c = select i1 %x, { i64*, double } %a, { i64*, double } %b
-  ret { i64*, double } %c
-}
-
-; CHECK-LABEL: @SelectStruct2
-; CHECK: select i1 {{.*}}, { i64, i64 }
-; CHECK-NEXT: select i1 {{.*}}, { i64, i64 } { i64 -1, i64 -1 }, { i64, i64 }
-; CHECK-ORIGINS: select i1
-; CHECK-ORIGINS: select i1
-; CHECK-NEXT: select i1 {{.*}}, { i64*, double }
-; CHECK: ret { i64*, double }
-
-
-define i8* @IntToPtr(i64 %x) nounwind uwtable readnone sanitize_memory {
-entry:
-  %0 = inttoptr i64 %x to i8*
-  ret i8* %0
-}
-
-; CHECK-LABEL: @IntToPtr
-; CHECK: load i64, i64*{{.*}}__msan_param_tls
-; CHECK-ORIGINS-NEXT: load i32, i32*{{.*}}__msan_param_origin_tls
-; CHECK-NEXT: inttoptr
-; CHECK-NEXT: store i64{{.*}}__msan_retval_tls
-; CHECK: ret i8*
-
-
-define i8* @IntToPtr_ZExt(i16 %x) nounwind uwtable readnone sanitize_memory {
-entry:
-  %0 = inttoptr i16 %x to i8*
-  ret i8* %0
-}
-
-; CHECK-LABEL: @IntToPtr_ZExt
-; CHECK: load i16, i16*{{.*}}__msan_param_tls
-; CHECK: zext
-; CHECK-NEXT: inttoptr
-; CHECK-NEXT: store i64{{.*}}__msan_retval_tls
-; CHECK: ret i8*
-
-
-; Check that we insert exactly one check on udiv
-; (2nd arg shadow is checked, 1st arg shadow is propagated)
-
-define i32 @Div(i32 %a, i32 %b) nounwind uwtable readnone sanitize_memory {
-entry:
-  %div = udiv i32 %a, %b
-  ret i32 %div
-}
-
-; CHECK-LABEL: @Div
-; CHECK: icmp
-; CHECK: call void @__msan_warning
-; CHECK-NOT: icmp
-; CHECK: udiv
-; CHECK-NOT: icmp
-; CHECK: ret i32
-
-
-; Check that we propagate shadow for x<0, x>=0, etc (i.e. sign bit tests)
-
-define zeroext i1 @ICmpSLTZero(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp slt i32 %x, 0
-  ret i1 %1
-}
-
-; CHECK-LABEL: @ICmpSLTZero
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-define zeroext i1 @ICmpSGEZero(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp sge i32 %x, 0
-  ret i1 %1
-}
-
-; CHECK-LABEL: @ICmpSGEZero
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp sge
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-define zeroext i1 @ICmpSGTZero(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp sgt i32 0, %x
-  ret i1 %1
-}
-
-; CHECK-LABEL: @ICmpSGTZero
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp sgt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-define zeroext i1 @ICmpSLEZero(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp sle i32 0, %x
-  ret i1 %1
-}
-
-; CHECK-LABEL: @ICmpSLEZero
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp sle
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-
-; Check that we propagate shadow for x<=-1, x>-1, etc (i.e. sign bit tests)
-
-define zeroext i1 @ICmpSLTAllOnes(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp slt i32 -1, %x
-  ret i1 %1
-}
-
-; CHECK-LABEL: @ICmpSLTAllOnes
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-define zeroext i1 @ICmpSGEAllOnes(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp sge i32 -1, %x
-  ret i1 %1
-}
-
-; CHECK-LABEL: @ICmpSGEAllOnes
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp sge
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-define zeroext i1 @ICmpSGTAllOnes(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp sgt i32 %x, -1
-  ret i1 %1
-}
-
-; CHECK-LABEL: @ICmpSGTAllOnes
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp sgt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-define zeroext i1 @ICmpSLEAllOnes(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp sle i32 %x, -1
-  ret i1 %1
-}
-
-; CHECK-LABEL: @ICmpSLEAllOnes
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp sle
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-
-; Check that we propagate shadow for x<0, x>=0, etc (i.e. sign bit tests)
-; of the vector arguments.
-
-define <2 x i1> @ICmpSLT_vector_Zero(<2 x i32*> %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp slt <2 x i32*> %x, zeroinitializer
-  ret <2 x i1> %1
-}
-
-; CHECK-LABEL: @ICmpSLT_vector_Zero
-; CHECK: icmp slt <2 x i64>
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp slt <2 x i32*>
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret <2 x i1>
-
-; Check that we propagate shadow for x<=-1, x>0, etc (i.e. sign bit tests)
-; of the vector arguments.
-
-define <2 x i1> @ICmpSLT_vector_AllOnes(<2 x i32> %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp slt <2 x i32> <i32 -1, i32 -1>, %x
-  ret <2 x i1> %1
-}
-
-; CHECK-LABEL: @ICmpSLT_vector_AllOnes
-; CHECK: icmp slt <2 x i32>
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp slt <2 x i32>
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret <2 x i1>
-
-
-; Check that we propagate shadow for unsigned relational comparisons with
-; constants
-
-define zeroext i1 @ICmpUGTConst(i32 %x) nounwind uwtable readnone sanitize_memory {
-entry:
-  %cmp = icmp ugt i32 %x, 7
-  ret i1 %cmp
-}
-
-; CHECK-LABEL: @ICmpUGTConst
-; CHECK: icmp ugt i32
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp ugt i32
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp ugt i32
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-
-; Check that loads of shadow have the same alignment as the original loads.
-; Check that loads of origin have the alignment of max(4, original alignment).
-
-define i32 @ShadowLoadAlignmentLarge() nounwind uwtable sanitize_memory {
-  %y = alloca i32, align 64
-  %1 = load volatile i32, i32* %y, align 64
-  ret i32 %1
-}
-
-; CHECK-LABEL: @ShadowLoadAlignmentLarge
-; CHECK: load volatile i32, i32* {{.*}} align 64
-; CHECK: load i32, i32* {{.*}} align 64
-; CHECK: ret i32
-
-define i32 @ShadowLoadAlignmentSmall() nounwind uwtable sanitize_memory {
-  %y = alloca i32, align 2
-  %1 = load volatile i32, i32* %y, align 2
-  ret i32 %1
-}
-
-; CHECK-LABEL: @ShadowLoadAlignmentSmall
-; CHECK: load volatile i32, i32* {{.*}} align 2
-; CHECK: load i32, i32* {{.*}} align 2
-; CHECK-ORIGINS: load i32, i32* {{.*}} align 4
-; CHECK: ret i32
-
-
-; Test vector manipulation instructions.
-; Check that the same bit manipulation is applied to the shadow values.
-; Check that there is a zero test of the shadow of %idx argument, where present.
-
-define i32 @ExtractElement(<4 x i32> %vec, i32 %idx) sanitize_memory {
-  %x = extractelement <4 x i32> %vec, i32 %idx
-  ret i32 %x
-}
-
-; CHECK-LABEL: @ExtractElement
-; CHECK: extractelement
-; CHECK: call void @__msan_warning
-; CHECK: extractelement
-; CHECK: ret i32
-
-define <4 x i32> @InsertElement(<4 x i32> %vec, i32 %idx, i32 %x) sanitize_memory {
-  %vec1 = insertelement <4 x i32> %vec, i32 %x, i32 %idx
-  ret <4 x i32> %vec1
-}
-
-; CHECK-LABEL: @InsertElement
-; CHECK: insertelement
-; CHECK: call void @__msan_warning
-; CHECK: insertelement
-; CHECK: ret <4 x i32>
-
-define <4 x i32> @ShuffleVector(<4 x i32> %vec, <4 x i32> %vec1) sanitize_memory {
-  %vec2 = shufflevector <4 x i32> %vec, <4 x i32> %vec1,
-                        <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  ret <4 x i32> %vec2
-}
-
-; CHECK-LABEL: @ShuffleVector
-; CHECK: shufflevector
-; CHECK-NOT: call void @__msan_warning
-; CHECK: shufflevector
-; CHECK: ret <4 x i32>
-
-
-; Test bswap intrinsic instrumentation
-define i32 @BSwap(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %y = tail call i32 @llvm.bswap.i32(i32 %x)
-  ret i32 %y
-}
-
-declare i32 @llvm.bswap.i32(i32) nounwind readnone
-
-; CHECK-LABEL: @BSwap
-; CHECK-NOT: call void @__msan_warning
-; CHECK: @llvm.bswap.i32
-; CHECK-NOT: call void @__msan_warning
-; CHECK: @llvm.bswap.i32
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i32
-
-; Test handling of vectors of pointers.
-; Check that shadow of such vector is a vector of integers.
-
-define <8 x i8*> @VectorOfPointers(<8 x i8*>* %p) nounwind uwtable sanitize_memory {
-  %x = load <8 x i8*>, <8 x i8*>* %p
-  ret <8 x i8*> %x
-}
-
-; CHECK-LABEL: @VectorOfPointers
-; CHECK: load <8 x i8*>, <8 x i8*>*
-; CHECK: load <8 x i64>, <8 x i64>*
-; CHECK: store <8 x i64> {{.*}} @__msan_retval_tls
-; CHECK: ret <8 x i8*>
-
-; Test handling of va_copy.
-
-declare void @llvm.va_copy(i8*, i8*) nounwind
-
-define void @VACopy(i8* %p1, i8* %p2) nounwind uwtable sanitize_memory {
-  call void @llvm.va_copy(i8* %p1, i8* %p2) nounwind
-  ret void
-}
-
-; CHECK-LABEL: @VACopy
-; CHECK: call void @llvm.memset.p0i8.i64({{.*}}, i8 0, i64 24, i32 8, i1 false)
-; CHECK: ret void
-
-
-; Test that va_start instrumentation does not use va_arg_tls*.
-; It should work with a local stack copy instead.
-
-%struct.__va_list_tag = type { i32, i32, i8*, i8* }
-declare void @llvm.va_start(i8*) nounwind
-
-; Function Attrs: nounwind uwtable
-define void @VAStart(i32 %x, ...) sanitize_memory {
-entry:
-  %x.addr = alloca i32, align 4
-  %va = alloca [1 x %struct.__va_list_tag], align 16
-  store i32 %x, i32* %x.addr, align 4
-  %arraydecay = getelementptr inbounds [1 x %struct.__va_list_tag], [1 x %struct.__va_list_tag]* %va, i32 0, i32 0
-  %arraydecay1 = bitcast %struct.__va_list_tag* %arraydecay to i8*
-  call void @llvm.va_start(i8* %arraydecay1)
-  ret void
-}
-
-; CHECK-LABEL: @VAStart
-; CHECK: call void @llvm.va_start
-; CHECK-NOT: @__msan_va_arg_tls
-; CHECK-NOT: @__msan_va_arg_overflow_size_tls
-; CHECK: ret void
-
-
-; Test handling of volatile stores.
-; Check that MemorySanitizer does not add a check of the value being stored.
-
-define void @VolatileStore(i32* nocapture %p, i32 %x) nounwind uwtable sanitize_memory {
-entry:
-  store volatile i32 %x, i32* %p, align 4
-  ret void
-}
-
-; CHECK-LABEL: @VolatileStore
-; CHECK-NOT: @__msan_warning
-; CHECK: ret void
-
-
-; Test that checks are omitted and returned value is always initialized if
-; sanitize_memory attribute is missing.
-
-define i32 @NoSanitizeMemory(i32 %x) uwtable {
-entry:
-  %tobool = icmp eq i32 %x, 0
-  br i1 %tobool, label %if.end, label %if.then
-
-if.then:                                          ; preds = %entry
-  tail call void @bar()
-  br label %if.end
-
-if.end:                                           ; preds = %entry, %if.then
-  ret i32 %x
-}
-
-declare void @bar()
-
-; CHECK-LABEL: @NoSanitizeMemory
-; CHECK-NOT: @__msan_warning
-; CHECK: store i32 0, {{.*}} @__msan_retval_tls
-; CHECK-NOT: @__msan_warning
-; CHECK: ret i32
-
-
-; Test that stack allocations are unpoisoned in functions missing
-; sanitize_memory attribute
-
-define i32 @NoSanitizeMemoryAlloca() {
-entry:
-  %p = alloca i32, align 4
-  %x = call i32 @NoSanitizeMemoryAllocaHelper(i32* %p)
-  ret i32 %x
-}
-
-declare i32 @NoSanitizeMemoryAllocaHelper(i32* %p)
-
-; CHECK-LABEL: @NoSanitizeMemoryAlloca
-; CHECK: call void @llvm.memset.p0i8.i64(i8* {{.*}}, i8 0, i64 4, i32 4, i1 false)
-; CHECK: call i32 @NoSanitizeMemoryAllocaHelper(i32*
-; CHECK: ret i32
-
-
-; Test that undef is unpoisoned in functions missing
-; sanitize_memory attribute
-
-define i32 @NoSanitizeMemoryUndef() {
-entry:
-  %x = call i32 @NoSanitizeMemoryUndefHelper(i32 undef)
-  ret i32 %x
-}
-
-declare i32 @NoSanitizeMemoryUndefHelper(i32 %x)
-
-; CHECK-LABEL: @NoSanitizeMemoryUndef
-; CHECK: store i32 0, i32* {{.*}} @__msan_param_tls
-; CHECK: call i32 @NoSanitizeMemoryUndefHelper(i32 undef)
-; CHECK: ret i32
-
-
-; Test PHINode instrumentation in blacklisted functions
-
-define i32 @NoSanitizeMemoryPHI(i32 %x) {
-entry:
-  %tobool = icmp ne i32 %x, 0
-  br i1 %tobool, label %cond.true, label %cond.false
-
-cond.true:                                        ; preds = %entry
-  br label %cond.end
-
-cond.false:                                       ; preds = %entry
-  br label %cond.end
-
-cond.end:                                         ; preds = %cond.false, %cond.true
-  %cond = phi i32 [ undef, %cond.true ], [ undef, %cond.false ]
-  ret i32 %cond
-}
-
-; CHECK: [[A:%.*]] = phi i32 [ undef, %cond.true ], [ undef, %cond.false ]
-; CHECK: store i32 0, i32* bitcast {{.*}} @__msan_retval_tls
-; CHECK: ret i32 [[A]]
-
-
-; Test that there are no __msan_param_origin_tls stores when
-; argument shadow is a compile-time zero constant (which is always the case
-; in functions missing sanitize_memory attribute).
-
-define i32 @NoSanitizeMemoryParamTLS(i32* nocapture readonly %x) {
-entry:
-  %0 = load i32, i32* %x, align 4
-  %call = tail call i32 @NoSanitizeMemoryParamTLSHelper(i32 %0)
-  ret i32 %call
-}
-
-declare i32 @NoSanitizeMemoryParamTLSHelper(i32 %x)
-
-; CHECK-LABEL: define i32 @NoSanitizeMemoryParamTLS(
-; CHECK-NOT: __msan_param_origin_tls
-; CHECK: ret i32
-
-
-; Test argument shadow alignment
-
-define <2 x i64> @ArgumentShadowAlignment(i64 %a, <2 x i64> %b) sanitize_memory {
-entry:
-  ret <2 x i64> %b
-}
-
-; CHECK-LABEL: @ArgumentShadowAlignment
-; CHECK: load <2 x i64>, <2 x i64>* {{.*}} @__msan_param_tls {{.*}}, align 8
-; CHECK: store <2 x i64> {{.*}} @__msan_retval_tls {{.*}}, align 8
-; CHECK: ret <2 x i64>
-
-
-; Test origin propagation for insertvalue
-
-define { i64, i32 } @make_pair_64_32(i64 %x, i32 %y) sanitize_memory {
-entry:
-  %a = insertvalue { i64, i32 } undef, i64 %x, 0
-  %b = insertvalue { i64, i32 } %a, i32 %y, 1
-  ret { i64, i32 } %b
-}
-
-; CHECK-ORIGINS: @make_pair_64_32
-; First element shadow
-; CHECK-ORIGINS: insertvalue { i64, i32 } { i64 -1, i32 -1 }, i64 {{.*}}, 0
-; First element origin
-; CHECK-ORIGINS: icmp ne i64
-; CHECK-ORIGINS: select i1
-; First element app value
-; CHECK-ORIGINS: insertvalue { i64, i32 } undef, i64 {{.*}}, 0
-; Second element shadow
-; CHECK-ORIGINS: insertvalue { i64, i32 } {{.*}}, i32 {{.*}}, 1
-; Second element origin
-; CHECK-ORIGINS: icmp ne i32
-; CHECK-ORIGINS: select i1
-; Second element app value
-; CHECK-ORIGINS: insertvalue { i64, i32 } {{.*}}, i32 {{.*}}, 1
-; CHECK-ORIGINS: ret { i64, i32 }
-
-
-; Test shadow propagation for aggregates passed through ellipsis.
-
-%struct.StructByVal = type { i32, i32, i32, i32 }
-
-declare void @VAArgStructFn(i32 %guard, ...)
-
-define void @VAArgStruct(%struct.StructByVal* nocapture %s) sanitize_memory {
-entry:
-  %agg.tmp2 = alloca %struct.StructByVal, align 8
-  %0 = bitcast %struct.StructByVal* %s to i8*
-  %agg.tmp.sroa.0.0..sroa_cast = bitcast %struct.StructByVal* %s to i64*
-  %agg.tmp.sroa.0.0.copyload = load i64, i64* %agg.tmp.sroa.0.0..sroa_cast, align 4
-  %agg.tmp.sroa.2.0..sroa_idx = getelementptr inbounds %struct.StructByVal, %struct.StructByVal* %s, i64 0, i32 2
-  %agg.tmp.sroa.2.0..sroa_cast = bitcast i32* %agg.tmp.sroa.2.0..sroa_idx to i64*
-  %agg.tmp.sroa.2.0.copyload = load i64, i64* %agg.tmp.sroa.2.0..sroa_cast, align 4
-  %1 = bitcast %struct.StructByVal* %agg.tmp2 to i8*
-  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %1, i8* %0, i64 16, i32 4, i1 false)
-  call void (i32, ...) @VAArgStructFn(i32 undef, i64 %agg.tmp.sroa.0.0.copyload, i64 %agg.tmp.sroa.2.0.copyload, i64 %agg.tmp.sroa.0.0.copyload, i64 %agg.tmp.sroa.2.0.copyload, %struct.StructByVal* byval align 8 %agg.tmp2)
-  ret void
-}
-
-; "undef" and the first 2 structs go to general purpose registers;
-; the third struct goes to the overflow area byval
-
-; CHECK-LABEL: @VAArgStruct
-; undef not stored to __msan_va_arg_tls - it's a fixed argument
-; first struct through general purpose registers
-; CHECK: store i64 {{.*}}, i64* {{.*}}@__msan_va_arg_tls{{.*}}, i64 8){{.*}}, align 8
-; CHECK: store i64 {{.*}}, i64* {{.*}}@__msan_va_arg_tls{{.*}}, i64 16){{.*}}, align 8
-; second struct through general purpose registers
-; CHECK: store i64 {{.*}}, i64* {{.*}}@__msan_va_arg_tls{{.*}}, i64 24){{.*}}, align 8
-; CHECK: store i64 {{.*}}, i64* {{.*}}@__msan_va_arg_tls{{.*}}, i64 32){{.*}}, align 8
-; third struct through the overflow area byval
-; CHECK: ptrtoint %struct.StructByVal* {{.*}} to i64
-; CHECK: bitcast { i32, i32, i32, i32 }* {{.*}}@__msan_va_arg_tls {{.*}}, i64 176
-; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64
-; CHECK: store i64 16, i64* @__msan_va_arg_overflow_size_tls
-; CHECK: call void (i32, ...) @VAArgStructFn
-; CHECK: ret void
-
-
-declare i32 @InnerTailCall(i32 %a)
-
-define void @MismatchedReturnTypeTailCall(i32 %a) sanitize_memory {
-  %b = tail call i32 @InnerTailCall(i32 %a)
-  ret void
-}
-
-; We used to strip off the 'tail' modifier, but now that we unpoison return slot
-; shadow before the call, we don't need to anymore.
-
-; CHECK-LABEL: define void @MismatchedReturnTypeTailCall
-; CHECK: tail call i32 @InnerTailCall
-; CHECK: ret void
-
-
-declare i32 @MustTailCall(i32 %a)
-
-define i32 @CallMustTailCall(i32 %a) sanitize_memory {
-  %b = musttail call i32 @MustTailCall(i32 %a)
-  ret i32 %b
-}
-
-; For "musttail" calls we can not insert any shadow manipulating code between
-; call and the return instruction. And we don't need to, because everything is
-; taken care of in the callee.
-
-; CHECK-LABEL: define i32 @CallMustTailCall
-; CHECK: musttail call i32 @MustTailCall
-; No instrumentation between call and ret.
-; CHECK-NEXT: ret i32
-
-declare i32* @MismatchingMustTailCall(i32 %a)
-
-define i8* @MismatchingCallMustTailCall(i32 %a) sanitize_memory {
-  %b = musttail call i32* @MismatchingMustTailCall(i32 %a)
-  %c = bitcast i32* %b to i8*
-  ret i8* %c
-}
-
-; For "musttail" calls we can not insert any shadow manipulating code between
-; call and the return instruction. And we don't need to, because everything is
-; taken care of in the callee.
-
-; CHECK-LABEL: define i8* @MismatchingCallMustTailCall
-; CHECK: musttail call i32* @MismatchingMustTailCall
-; No instrumentation between call and ret.
-; CHECK-NEXT: bitcast i32* {{.*}} to i8*
-; CHECK-NEXT: ret i8*
-
-
-; CHECK-LABEL: define internal void @msan.module_ctor() {
-; CHECK: call void @__msan_init()
Index: test/Instrumentation/MemorySanitizer/msan_common_basic.ll
===================================================================
--- test/Instrumentation/MemorySanitizer/msan_common_basic.ll
+++ test/Instrumentation/MemorySanitizer/msan_common_basic.ll
@@ -0,0 +1,518 @@
+; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck -check-prefix=CHECK -check-prefix=CHECK-MSAN %s
+; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=1 -S | FileCheck -check-prefix=CHECK -check-prefix=CHECK-ORIGINS -check-prefix=CHECK-MSAN %s
+; RUN: opt < %s -msan -msan-kernel=1 -S | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-KMSAN -check-prefix=CHECK-ORIGINS
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Check that we generate PHIs for shadow.
+define void @FuncWithPhi(i32* nocapture %a, i32* %b, i32* nocapture %c) nounwind uwtable sanitize_memory {
+entry:
+  %tobool = icmp eq i32* %b, null
+  br i1 %tobool, label %if.else, label %if.then
+
+  if.then:                                          ; preds = %entry
+  %0 = load i32, i32* %b, align 4
+  br label %if.end
+
+  if.else:                                          ; preds = %entry
+  %1 = load i32, i32* %c, align 4
+  br label %if.end
+
+  if.end:                                           ; preds = %if.else, %if.then
+  %t.0 = phi i32 [ %0, %if.then ], [ %1, %if.else ]
+  store i32 %t.0, i32* %a, align 4
+  ret void
+}
+
+; CHECK-LABEL: @FuncWithPhi
+; CHECK: = phi
+; CHECK-NEXT: = phi
+; CHECK: store
+; CHECK: store
+; CHECK: ret void
+
+
+; Compute shadow for "x << 10"
+define void @ShlConst(i32* nocapture %x) nounwind uwtable sanitize_memory {
+entry:
+  %0 = load i32, i32* %x, align 4
+  %1 = shl i32 %0, 10
+  store i32 %1, i32* %x, align 4
+  ret void
+}
+
+; CHECK-LABEL: @ShlConst
+; CHECK: = load
+; CHECK: = load
+; CHECK: shl
+; CHECK: shl
+; CHECK: store
+; CHECK: store
+; CHECK: ret void
+
+; Compute shadow for "10 << x": it should have 'sext i1'.
+define void @ShlNonConst(i32* nocapture %x) nounwind uwtable sanitize_memory {
+entry:
+  %0 = load i32, i32* %x, align 4
+  %1 = shl i32 10, %0
+  store i32 %1, i32* %x, align 4
+  ret void
+}
+
+; CHECK-LABEL: @ShlNonConst
+; CHECK: = load
+; CHECK: = load
+; CHECK: = sext i1
+; CHECK: store
+; CHECK: store
+; CHECK: ret void
+
+; SExt
+define void @SExt(i32* nocapture %a, i16* nocapture %b) nounwind uwtable sanitize_memory {
+entry:
+  %0 = load i16, i16* %b, align 2
+  %1 = sext i16 %0 to i32
+  store i32 %1, i32* %a, align 4
+  ret void
+}
+
+; CHECK-LABEL: @SExt
+; CHECK: = load
+; CHECK: = load
+; CHECK: = sext
+; CHECK: = sext
+; CHECK: store
+; CHECK: store
+; CHECK: ret void
+
+
+; memset
+define void @MemSet(i8* nocapture %x) nounwind uwtable sanitize_memory {
+entry:
+  call void @llvm.memset.p0i8.i64(i8* %x, i8 42, i64 10, i32 1, i1 false)
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i32, i1) nounwind
+
+; CHECK-LABEL: @MemSet
+; CHECK: call i8* @__msan_memset
+; CHECK: ret void
+
+
+; memcpy
+define void @MemCpy(i8* nocapture %x, i8* nocapture %y) nounwind uwtable sanitize_memory {
+entry:
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %x, i8* %y, i64 10, i32 1, i1 false)
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1) nounwind
+
+; CHECK-LABEL: @MemCpy
+; CHECK: call i8* @__msan_memcpy
+; CHECK: ret void
+
+
+; memmove is lowered to a call
+define void @MemMove(i8* nocapture %x, i8* nocapture %y) nounwind uwtable sanitize_memory {
+entry:
+  call void @llvm.memmove.p0i8.p0i8.i64(i8* %x, i8* %y, i64 10, i32 1, i1 false)
+  ret void
+}
+
+declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1) nounwind
+
+; CHECK-LABEL: @MemMove
+; CHECK: call i8* @__msan_memmove
+; CHECK: ret void
+
+
+;; ------------
+;; Placeholder tests that will fail once element atomic @llvm.mem[cpy|move|set] instrinsics have
+;; been added to the MemIntrinsic class hierarchy. These will act as a reminder to
+;; verify that MSAN handles these intrinsics properly once they have been
+;; added to that class hierarchy.
+declare void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* nocapture writeonly, i8, i64, i32) nounwind
+declare void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32) nounwind
+declare void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32) nounwind
+
+define void @atomic_memcpy(i8* nocapture %x, i8* nocapture %y) nounwind {
+  ; CHECK-LABEL: atomic_memcpy
+  ; CHECK-KMSAN: .split:
+  ; CHECK-NEXT: call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %x, i8* align 2 %y, i64 16, i32 1)
+  ; CHECK-NEXT: ret void
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %x, i8* align 2 %y, i64 16, i32 1)
+  ret void
+}
+
+define void @atomic_memmove(i8* nocapture %x, i8* nocapture %y) nounwind {
+  ; CHECK-LABEL: atomic_memmove
+  ; CHECK-KMSAN: .split:
+  ; CHECK-NEXT: call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %x, i8* align 2 %y, i64 16, i32 1)
+  ; CHECK-NEXT: ret void
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %x, i8* align 2 %y, i64 16, i32 1)
+  ret void
+}
+
+define void @atomic_memset(i8* nocapture %x) nounwind {
+  ; CHECK-LABEL: atomic_memset
+  ; CHECK-KMSAN: .split:
+  ; CHECK-NEXT: call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 1 %x, i8 88, i64 16, i32 1)
+  ; CHECK-NEXT: ret void
+  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 1 %x, i8 88, i64 16, i32 1)
+  ret void
+}
+
+;; ------------
+
+; Check that we insert exactly one check on udiv
+; (2nd arg shadow is checked, 1st arg shadow is propagated)
+
+define i32 @Div(i32 %a, i32 %b) nounwind uwtable readnone sanitize_memory {
+entry:
+  %div = udiv i32 %a, %b
+  ret i32 %div
+}
+
+; CHECK-LABEL: @Div
+; CHECK: icmp
+; CHECK: call void @__msan_warning{{.*}}
+; CHECK-NOT: icmp
+; CHECK: udiv
+; CHECK-NOT: icmp
+; CHECK: ret i32
+
+
+; Check that we propagate shadow for x<0, x>=0, etc (i.e. sign bit tests)
+
+define zeroext i1 @ICmpSLTZero(i32 %x) nounwind uwtable readnone sanitize_memory {
+  %1 = icmp slt i32 %x, 0
+  ret i1 %1
+}
+
+; CHECK-LABEL: @ICmpSLTZero
+; CHECK: icmp slt
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: icmp slt
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: ret i1
+
+define zeroext i1 @ICmpSGEZero(i32 %x) nounwind uwtable readnone sanitize_memory {
+  %1 = icmp sge i32 %x, 0
+  ret i1 %1
+}
+
+; CHECK-LABEL: @ICmpSGEZero
+; CHECK: icmp slt
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: icmp sge
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: ret i1
+
+define zeroext i1 @ICmpSGTZero(i32 %x) nounwind uwtable readnone sanitize_memory {
+  %1 = icmp sgt i32 0, %x
+  ret i1 %1
+}
+
+; CHECK-LABEL: @ICmpSGTZero
+; CHECK: icmp slt
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: icmp sgt
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: ret i1
+
+define zeroext i1 @ICmpSLEZero(i32 %x) nounwind uwtable readnone sanitize_memory {
+  %1 = icmp sle i32 0, %x
+  ret i1 %1
+}
+
+; CHECK-LABEL: @ICmpSLEZero
+; CHECK: icmp slt
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: icmp sle
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: ret i1
+
+
+; Check that we propagate shadow for x<=-1, x>-1, etc (i.e. sign bit tests)
+
+define zeroext i1 @ICmpSLTAllOnes(i32 %x) nounwind uwtable readnone sanitize_memory {
+  %1 = icmp slt i32 -1, %x
+  ret i1 %1
+}
+
+; CHECK-LABEL: @ICmpSLTAllOnes
+; CHECK: icmp slt
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: icmp slt
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: ret i1
+
+define zeroext i1 @ICmpSGEAllOnes(i32 %x) nounwind uwtable readnone sanitize_memory {
+  %1 = icmp sge i32 -1, %x
+  ret i1 %1
+}
+
+; CHECK-LABEL: @ICmpSGEAllOnes
+; CHECK: icmp slt
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: icmp sge
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: ret i1
+
+define zeroext i1 @ICmpSGTAllOnes(i32 %x) nounwind uwtable readnone sanitize_memory {
+  %1 = icmp sgt i32 %x, -1
+  ret i1 %1
+}
+
+; CHECK-LABEL: @ICmpSGTAllOnes
+; CHECK: icmp slt
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: icmp sgt
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: ret i1
+
+define zeroext i1 @ICmpSLEAllOnes(i32 %x) nounwind uwtable readnone sanitize_memory {
+  %1 = icmp sle i32 %x, -1
+  ret i1 %1
+}
+
+; CHECK-LABEL: @ICmpSLEAllOnes
+; CHECK: icmp slt
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: icmp sle
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: ret i1
+
+
+; Check that we propagate shadow for x<0, x>=0, etc (i.e. sign bit tests)
+; of the vector arguments.
+
+define <2 x i1> @ICmpSLT_vector_Zero(<2 x i32*> %x) nounwind uwtable readnone sanitize_memory {
+  %1 = icmp slt <2 x i32*> %x, zeroinitializer
+  ret <2 x i1> %1
+}
+
+; CHECK-LABEL: @ICmpSLT_vector_Zero
+; CHECK: icmp slt <2 x i64>
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: icmp slt <2 x i32*>
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: ret <2 x i1>
+
+; Check that we propagate shadow for x<=-1, x>0, etc (i.e. sign bit tests)
+; of the vector arguments.
+
+define <2 x i1> @ICmpSLT_vector_AllOnes(<2 x i32> %x) nounwind uwtable readnone sanitize_memory {
+  %1 = icmp slt <2 x i32> <i32 -1, i32 -1>, %x
+  ret <2 x i1> %1
+}
+
+; CHECK-LABEL: @ICmpSLT_vector_AllOnes
+; CHECK: icmp slt <2 x i32>
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: icmp slt <2 x i32>
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: ret <2 x i1>
+
+
+; Check that we propagate shadow for unsigned relational comparisons with
+; constants
+
+define zeroext i1 @ICmpUGTConst(i32 %x) nounwind uwtable readnone sanitize_memory {
+entry:
+  %cmp = icmp ugt i32 %x, 7
+  ret i1 %cmp
+}
+
+; CHECK-LABEL: @ICmpUGTConst
+; CHECK: icmp ugt i32
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: icmp ugt i32
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: icmp ugt i32
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: ret i1
+
+
+; Check that loads of shadow have the same alignment as the original loads.
+; Check that loads of origin have the alignment of max(4, original alignment).
+
+define i32 @ShadowLoadAlignmentLarge() nounwind uwtable sanitize_memory {
+  %y = alloca i32, align 64
+  %1 = load volatile i32, i32* %y, align 64
+  ret i32 %1
+}
+
+; CHECK-LABEL: @ShadowLoadAlignmentLarge
+; CHECK: load volatile i32, i32* {{.*}} align 64
+; CHECK: load i32, i32* {{.*}} align 64
+; CHECK: ret i32
+
+define i32 @ShadowLoadAlignmentSmall() nounwind uwtable sanitize_memory {
+  %y = alloca i32, align 2
+  %1 = load volatile i32, i32* %y, align 2
+  ret i32 %1
+}
+
+; CHECK-LABEL: @ShadowLoadAlignmentSmall
+; CHECK: load volatile i32, i32* {{.*}} align 2
+; CHECK: load i32, i32* {{.*}} align 2
+; CHECK-ORIGINS: load i32, i32* {{.*}} align 4
+; CHECK: ret i32
+
+
+; Test vector manipulation instructions.
+; Check that the same bit manipulation is applied to the shadow values.
+; Check that there is a zero test of the shadow of %idx argument, where present.
+
+define i32 @ExtractElement(<4 x i32> %vec, i32 %idx) sanitize_memory {
+  %x = extractelement <4 x i32> %vec, i32 %idx
+  ret i32 %x
+}
+
+; CHECK-LABEL: @ExtractElement
+; CHECK: extractelement
+; CHECK: call void @__msan_warning{{.*}}
+; CHECK: extractelement
+; CHECK: ret i32
+
+define <4 x i32> @InsertElement(<4 x i32> %vec, i32 %idx, i32 %x) sanitize_memory {
+  %vec1 = insertelement <4 x i32> %vec, i32 %x, i32 %idx
+  ret <4 x i32> %vec1
+}
+
+; CHECK-LABEL: @InsertElement
+; CHECK: insertelement
+; CHECK: call void @__msan_warning{{.*}}
+; CHECK: insertelement
+; CHECK: ret <4 x i32>
+
+define <4 x i32> @ShuffleVector(<4 x i32> %vec, <4 x i32> %vec1) sanitize_memory {
+  %vec2 = shufflevector <4 x i32> %vec, <4 x i32> %vec1,
+                        <4 x i32> <i32 0, i32 4, i32 1, i32 5>
+  ret <4 x i32> %vec2
+}
+
+; CHECK-LABEL: @ShuffleVector
+; CHECK: shufflevector
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: shufflevector
+; CHECK: ret <4 x i32>
+
+
+; Test bswap intrinsic instrumentation
+define i32 @BSwap(i32 %x) nounwind uwtable readnone sanitize_memory {
+  %y = tail call i32 @llvm.bswap.i32(i32 %x)
+  ret i32 %y
+}
+
+declare i32 @llvm.bswap.i32(i32) nounwind readnone
+
+; CHECK-LABEL: @BSwap
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: @llvm.bswap.i32
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: @llvm.bswap.i32
+; CHECK-NOT: call void @__msan_warning{{.*}}
+; CHECK: ret i32
+
+; Test handling of vectors of pointers.
+; Check that shadow of such vector is a vector of integers.
+
+define <8 x i8*> @VectorOfPointers(<8 x i8*>* %p) nounwind uwtable sanitize_memory {
+  %x = load <8 x i8*>, <8 x i8*>* %p
+  ret <8 x i8*> %x
+}
+
+; CHECK-LABEL: @VectorOfPointers
+; CHECK: load <8 x i8*>, <8 x i8*>*
+; CHECK: load <8 x i64>, <8 x i64>*
+; CHECK-MSAN: store <8 x i64> {{.*}} @__msan_retval_tls
+; CHECK-KMSAN: %retval_shadow
+; CHECK-KMSAN: store <8 x i64>
+; CHECK-KMSAN: store {{.*}} %retval_origin
+; CHECK: ret <8 x i8*>
+
+
+; Test origin propagation for insertvalue
+
+define { i64, i32 } @make_pair_64_32(i64 %x, i32 %y) sanitize_memory {
+entry:
+  %a = insertvalue { i64, i32 } undef, i64 %x, 0
+  %b = insertvalue { i64, i32 } %a, i32 %y, 1
+  ret { i64, i32 } %b
+}
+
+; CHECK-ORIGINS: @make_pair_64_32
+; First element shadow
+; CHECK-ORIGINS: insertvalue { i64, i32 } { i64 -1, i32 -1 }, i64 {{.*}}, 0
+; First element origin
+; CHECK-ORIGINS: icmp ne i64
+; CHECK-ORIGINS: select i1
+; First element app value
+; CHECK-ORIGINS: insertvalue { i64, i32 } undef, i64 {{.*}}, 0
+; Second element shadow
+; CHECK-ORIGINS: insertvalue { i64, i32 } {{.*}}, i32 {{.*}}, 1
+; Second element origin
+; CHECK-ORIGINS: icmp ne i32
+; CHECK-ORIGINS: select i1
+; Second element app value
+; CHECK-ORIGINS: insertvalue { i64, i32 } {{.*}}, i32 {{.*}}, 1
+; CHECK-ORIGINS: ret { i64, i32 }
+
+
+declare i32 @InnerTailCall(i32 %a)
+
+define void @MismatchedReturnTypeTailCall(i32 %a) sanitize_memory {
+  %b = tail call i32 @InnerTailCall(i32 %a)
+  ret void
+}
+
+; We used to strip off the 'tail' modifier, but now that we unpoison return slot
+; shadow before the call, we don't need to anymore.
+
+; CHECK-LABEL: define void @MismatchedReturnTypeTailCall
+; CHECK: tail call i32 @InnerTailCall
+; CHECK: ret void
+
+
+declare i32 @MustTailCall(i32 %a)
+
+define i32 @CallMustTailCall(i32 %a) sanitize_memory {
+  %b = musttail call i32 @MustTailCall(i32 %a)
+  ret i32 %b
+}
+
+; For "musttail" calls we can not insert any shadow manipulating code between
+; call and the return instruction. And we don't need to, because everything is
+; taken care of in the callee.
+
+; CHECK-LABEL: define i32 @CallMustTailCall
+; CHECK: musttail call i32 @MustTailCall
+; No instrumentation between call and ret.
+; CHECK-NEXT: ret i32
+
+declare i32* @MismatchingMustTailCall(i32 %a)
+
+define i8* @MismatchingCallMustTailCall(i32 %a) sanitize_memory {
+  %b = musttail call i32* @MismatchingMustTailCall(i32 %a)
+  %c = bitcast i32* %b to i8*
+  ret i8* %c
+}
+
+; For "musttail" calls we can not insert any shadow manipulating code between
+; call and the return instruction. And we don't need to, because everything is
+; taken care of in the callee.
+
+; CHECK-LABEL: define i8* @MismatchingCallMustTailCall
+; CHECK: musttail call i32* @MismatchingMustTailCall
+; No instrumentation between call and ret.
+; CHECK-NEXT: bitcast i32* {{.*}} to i8*
+; CHECK-NEXT: ret i8*
+
+
+; CHECK-LABEL: define internal void @msan.module_ctor() {
+; CHECK: call void @__msan_init()
Index: test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll
===================================================================
--- test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll
+++ test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll
@@ -0,0 +1,509 @@
+; RUN: opt < %s -msan -msan-kernel=1 -S | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-KMSAN -check-prefix=CHECK-ORIGINS
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Make sure the module constructors are there.
+; TODO(glider): why have them?
+; CHECK: @llvm.global_ctors {{.*}} { i32 0, void ()* @msan.module_ctor, i8* null }
+
+; Check the instrumentation prologue.
+define void @Empty() nounwind uwtable sanitize_memory {
+entry:
+  ret void
+}
+
+; CHECK-LABEL: @Empty
+; CHECK: entry:
+; CHECK: @__msan_get_context_state()
+; CHECK: %param_shadow
+; CHECK: %retval_shadow
+; CHECK: %va_arg_shadow
+; CHECK: %va_arg_overflow_size
+; CHECK: %param_origin
+; CHECK: %retval_origin
+; CHECK: entry.split:
+
+; Check instrumentation of stores
+
+define void @Store(i32* nocapture %p, i32 %x) nounwind uwtable sanitize_memory {
+entry:
+  store i32 %x, i32* %p, align 4
+  ret void
+}
+
+; CHECK-LABEL: @Store
+; CHECK: entry:
+; CHECK: @__msan_get_context_state()
+; CHECK: entry.split:
+; CHECK: %param_origin
+; Load the shadow of %p
+; CHECK: load
+; CHECK: [[REG:%[0-9]+]] = bitcast i32* %p to i8*
+; CHECK: @__msan_metadata_ptr_for_store_4(i8* [[REG]])
+; TODO: THIS STORE SHOULD GO BEFORE STORING THE ORIGIN
+; CHECK: store
+; CHECK: icmp
+; CHECK: br i1
+; CHECK: <label>
+; CHECK: @__msan_chain_origin
+; Storing origin here:
+; CHECK: store
+; CHECK: br label
+; CHECK: <label>
+; CHECK: store
+; CHECK: ret void
+
+
+; Check instrumentation of aligned stores
+; Shadow store has the same alignment as the original store; origin store
+; does not specify explicit alignment.
+
+define void @AlignedStore(i32* nocapture %p, i32 %x) nounwind uwtable sanitize_memory {
+entry:
+  store i32 %x, i32* %p, align 32
+  ret void
+}
+
+; CHECK-LABEL: @AlignedStore
+; CHECK: entry:
+; CHECK: @__msan_get_context_state()
+; CHECK: entry.split:
+; CHECK: store {{.*}} align 32
+; CHECK-ORIGINS: icmp
+; CHECK-ORIGINS: br i1
+; CHECK-ORIGINS: <label>
+; CHECK-ORIGINS: store {{.*}} align 32
+; CHECK-ORIGINS: br label
+; CHECK-ORIGINS: <label>
+; CHECK: store {{.*}} align 32
+; CHECK: ret void
+
+
+; load followed by cmp: check that we load the shadow and call __msan_warning_32().
+define void @LoadAndCmp(i32* nocapture %a) nounwind uwtable sanitize_memory {
+entry:
+  %0 = load i32, i32* %a, align 4
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void (...) @foo() nounwind
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  ret void
+}
+
+declare void @foo(...)
+
+; CHECK-LABEL: @LoadAndCmp
+; CHECK: = load
+; CHECK: = load
+; CHECK: call void @__msan_warning{{.*}}
+; CHECK: ret void
+
+; Check that we store the shadow for the retval.
+define i32 @ReturnInt() nounwind uwtable readnone sanitize_memory {
+entry:
+  ret i32 123
+}
+
+; CHECK-LABEL: @ReturnInt
+; CHECK: bitcast {{.*}}%retval_shadow
+; CHECK: store i32 0, {{.*}} align 8
+; CHECK: store i32 0, {{.*}} %retval_origin
+; CHECK: ret i32
+
+; Check that we get the shadow for the retval.
+define void @CopyRetVal(i32* nocapture %a) nounwind uwtable sanitize_memory {
+entry:
+  %call = tail call i32 @ReturnInt() nounwind
+  store i32 %call, i32* %a, align 4
+  ret void
+}
+
+; CHECK-LABEL: @CopyRetVal
+; Zero out the return value shadow before calling ReturnInt().
+; CHECK: %retval_shadow
+; Call ReturnInt()
+; CHECK: tail call i32 @ReturnInt
+; Load the shadow and origin for the return value
+; CHECK: bitcast {{.*}} %retval_shadow
+; CHECK: load
+; CHECK: load {{.*}} %retval_origin
+; Check the address shadow for |a| before writing to it.
+; CHECK: bitcast {{.*}} %a
+; CHECK: icmp
+; CHECK: br
+; CHECK: <label>
+; CHECK: call void @__msan_warning{{.*}}
+; CHECK: <label>
+; CHECK: @__msan_metadata_ptr_for_store_4
+; CHECK: <label>
+; CHECK: call i32 @__msan_chain_origin
+; CHECK: store
+; CHECK: <label>
+; CHECK: store
+; CHECK: ret void
+
+
+
+; Check that we propagate shadow for "select"
+
+define i32 @Select(i32 %a, i32 %b, i1 %c) nounwind uwtable readnone sanitize_memory {
+entry:
+  %cond = select i1 %c, i32 %a, i32 %b
+  ret i32 %cond
+}
+
+; CHECK-LABEL: @Select
+; CHECK: select i1
+; CHECK-DAG: or i32
+; CHECK-DAG: xor i32
+; CHECK: or i32
+; CHECK-DAG: select i1
+; CHECK-ORIGINS-DAG: select
+; CHECK-ORIGINS-DAG: select
+; CHECK-DAG: select i1
+; CHECK: %retval_shadow
+; CHECK: store i32
+; CHECK: store i32 {{.*}} %retval_origin
+; CHECK: ret i32
+
+
+; Check that we propagate origin for "select" with vector condition.
+; Select condition is flattened to i1, which is then used to select one of the
+; argument origins.
+
+define <8 x i16> @SelectVector(<8 x i16> %a, <8 x i16> %b, <8 x i1> %c) nounwind uwtable readnone sanitize_memory {
+entry:
+  %cond = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %cond
+}
+
+; CHECK-LABEL: @SelectVector
+; CHECK: select <8 x i1>
+; CHECK-DAG: or <8 x i16>
+; CHECK-DAG: xor <8 x i16>
+; CHECK: or <8 x i16>
+; CHECK-DAG: select <8 x i1>
+; CHECK-ORIGINS-DAG: select
+; CHECK-ORIGINS-DAG: select
+; CHECK-DAG: select <8 x i1>
+; CHECK: %retval_shadow
+; CHECK: store <8 x i16>{{.*}}
+; CHECK: store i32 {{.*}} %retval_origin
+; CHECK: ret <8 x i16>
+
+
+; Check that we propagate origin for "select" with scalar condition and vector
+; arguments. Select condition shadow is sign-extended to the vector type and
+; mixed into the result shadow.
+
+define <8 x i16> @SelectVector2(<8 x i16> %a, <8 x i16> %b, i1 %c) nounwind uwtable readnone sanitize_memory {
+entry:
+  %cond = select i1 %c, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %cond
+}
+
+; CHECK-LABEL: @SelectVector2
+; CHECK: select i1
+; CHECK-DAG: or <8 x i16>
+; CHECK-DAG: xor <8 x i16>
+; CHECK: or <8 x i16>
+; CHECK-DAG: select i1
+; CHECK-ORIGINS-DAG: select i1
+; CHECK-ORIGINS-DAG: select i1
+; CHECK-DAG: select i1
+; CHECK: ret <8 x i16>
+
+
+define { i64, i64 } @SelectStruct(i1 zeroext %x, { i64, i64 } %a, { i64, i64 } %b) readnone sanitize_memory {
+entry:
+  %c = select i1 %x, { i64, i64 } %a, { i64, i64 } %b
+  ret { i64, i64 } %c
+}
+
+; CHECK-LABEL: @SelectStruct
+; CHECK: select i1 {{.*}}, { i64, i64 }
+; CHECK-NEXT: select i1 {{.*}}, { i64, i64 } { i64 -1, i64 -1 }, { i64, i64 }
+; CHECK-ORIGINS: select i1
+; CHECK-ORIGINS: select i1
+; CHECK-NEXT: select i1 {{.*}}, { i64, i64 }
+; CHECK: ret { i64, i64 }
+
+
+define { i64*, double } @SelectStruct2(i1 zeroext %x, { i64*, double } %a, { i64*, double } %b) readnone sanitize_memory {
+entry:
+  %c = select i1 %x, { i64*, double } %a, { i64*, double } %b
+  ret { i64*, double } %c
+}
+
+; CHECK-LABEL: @SelectStruct2
+; CHECK: select i1 {{.*}}, { i64, i64 }
+; CHECK-NEXT: select i1 {{.*}}, { i64, i64 } { i64 -1, i64 -1 }, { i64, i64 }
+; CHECK-ORIGINS: select i1
+; CHECK-ORIGINS: select i1
+; CHECK-NEXT: select i1 {{.*}}, { i64*, double }
+; CHECK: ret { i64*, double }
+
+
+define i8* @IntToPtr(i64 %x) nounwind uwtable readnone sanitize_memory {
+entry:
+  %0 = inttoptr i64 %x to i8*
+  ret i8* %0
+}
+
+; CHECK-LABEL: @IntToPtr
+; CHECK: %param_shadow
+; CHECK: load i64
+; CHECK-NEXT: %param_origin
+; CHECK: load
+; CHECK-NEXT: inttoptr
+; CHECK-NEXT: %retval_shadow
+; CHECK: store {{.*}} %retval_origin
+; CHECK: ret i8*
+
+
+define i8* @IntToPtr_ZExt(i16 %x) nounwind uwtable readnone sanitize_memory {
+entry:
+  %0 = inttoptr i16 %x to i8*
+  ret i8* %0
+}
+
+; CHECK-LABEL: @IntToPtr_ZExt
+; CHECK: %param_shadow
+; CHECK: load i16
+; CHECK: zext
+; CHECK-NEXT: inttoptr
+; CHECK-NEXT: %retval_shadow
+; CHECK-NEXT: store
+; CHECK: store {{.*}} %retval_origin
+; CHECK: ret i8*
+
+
+; Test handling of va_copy.
+
+declare void @llvm.va_copy(i8*, i8*) nounwind
+
+define void @VACopy(i8* %p1, i8* %p2) nounwind uwtable sanitize_memory {
+  call void @llvm.va_copy(i8* %p1, i8* %p2) nounwind
+  ret void
+}
+
+; CHECK-LABEL: @VACopy
+; CHECK: @__msan_unpoison(i8* %p1, i64 24)
+; CHECK: ret void
+
+
+; Test that va_start instrumentation does not use va_arg_tls*.
+; It should work with a local stack copy instead.
+
+%struct.__va_list_tag = type { i32, i32, i8*, i8* }
+declare void @llvm.va_start(i8*) nounwind
+
+; Function Attrs: nounwind uwtable
+define void @VAStart(i32 %x, ...) sanitize_memory {
+entry:
+  %x.addr = alloca i32, align 4
+  %va = alloca [1 x %struct.__va_list_tag], align 16
+  store i32 %x, i32* %x.addr, align 4
+  %arraydecay = getelementptr inbounds [1 x %struct.__va_list_tag], [1 x %struct.__va_list_tag]* %va, i32 0, i32 0
+  %arraydecay1 = bitcast %struct.__va_list_tag* %arraydecay to i8*
+  call void @llvm.va_start(i8* %arraydecay1)
+  ret void
+}
+
+; CHECK-LABEL: @VAStart
+; CHECK: call void @llvm.va_start
+; CHECK-NOT: @__msan_va_arg_tls
+; CHECK-NOT: @__msan_va_arg_overflow_size_tls
+; CHECK: ret void
+
+
+; Test handling of volatile stores.
+; Check that MemorySanitizer does not add a check of the value being stored.
+
+define void @VolatileStore(i32* nocapture %p, i32 %x) nounwind uwtable sanitize_memory {
+entry:
+  store volatile i32 %x, i32* %p, align 4
+  ret void
+}
+
+; CHECK-LABEL: @VolatileStore
+; CHECK: @__msan_warning{{.*}}
+; CHECL: __msan_metadata_ptr_for_store_4
+; CHECK-NOT: @__msan_warning{{.*}}
+; CHECK: ret void
+
+
+; Test that checks are omitted and returned value is always initialized if
+; sanitize_memory attribute is missing.
+
+define i32 @NoSanitizeMemory(i32 %x) uwtable {
+entry:
+  %tobool = icmp eq i32 %x, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void @bar()
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  ret i32 %x
+}
+
+declare void @bar()
+
+; CHECK-LABEL: @NoSanitizeMemory
+; CHECK-NOT: @__msan_warning{{.*}}
+; CHECK: %retval_shadow
+; CHECK: store i32 0, i32* %_msret
+; CHECK: store i32 0, i32* %retval_origin
+; CHECK-NOT: @__msan_warning{{.*}}
+; CHECK: ret i32
+
+
+; Test that stack allocations are unpoisoned in functions missing
+; sanitize_memory attribute
+
+define i32 @NoSanitizeMemoryAlloca() {
+entry:
+  %p = alloca i32, align 4
+  %x = call i32 @NoSanitizeMemoryAllocaHelper(i32* %p)
+  ret i32 %x
+}
+
+declare i32 @NoSanitizeMemoryAllocaHelper(i32* %p)
+
+; CHECK-LABEL: @NoSanitizeMemoryAlloca
+; CHECK: call i8* @llvm.returnaddress(i32 0)
+; CHECK: call void @__msan_poison_alloca({{.*}}, i64 4, {{.*}})
+; CHECK: call i32 @NoSanitizeMemoryAllocaHelper(i32* 
+; CHECK: ret i32
+
+
+; Test that undef is unpoisoned in functions missing
+; sanitize_memory attribute
+
+define i32 @NoSanitizeMemoryUndef() {
+entry:
+  %x = call i32 @NoSanitizeMemoryUndefHelper(i32 undef)
+  ret i32 %x
+}
+
+declare i32 @NoSanitizeMemoryUndefHelper(i32 %x)
+
+; CHECK-LABEL: @NoSanitizeMemoryUndef
+; CHECK: %param_shadow
+; CHECK: store i32 0
+; CHECK: call i32 @NoSanitizeMemoryUndefHelper(i32 undef)
+; CHECK: ret i32
+
+
+; Test PHINode instrumentation in blacklisted functions
+
+define i32 @NoSanitizeMemoryPHI(i32 %x) {
+entry:
+  %tobool = icmp ne i32 %x, 0
+  br i1 %tobool, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %entry
+  br label %cond.end
+
+cond.false:                                       ; preds = %entry
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi i32 [ undef, %cond.true ], [ undef, %cond.false ]
+  ret i32 %cond
+}
+
+; CHECK: [[A:%.*]] = phi i32 [ undef, %cond.true ], [ undef, %cond.false ]
+; CHECK: %retval_shadow
+; CHECK: store i32 0
+; CHECK: %retval_origin
+; CHECK: ret i32 [[A]]
+
+
+; Test that there are no __msan_param_origin_tls stores when
+; argument shadow is a compile-time zero constant (which is always the case
+; in functions missing sanitize_memory attribute).
+
+define i32 @NoSanitizeMemoryParamTLS(i32* nocapture readonly %x) {
+entry:
+  %0 = load i32, i32* %x, align 4
+  %call = tail call i32 @NoSanitizeMemoryParamTLSHelper(i32 %0)
+  ret i32 %call
+}
+
+declare i32 @NoSanitizeMemoryParamTLSHelper(i32 %x)
+
+; CHECK-LABEL: define i32 @NoSanitizeMemoryParamTLS(
+; CHECK-NOT: __msan_param_origin_tls
+; CHECK: ret i32
+
+
+; Test argument shadow alignment
+
+define <2 x i64> @ArgumentShadowAlignment(i64 %a, <2 x i64> %b) sanitize_memory {
+entry:
+  ret <2 x i64> %b
+}
+
+; CHECK-LABEL: @ArgumentShadowAlignment
+; CHECK: %param_shadow
+; CHECK: load <2 x i64>, <2 x i64>* {{.*}}, align 8
+; CHECK: bitcast {{.*}} %retval_shadow to <2 x i64>*
+; CHECK: store <2 x i64> {{.*}}, <2 x i64>* {{.*}} align 8
+; CHECK: store i32 {{.*}}, i32* %retval_origin
+; CHECK: ret <2 x i64>
+
+
+; Test shadow propagation for aggregates passed through ellipsis.
+
+%struct.StructByVal = type { i32, i32, i32, i32 }
+
+declare void @VAArgStructFn(i32 %guard, ...)
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1) nounwind
+
+define void @VAArgStruct(%struct.StructByVal* nocapture %s) sanitize_memory {
+entry:
+  %agg.tmp2 = alloca %struct.StructByVal, align 8
+  %0 = bitcast %struct.StructByVal* %s to i8*
+  %agg.tmp.sroa.0.0..sroa_cast = bitcast %struct.StructByVal* %s to i64*
+  %agg.tmp.sroa.0.0.copyload = load i64, i64* %agg.tmp.sroa.0.0..sroa_cast, align 4
+  %agg.tmp.sroa.2.0..sroa_idx = getelementptr inbounds %struct.StructByVal, %struct.StructByVal* %s, i64 0, i32 2
+  %agg.tmp.sroa.2.0..sroa_cast = bitcast i32* %agg.tmp.sroa.2.0..sroa_idx to i64*
+  %agg.tmp.sroa.2.0.copyload = load i64, i64* %agg.tmp.sroa.2.0..sroa_cast, align 4
+  %1 = bitcast %struct.StructByVal* %agg.tmp2 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %1, i8* %0, i64 16, i32 4, i1 false)
+  call void (i32, ...) @VAArgStructFn(i32 undef, i64 %agg.tmp.sroa.0.0.copyload, i64 %agg.tmp.sroa.2.0.copyload, i64 %agg.tmp.sroa.0.0.copyload, i64 %agg.tmp.sroa.2.0.copyload, %struct.StructByVal* byval align 8 %agg.tmp2)
+  ret void
+}
+
+
+; "undef" and the first 2 structs go to general purpose registers;
+; the third struct goes to the overflow area byval
+
+; CHECK-LABEL: @VAArgStruct
+; undef not stored to %va_arg_shadow - it's a fixed argument
+; first struct through general purpose registers
+; CHECK: ptrtoint {{.*}} %va_arg_shadow
+; CHECK: store i64 {{.*}} align 8
+; CHECK: ptrtoint {{.*}} %va_arg_shadow
+; CHECK: store i64 {{.*}} align 8
+; second struct through general purpose registers
+; CHECK: ptrtoint {{.*}} %va_arg_shadow
+; CHECK: store i64 {{.*}} align 8
+; CHECK: ptrtoint {{.*}} %va_arg_shadow
+; CHECK: store i64 {{.*}} align 8
+; third struct through the overflow area byval
+; CHECK: %va_arg_shadow to i64
+; CHECK: add i64 {{.*}}, 176
+; CHECK: call void @__msan_store_arg_shadow({{.*}}, i64 16)
+; CHECK: store i64 16, i64* %va_arg_overflow_size
+; CHECK: call void (i32, ...) @VAArgStructFn
+; CHECK: ret void
+
Index: test/Instrumentation/MemorySanitizer/msan_userspace_basic.ll
===================================================================
--- test/Instrumentation/MemorySanitizer/msan_userspace_basic.ll
+++ test/Instrumentation/MemorySanitizer/msan_userspace_basic.ll
@@ -116,163 +116,6 @@
 ; CHECK: ret void
 
 
-; Check that we generate PHIs for shadow.
-define void @FuncWithPhi(i32* nocapture %a, i32* %b, i32* nocapture %c) nounwind uwtable sanitize_memory {
-entry:
-  %tobool = icmp eq i32* %b, null
-  br i1 %tobool, label %if.else, label %if.then
-
-  if.then:                                          ; preds = %entry
-  %0 = load i32, i32* %b, align 4
-  br label %if.end
-
-  if.else:                                          ; preds = %entry
-  %1 = load i32, i32* %c, align 4
-  br label %if.end
-
-  if.end:                                           ; preds = %if.else, %if.then
-  %t.0 = phi i32 [ %0, %if.then ], [ %1, %if.else ]
-  store i32 %t.0, i32* %a, align 4
-  ret void
-}
-
-; CHECK-LABEL: @FuncWithPhi
-; CHECK: = phi
-; CHECK-NEXT: = phi
-; CHECK: store
-; CHECK: store
-; CHECK: ret void
-
-; Compute shadow for "x << 10"
-define void @ShlConst(i32* nocapture %x) nounwind uwtable sanitize_memory {
-entry:
-  %0 = load i32, i32* %x, align 4
-  %1 = shl i32 %0, 10
-  store i32 %1, i32* %x, align 4
-  ret void
-}
-
-; CHECK-LABEL: @ShlConst
-; CHECK: = load
-; CHECK: = load
-; CHECK: shl
-; CHECK: shl
-; CHECK: store
-; CHECK: store
-; CHECK: ret void
-
-; Compute shadow for "10 << x": it should have 'sext i1'.
-define void @ShlNonConst(i32* nocapture %x) nounwind uwtable sanitize_memory {
-entry:
-  %0 = load i32, i32* %x, align 4
-  %1 = shl i32 10, %0
-  store i32 %1, i32* %x, align 4
-  ret void
-}
-
-; CHECK-LABEL: @ShlNonConst
-; CHECK: = load
-; CHECK: = load
-; CHECK: = sext i1
-; CHECK: store
-; CHECK: store
-; CHECK: ret void
-
-; SExt
-define void @SExt(i32* nocapture %a, i16* nocapture %b) nounwind uwtable sanitize_memory {
-entry:
-  %0 = load i16, i16* %b, align 2
-  %1 = sext i16 %0 to i32
-  store i32 %1, i32* %a, align 4
-  ret void
-}
-
-; CHECK-LABEL: @SExt
-; CHECK: = load
-; CHECK: = load
-; CHECK: = sext
-; CHECK: = sext
-; CHECK: store
-; CHECK: store
-; CHECK: ret void
-
-
-; memset
-define void @MemSet(i8* nocapture %x) nounwind uwtable sanitize_memory {
-entry:
-  call void @llvm.memset.p0i8.i64(i8* %x, i8 42, i64 10, i32 1, i1 false)
-  ret void
-}
-
-declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i32, i1) nounwind
-
-; CHECK-LABEL: @MemSet
-; CHECK: call i8* @__msan_memset
-; CHECK: ret void
-
-
-; memcpy
-define void @MemCpy(i8* nocapture %x, i8* nocapture %y) nounwind uwtable sanitize_memory {
-entry:
-  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %x, i8* %y, i64 10, i32 1, i1 false)
-  ret void
-}
-
-declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1) nounwind
-
-; CHECK-LABEL: @MemCpy
-; CHECK: call i8* @__msan_memcpy
-; CHECK: ret void
-
-
-; memmove is lowered to a call
-define void @MemMove(i8* nocapture %x, i8* nocapture %y) nounwind uwtable sanitize_memory {
-entry:
-  call void @llvm.memmove.p0i8.p0i8.i64(i8* %x, i8* %y, i64 10, i32 1, i1 false)
-  ret void
-}
-
-declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1) nounwind
-
-; CHECK-LABEL: @MemMove
-; CHECK: call i8* @__msan_memmove
-; CHECK: ret void
-
-;; ------------
-;; Placeholder tests that will fail once element atomic @llvm.mem[cpy|move|set] instrinsics have
-;; been added to the MemIntrinsic class hierarchy. These will act as a reminder to
-;; verify that MSAN handles these intrinsics properly once they have been
-;; added to that class hierarchy.
-declare void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* nocapture writeonly, i8, i64, i32) nounwind
-declare void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32) nounwind
-declare void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32) nounwind
-
-define void @atomic_memcpy(i8* nocapture %x, i8* nocapture %y) nounwind {
-  ; CHECK-LABEL: atomic_memcpy
-  ; CHECK-NEXT: call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %x, i8* align 2 %y, i64 16, i32 1)
-  ; CHECK-NEXT: ret void
-  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %x, i8* align 2 %y, i64 16, i32 1)
-  ret void
-}
-
-define void @atomic_memmove(i8* nocapture %x, i8* nocapture %y) nounwind {
-  ; CHECK-LABEL: atomic_memmove
-  ; CHECK-NEXT: call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %x, i8* align 2 %y, i64 16, i32 1)
-  ; CHECK-NEXT: ret void
-  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %x, i8* align 2 %y, i64 16, i32 1)
-  ret void
-}
-
-define void @atomic_memset(i8* nocapture %x) nounwind {
-  ; CHECK-LABEL: atomic_memset
-  ; CHECK-NEXT: call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 1 %x, i8 88, i64 16, i32 1)
-  ; CHECK-NEXT: ret void
-  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 1 %x, i8 88, i64 16, i32 1)
-  ret void
-}
-
-;; ------------
-
 
 ; Check that we propagate shadow for "select"
 
@@ -400,272 +243,6 @@
 ; CHECK: ret i8*
 
 
-; Check that we insert exactly one check on udiv
-; (2nd arg shadow is checked, 1st arg shadow is propagated)
-
-define i32 @Div(i32 %a, i32 %b) nounwind uwtable readnone sanitize_memory {
-entry:
-  %div = udiv i32 %a, %b
-  ret i32 %div
-}
-
-; CHECK-LABEL: @Div
-; CHECK: icmp
-; CHECK: call void @__msan_warning
-; CHECK-NOT: icmp
-; CHECK: udiv
-; CHECK-NOT: icmp
-; CHECK: ret i32
-
-
-; Check that we propagate shadow for x<0, x>=0, etc (i.e. sign bit tests)
-
-define zeroext i1 @ICmpSLTZero(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp slt i32 %x, 0
-  ret i1 %1
-}
-
-; CHECK-LABEL: @ICmpSLTZero
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-define zeroext i1 @ICmpSGEZero(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp sge i32 %x, 0
-  ret i1 %1
-}
-
-; CHECK-LABEL: @ICmpSGEZero
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp sge
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-define zeroext i1 @ICmpSGTZero(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp sgt i32 0, %x
-  ret i1 %1
-}
-
-; CHECK-LABEL: @ICmpSGTZero
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp sgt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-define zeroext i1 @ICmpSLEZero(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp sle i32 0, %x
-  ret i1 %1
-}
-
-; CHECK-LABEL: @ICmpSLEZero
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp sle
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-
-; Check that we propagate shadow for x<=-1, x>-1, etc (i.e. sign bit tests)
-
-define zeroext i1 @ICmpSLTAllOnes(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp slt i32 -1, %x
-  ret i1 %1
-}
-
-; CHECK-LABEL: @ICmpSLTAllOnes
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-define zeroext i1 @ICmpSGEAllOnes(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp sge i32 -1, %x
-  ret i1 %1
-}
-
-; CHECK-LABEL: @ICmpSGEAllOnes
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp sge
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-define zeroext i1 @ICmpSGTAllOnes(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp sgt i32 %x, -1
-  ret i1 %1
-}
-
-; CHECK-LABEL: @ICmpSGTAllOnes
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp sgt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-define zeroext i1 @ICmpSLEAllOnes(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp sle i32 %x, -1
-  ret i1 %1
-}
-
-; CHECK-LABEL: @ICmpSLEAllOnes
-; CHECK: icmp slt
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp sle
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-
-; Check that we propagate shadow for x<0, x>=0, etc (i.e. sign bit tests)
-; of the vector arguments.
-
-define <2 x i1> @ICmpSLT_vector_Zero(<2 x i32*> %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp slt <2 x i32*> %x, zeroinitializer
-  ret <2 x i1> %1
-}
-
-; CHECK-LABEL: @ICmpSLT_vector_Zero
-; CHECK: icmp slt <2 x i64>
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp slt <2 x i32*>
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret <2 x i1>
-
-; Check that we propagate shadow for x<=-1, x>0, etc (i.e. sign bit tests)
-; of the vector arguments.
-
-define <2 x i1> @ICmpSLT_vector_AllOnes(<2 x i32> %x) nounwind uwtable readnone sanitize_memory {
-  %1 = icmp slt <2 x i32> <i32 -1, i32 -1>, %x
-  ret <2 x i1> %1
-}
-
-; CHECK-LABEL: @ICmpSLT_vector_AllOnes
-; CHECK: icmp slt <2 x i32>
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp slt <2 x i32>
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret <2 x i1>
-
-
-; Check that we propagate shadow for unsigned relational comparisons with
-; constants
-
-define zeroext i1 @ICmpUGTConst(i32 %x) nounwind uwtable readnone sanitize_memory {
-entry:
-  %cmp = icmp ugt i32 %x, 7
-  ret i1 %cmp
-}
-
-; CHECK-LABEL: @ICmpUGTConst
-; CHECK: icmp ugt i32
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp ugt i32
-; CHECK-NOT: call void @__msan_warning
-; CHECK: icmp ugt i32
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i1
-
-
-; Check that loads of shadow have the same alignment as the original loads.
-; Check that loads of origin have the alignment of max(4, original alignment).
-
-define i32 @ShadowLoadAlignmentLarge() nounwind uwtable sanitize_memory {
-  %y = alloca i32, align 64
-  %1 = load volatile i32, i32* %y, align 64
-  ret i32 %1
-}
-
-; CHECK-LABEL: @ShadowLoadAlignmentLarge
-; CHECK: load volatile i32, i32* {{.*}} align 64
-; CHECK: load i32, i32* {{.*}} align 64
-; CHECK: ret i32
-
-define i32 @ShadowLoadAlignmentSmall() nounwind uwtable sanitize_memory {
-  %y = alloca i32, align 2
-  %1 = load volatile i32, i32* %y, align 2
-  ret i32 %1
-}
-
-; CHECK-LABEL: @ShadowLoadAlignmentSmall
-; CHECK: load volatile i32, i32* {{.*}} align 2
-; CHECK: load i32, i32* {{.*}} align 2
-; CHECK-ORIGINS: load i32, i32* {{.*}} align 4
-; CHECK: ret i32
-
-
-; Test vector manipulation instructions.
-; Check that the same bit manipulation is applied to the shadow values.
-; Check that there is a zero test of the shadow of %idx argument, where present.
-
-define i32 @ExtractElement(<4 x i32> %vec, i32 %idx) sanitize_memory {
-  %x = extractelement <4 x i32> %vec, i32 %idx
-  ret i32 %x
-}
-
-; CHECK-LABEL: @ExtractElement
-; CHECK: extractelement
-; CHECK: call void @__msan_warning
-; CHECK: extractelement
-; CHECK: ret i32
-
-define <4 x i32> @InsertElement(<4 x i32> %vec, i32 %idx, i32 %x) sanitize_memory {
-  %vec1 = insertelement <4 x i32> %vec, i32 %x, i32 %idx
-  ret <4 x i32> %vec1
-}
-
-; CHECK-LABEL: @InsertElement
-; CHECK: insertelement
-; CHECK: call void @__msan_warning
-; CHECK: insertelement
-; CHECK: ret <4 x i32>
-
-define <4 x i32> @ShuffleVector(<4 x i32> %vec, <4 x i32> %vec1) sanitize_memory {
-  %vec2 = shufflevector <4 x i32> %vec, <4 x i32> %vec1,
-                        <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  ret <4 x i32> %vec2
-}
-
-; CHECK-LABEL: @ShuffleVector
-; CHECK: shufflevector
-; CHECK-NOT: call void @__msan_warning
-; CHECK: shufflevector
-; CHECK: ret <4 x i32>
-
-
-; Test bswap intrinsic instrumentation
-define i32 @BSwap(i32 %x) nounwind uwtable readnone sanitize_memory {
-  %y = tail call i32 @llvm.bswap.i32(i32 %x)
-  ret i32 %y
-}
-
-declare i32 @llvm.bswap.i32(i32) nounwind readnone
-
-; CHECK-LABEL: @BSwap
-; CHECK-NOT: call void @__msan_warning
-; CHECK: @llvm.bswap.i32
-; CHECK-NOT: call void @__msan_warning
-; CHECK: @llvm.bswap.i32
-; CHECK-NOT: call void @__msan_warning
-; CHECK: ret i32
-
-; Test handling of vectors of pointers.
-; Check that shadow of such vector is a vector of integers.
-
-define <8 x i8*> @VectorOfPointers(<8 x i8*>* %p) nounwind uwtable sanitize_memory {
-  %x = load <8 x i8*>, <8 x i8*>* %p
-  ret <8 x i8*> %x
-}
-
-; CHECK-LABEL: @VectorOfPointers
-; CHECK: load <8 x i8*>, <8 x i8*>*
-; CHECK: load <8 x i64>, <8 x i64>*
-; CHECK: store <8 x i64> {{.*}} @__msan_retval_tls
-; CHECK: ret <8 x i8*>
-
 ; Test handling of va_copy.
 
 declare void @llvm.va_copy(i8*, i8*) nounwind
@@ -715,7 +292,7 @@
 }
 
 ; CHECK-LABEL: @VolatileStore
-; CHECK-NOT: @__msan_warning
+; CHECK-NOT: @__msan_warning{{.*}}
 ; CHECK: ret void
 
 
@@ -738,9 +315,9 @@
 declare void @bar()
 
 ; CHECK-LABEL: @NoSanitizeMemory
-; CHECK-NOT: @__msan_warning
+; CHECK-NOT: @__msan_warning{{.*}}
 ; CHECK: store i32 0, {{.*}} @__msan_retval_tls
-; CHECK-NOT: @__msan_warning
+; CHECK-NOT: @__msan_warning{{.*}}
 ; CHECK: ret i32
 
 
@@ -833,38 +410,12 @@
 ; CHECK: ret <2 x i64>
 
 
-; Test origin propagation for insertvalue
-
-define { i64, i32 } @make_pair_64_32(i64 %x, i32 %y) sanitize_memory {
-entry:
-  %a = insertvalue { i64, i32 } undef, i64 %x, 0
-  %b = insertvalue { i64, i32 } %a, i32 %y, 1
-  ret { i64, i32 } %b
-}
-
-; CHECK-ORIGINS: @make_pair_64_32
-; First element shadow
-; CHECK-ORIGINS: insertvalue { i64, i32 } { i64 -1, i32 -1 }, i64 {{.*}}, 0
-; First element origin
-; CHECK-ORIGINS: icmp ne i64
-; CHECK-ORIGINS: select i1
-; First element app value
-; CHECK-ORIGINS: insertvalue { i64, i32 } undef, i64 {{.*}}, 0
-; Second element shadow
-; CHECK-ORIGINS: insertvalue { i64, i32 } {{.*}}, i32 {{.*}}, 1
-; Second element origin
-; CHECK-ORIGINS: icmp ne i32
-; CHECK-ORIGINS: select i1
-; Second element app value
-; CHECK-ORIGINS: insertvalue { i64, i32 } {{.*}}, i32 {{.*}}, 1
-; CHECK-ORIGINS: ret { i64, i32 }
-
-
 ; Test shadow propagation for aggregates passed through ellipsis.
 
 %struct.StructByVal = type { i32, i32, i32, i32 }
 
 declare void @VAArgStructFn(i32 %guard, ...)
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1) nounwind
 
 define void @VAArgStruct(%struct.StructByVal* nocapture %s) sanitize_memory {
 entry:
@@ -881,6 +432,7 @@
   ret void
 }
 
+
 ; "undef" and the first 2 structs go to general purpose registers;
 ; the third struct goes to the overflow area byval
 
@@ -900,56 +452,3 @@
 ; CHECK: call void (i32, ...) @VAArgStructFn
 ; CHECK: ret void
 
-
-declare i32 @InnerTailCall(i32 %a)
-
-define void @MismatchedReturnTypeTailCall(i32 %a) sanitize_memory {
-  %b = tail call i32 @InnerTailCall(i32 %a)
-  ret void
-}
-
-; We used to strip off the 'tail' modifier, but now that we unpoison return slot
-; shadow before the call, we don't need to anymore.
-
-; CHECK-LABEL: define void @MismatchedReturnTypeTailCall
-; CHECK: tail call i32 @InnerTailCall
-; CHECK: ret void
-
-
-declare i32 @MustTailCall(i32 %a)
-
-define i32 @CallMustTailCall(i32 %a) sanitize_memory {
-  %b = musttail call i32 @MustTailCall(i32 %a)
-  ret i32 %b
-}
-
-; For "musttail" calls we can not insert any shadow manipulating code between
-; call and the return instruction. And we don't need to, because everything is
-; taken care of in the callee.
-
-; CHECK-LABEL: define i32 @CallMustTailCall
-; CHECK: musttail call i32 @MustTailCall
-; No instrumentation between call and ret.
-; CHECK-NEXT: ret i32
-
-declare i32* @MismatchingMustTailCall(i32 %a)
-
-define i8* @MismatchingCallMustTailCall(i32 %a) sanitize_memory {
-  %b = musttail call i32* @MismatchingMustTailCall(i32 %a)
-  %c = bitcast i32* %b to i8*
-  ret i8* %c
-}
-
-; For "musttail" calls we can not insert any shadow manipulating code between
-; call and the return instruction. And we don't need to, because everything is
-; taken care of in the callee.
-
-; CHECK-LABEL: define i8* @MismatchingCallMustTailCall
-; CHECK: musttail call i32* @MismatchingMustTailCall
-; No instrumentation between call and ret.
-; CHECK-NEXT: bitcast i32* {{.*}} to i8*
-; CHECK-NEXT: ret i8*
-
-
-; CHECK-LABEL: define internal void @msan.module_ctor() {
-; CHECK: call void @__msan_init()