Index: include/llvm/Transforms/Instrumentation.h =================================================================== --- include/llvm/Transforms/Instrumentation.h +++ include/llvm/Transforms/Instrumentation.h @@ -131,7 +131,8 @@ // Insert MemorySanitizer instrumentation (detection of uninitialized reads) FunctionPass *createMemorySanitizerPass(int TrackOrigins = 0, - bool Recover = false); + bool Recover = false, + bool EnableKmsan = false); FunctionPass *createHWAddressSanitizerPass(bool CompileKernel = false, bool Recover = false); Index: lib/Transforms/Instrumentation/MemorySanitizer.cpp =================================================================== --- lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -89,7 +89,38 @@ /// implementation ignores the load aspect of CAS/RMW, always returning a clean /// value. It implements the store part as a simple atomic store by storing a /// clean shadow. -// +/// +/// KernelMemorySanitizer (KMSAN) implementation. +/// +/// The major differences between KMSAN and MSan instrumentation are: +/// - KMSAN always tracks the origins and implies msan-keep-going=true; +/// - KMSAN allocates shadow and origin memory for each page separately, so +/// there are no explicit accesses to shadow and origin in the +/// instrumentation. +/// Shadow and origin values for a particular X-byte memory location +/// (X=1,2,4,8) are accessed through pointers obtained via the +/// __msan_metadata_ptr_for_load_X(ptr) +/// __msan_metadata_ptr_for_store_X(ptr) +/// functions. The corresponding functions check that the X-byte accesses +/// are possible and returns the pointers to shadow and origin memory. +/// Arbitrary sized accesses are handled with: +/// __msan_metadata_ptr_for_load_n(ptr, size) +/// __msan_metadata_ptr_for_store_n(ptr, size); +/// - TLS variables are stored in a single per-task struct. A call to a +/// function __msan_get_context_state() returning a pointer to that struct +/// is inserted into every instrumented function before the entry block; +/// - __msan_warning() takes a 32-bit origin parameter; +/// - local variables are poisoned with __msan_poison_alloca() upon function +/// entry and unpoisoned with __msan_unpoison_alloca() before leaving the +/// function; +/// - the pass doesn't declare any global variables or add global constructors +/// to the translation unit. +/// +/// Also, KMSAN currently ignores uninitialized memory passed into inline asm +/// calls, making sure we're on the safe side wrt. possible false positives. +/// +/// KernelMemorySanitizer only supports X86_64 at the moment. +/// //===----------------------------------------------------------------------===// #include "llvm/ADT/APInt.h" @@ -233,6 +264,11 @@ "inline checks (-1 means never use callbacks)."), cl::Hidden, cl::init(3500)); +static cl::opt + ClEnableKmsan("msan-kernel", + cl::desc("Enable KernelMemorySanitizer instrumentation"), + cl::Hidden, cl::init(false)); + // This is an experiment to enable handling of cases where shadow is a non-zero // compile-time constant. For some unexplainable reason they were silently // ignored in the instrumentation. @@ -400,11 +436,19 @@ // Pass identification, replacement for typeid. static char ID; - MemorySanitizer(int TrackOrigins = 0, bool Recover = false) - : FunctionPass(ID), - TrackOrigins(std::max(TrackOrigins, (int)ClTrackOrigins)), - Recover(Recover || ClKeepGoing) {} - + MemorySanitizer(int TrackOrigins = 0, bool Recover = false, + bool EnableKmsan = false) + : FunctionPass(ID) { + this->CompileKernel = + ClEnableKmsan.getNumOccurrences() > 0 ? ClEnableKmsan : EnableKmsan; + if (ClTrackOrigins.getNumOccurrences() > 0) + this->TrackOrigins = ClTrackOrigins; + else + this->TrackOrigins = this->CompileKernel ? 2 : TrackOrigins; + this->Recover = ClKeepGoing.getNumOccurrences() > 0 + ? ClKeepGoing + : (this->CompileKernel | Recover); + } StringRef getPassName() const override { return "MemorySanitizer"; } void getAnalysisUsage(AnalysisUsage &AU) const override { @@ -422,8 +466,12 @@ friend struct VarArgPowerPC64Helper; void initializeCallbacks(Module &M); + void createKernelApi(Module &M); void createUserspaceApi(Module &M); + /// True if we're compiling the Linux kernel. + bool CompileKernel; + /// Track origins (allocation points) of uninitialized values. int TrackOrigins; bool Recover; @@ -432,29 +480,39 @@ Type *IntptrTy; Type *OriginTy; + // XxxTLS variables represent the per-thread state in MSan and per-task state + // in KMSAN. + // For the userspace these point to thread-local globals. In the kernel land + // they point to the members of a per-task struct obtained via a call to + // __msan_get_context_state(). + /// Thread-local shadow storage for function parameters. - GlobalVariable *ParamTLS; + Value *ParamTLS; /// Thread-local origin storage for function parameters. - GlobalVariable *ParamOriginTLS; + Value *ParamOriginTLS; /// Thread-local shadow storage for function return value. - GlobalVariable *RetvalTLS; + Value *RetvalTLS; /// Thread-local origin storage for function return value. - GlobalVariable *RetvalOriginTLS; + Value *RetvalOriginTLS; /// Thread-local shadow storage for in-register va_arg function /// parameters (x86_64-specific). - GlobalVariable *VAArgTLS; + Value *VAArgTLS; + + /// Thread-local shadow storage for in-register va_arg function + /// parameters (x86_64-specific, KMSAN only). + Value *VAArgOriginTLS; /// Thread-local shadow storage for va_arg overflow area /// (x86_64-specific). - GlobalVariable *VAArgOverflowSizeTLS; + Value *VAArgOverflowSizeTLS; /// Thread-local space used to pass origin value to the UMR reporting /// function. - GlobalVariable *OriginTLS; + Value *OriginTLS; /// Are the instrumentation callbacks set up? bool CallbacksInitialized = false; @@ -480,6 +538,21 @@ /// MSan runtime replacements for memmove, memcpy and memset. Value *MemmoveFn, *MemcpyFn, *MemsetFn; + /// KMSAN callback for task-local function argument shadow. + Value *MsanGetContextStateFn; + + /// Functions for poisoning/unpoisoning local variables + Value *MsanPoisonAllocaFn, *MsanUnpoisonAllocaFn; + + /// Each of the MsanMetadataPtrXxx functions returns a pair of shadow/origin + /// pointers. + Value *MsanMetadataPtrForLoadN, *MsanMetadataPtrForStoreN; + Value *MsanMetadataPtrForLoad_1_8[4]; + Value *MsanMetadataPtrForStore_1_8[4]; + + /// Helper to choose between different MsanMetadataPtrXxx(). + Value *getKmsanShadowOriginAccessFn(bool isStore, int size); + /// Memory map parameters used in application-to-shadow calculation. const MemoryMapParams *MapParams; @@ -510,8 +583,9 @@ MemorySanitizer, "msan", "MemorySanitizer: detects uninitialized reads.", false, false) -FunctionPass *llvm::createMemorySanitizerPass(int TrackOrigins, bool Recover) { - return new MemorySanitizer(TrackOrigins, Recover); +FunctionPass *llvm::createMemorySanitizerPass(int TrackOrigins, bool Recover, + bool CompileKernel) { + return new MemorySanitizer(TrackOrigins, Recover, CompileKernel); } /// Create a non-const global initialized with the given string. @@ -526,6 +600,68 @@ GlobalValue::PrivateLinkage, StrConst, ""); } +/// Create KMSAN API callbacks. +void MemorySanitizer::createKernelApi(Module &M) { + IRBuilder<> IRB(*C); + + // These will be initialized in insertKmsanPrologue(). + RetvalTLS = nullptr; + RetvalOriginTLS = nullptr; + ParamTLS = nullptr; + ParamOriginTLS = nullptr; + VAArgTLS = nullptr; + VAArgOriginTLS = nullptr; + VAArgOverflowSizeTLS = nullptr; + // OriginTLS is unused in the kernel. + OriginTLS = nullptr; + + // __msan_warning() in the kernel takes an origin. + WarningFn = M.getOrInsertFunction("__msan_warning", IRB.getVoidTy(), + IRB.getInt32Ty()); + // Requests the per-task context state (kmsan_context_state*) from the + // runtime library. + MsanGetContextStateFn = M.getOrInsertFunction( + "__msan_get_context_state", + PointerType::get( + StructType::get(ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), + ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), + ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), + ArrayType::get(IRB.getInt64Ty(), + kParamTLSSize / 8), /* va_arg_origin */ + IRB.getInt64Ty(), + ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy, + OriginTy), + 0)); + + Type *RetTy = StructType::get(PointerType::get(IRB.getInt8Ty(), 0), + PointerType::get(IRB.getInt32Ty(), 0)); + + for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) { + std::string name_load = + "__msan_metadata_ptr_for_load_" + std::to_string(size); + std::string name_store = + "__msan_metadata_ptr_for_store_" + std::to_string(size); + MsanMetadataPtrForLoad_1_8[ind] = M.getOrInsertFunction( + name_load, RetTy, PointerType::get(IRB.getInt8Ty(), 0)); + MsanMetadataPtrForStore_1_8[ind] = M.getOrInsertFunction( + name_store, RetTy, PointerType::get(IRB.getInt8Ty(), 0)); + } + + MsanMetadataPtrForLoadN = M.getOrInsertFunction( + "__msan_metadata_ptr_for_load_n", RetTy, + PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty()); + MsanMetadataPtrForStoreN = M.getOrInsertFunction( + "__msan_metadata_ptr_for_store_n", RetTy, + PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty()); + + // Functions for poisoning and unpoisoning memory. + MsanPoisonAllocaFn = + M.getOrInsertFunction("__msan_poison_alloca", IRB.getVoidTy(), + IRB.getInt8PtrTy(), IntptrTy, IRB.getInt8PtrTy()); + MsanUnpoisonAllocaFn = M.getOrInsertFunction( + "__msan_unpoison_alloca", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy); +} + /// Insert declarations for userspace-specific functions and globals. void MemorySanitizer::createUserspaceApi(Module &M) { IRBuilder<> IRB(*C); @@ -615,10 +751,31 @@ StringRef(""), StringRef(""), /*hasSideEffects=*/true); - createUserspaceApi(M); + if (CompileKernel) { + createKernelApi(M); + } else { + createUserspaceApi(M); + } CallbacksInitialized = true; } +Value *MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore, int size) { + Value **Fns = + isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8; + switch (size) { + case 1: + return Fns[0]; + case 2: + return Fns[1]; + case 4: + return Fns[2]; + case 8: + return Fns[3]; + default: + return nullptr; + } +} + /// Module-level initialization. /// /// inserts a call to __msan_init to the module's constructor list. @@ -695,27 +852,28 @@ ColdCallWeights = MDBuilder(*C).createBranchWeights(1, 1000); OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000); - std::tie(MsanCtorFunction, std::ignore) = - createSanitizerCtorAndInitFunctions(M, kMsanModuleCtorName, kMsanInitName, - /*InitArgTypes=*/{}, - /*InitArgs=*/{}); - if (ClWithComdat) { - Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName); - MsanCtorFunction->setComdat(MsanCtorComdat); - appendToGlobalCtors(M, MsanCtorFunction, 0, MsanCtorFunction); - } else { - appendToGlobalCtors(M, MsanCtorFunction, 0); - } - - - if (TrackOrigins) - new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage, - IRB.getInt32(TrackOrigins), "__msan_track_origins"); + if (!CompileKernel) { + std::tie(MsanCtorFunction, std::ignore) = + createSanitizerCtorAndInitFunctions(M, kMsanModuleCtorName, + kMsanInitName, + /*InitArgTypes=*/{}, + /*InitArgs=*/{}); + if (ClWithComdat) { + Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName); + MsanCtorFunction->setComdat(MsanCtorComdat); + appendToGlobalCtors(M, MsanCtorFunction, 0, MsanCtorFunction); + } else { + appendToGlobalCtors(M, MsanCtorFunction, 0); + } - if (Recover) - new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage, - IRB.getInt32(Recover), "__msan_keep_going"); + if (TrackOrigins) + new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage, + IRB.getInt32(TrackOrigins), "__msan_track_origins"); + if (Recover) + new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage, + IRB.getInt32(Recover), "__msan_keep_going"); + } return true; } @@ -809,7 +967,10 @@ TLI = &MS.getAnalysis().getTLI(); MS.initializeCallbacks(*F.getParent()); - ActualFnStart = &F.getEntryBlock(); + if (MS.CompileKernel) + ActualFnStart = insertKmsanPrologue(F); + else + ActualFnStart = &F.getEntryBlock(); LLVM_DEBUG(if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '" @@ -883,7 +1044,7 @@ unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType()); unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits); - if (AsCall && SizeIndex < kNumberOfAccessSizes) { + if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) { Value *Fn = MS.MaybeStoreOriginFn[SizeIndex]; Value *ConvertedShadow2 = IRB.CreateZExt( ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex))); @@ -932,10 +1093,14 @@ void insertWarningFn(IRBuilder<> &IRB, Value *Origin) { if (!Origin) Origin = (Value *)IRB.getInt32(0); - if (MS.TrackOrigins) { - IRB.CreateStore(Origin, MS.OriginTLS); + if (MS.CompileKernel) { + IRB.CreateCall(MS.WarningFn, Origin); + } else { + if (MS.TrackOrigins) { + IRB.CreateStore(Origin, MS.OriginTLS); + } + IRB.CreateCall(MS.WarningFn, {}); } - IRB.CreateCall(MS.WarningFn, {}); IRB.CreateCall(MS.EmptyAsm, {}); // FIXME: Insert UnreachableInst if !MS.Recover? // This may invalidate some of the following checks and needs to be done @@ -961,7 +1126,7 @@ unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType()); unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits); - if (AsCall && SizeIndex < kNumberOfAccessSizes) { + if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) { Value *Fn = MS.MaybeWarningFn[SizeIndex]; Value *ConvertedShadow2 = IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex))); @@ -991,6 +1156,29 @@ LLVM_DEBUG(dbgs() << "DONE:\n" << F); } + BasicBlock *insertKmsanPrologue(Function &F) { + BasicBlock *ret = + SplitBlock(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHI()); + IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); + Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {}); + Constant *Zero = IRB.getInt32(0); + MS.ParamTLS = + IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(0)}, "param_shadow"); + MS.RetvalTLS = + IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(1)}, "retval_shadow"); + MS.VAArgTLS = + IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(2)}, "va_arg_shadow"); + MS.VAArgOriginTLS = + IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(3)}, "va_arg_origin"); + MS.VAArgOverflowSizeTLS = IRB.CreateGEP( + ContextState, {Zero, IRB.getInt32(4)}, "va_arg_overflow_size"); + MS.ParamOriginTLS = + IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(5)}, "param_origin"); + MS.RetvalOriginTLS = + IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(6)}, "retval_origin"); + return ret; + } + /// Add MemorySanitizer instrumentation to a function. bool runOnFunction() { // In the presence of unreachable blocks, we may see Phi nodes with @@ -1139,12 +1327,40 @@ return std::make_pair(ShadowPtr, OriginPtr); } + std::pair + getShadowOriginPtrKernel(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy, + unsigned Alignment, bool isStore) { + Value *ShadowOriginPtrs; + const DataLayout &DL = F.getParent()->getDataLayout(); + int Size = DL.getTypeStoreSize(ShadowTy); + + Value *Getter = MS.getKmsanShadowOriginAccessFn(isStore, Size); + Value *AddrCast = + IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0)); + if (Getter) { + ShadowOriginPtrs = IRB.CreateCall(Getter, AddrCast); + } else { + Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size); + ShadowOriginPtrs = IRB.CreateCall(isStore ? MS.MsanMetadataPtrForStoreN + : MS.MsanMetadataPtrForLoadN, + {AddrCast, SizeVal}); + } + Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0); + ShadowPtr = IRB.CreatePointerCast(ShadowPtr, PointerType::get(ShadowTy, 0)); + Value *OriginPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 1); + + return std::make_pair(ShadowPtr, OriginPtr); + } + std::pair getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy, unsigned Alignment, bool isStore) { - std::pair ret = - getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment); + std::pair ret; + if (MS.CompileKernel) + ret = getShadowOriginPtrKernel(Addr, IRB, ShadowTy, Alignment, isStore); + else + ret = getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment); return ret; } @@ -1163,7 +1379,8 @@ /// Compute the origin address for a given function argument. Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB, int ArgOffset) { - if (!MS.TrackOrigins) return nullptr; + if (!MS.TrackOrigins) + return nullptr; Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy); if (ArgOffset) Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); @@ -1303,6 +1520,7 @@ getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign, /*isStore*/ true) .first; + // TODO(glider): need to copy origins. if (Overflow) { // ParamTLS overflow. EntryIRB.CreateMemSet( @@ -2921,12 +3139,14 @@ if (ArgOffset + Size > kParamTLSSize) break; unsigned ParamAlignment = CS.getParamAlignment(i); unsigned Alignment = std::min(ParamAlignment, kShadowTLSAlignment); - Value *AShadowPtr = getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), - Alignment, /*isStore*/ false) - .first; + Value *AShadowPtr = + getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), Alignment, + /*isStore*/ false) + .first; Store = IRB.CreateMemCpy(ArgShadowBase, Alignment, AShadowPtr, Alignment, Size); + // TODO(glider): need to copy origins. } else { Size = DL.getTypeAllocSize(A->getType()); if (ArgOffset + Size > kParamTLSSize) break; @@ -3033,40 +3253,34 @@ "_msphi_o")); } - void visitAllocaInst(AllocaInst &I) { - setShadow(&I, getCleanShadow(&I)); - setOrigin(&I, getCleanOrigin()); - IRBuilder<> IRB(I.getNextNode()); - const DataLayout &DL = F.getParent()->getDataLayout(); - uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType()); - Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize); - if (I.isArrayAllocation()) - Len = IRB.CreateMul(Len, I.getArraySize()); + Value *getLocalVarDescription(AllocaInst &I) { + SmallString<2048> StackDescriptionStorage; + raw_svector_ostream StackDescription(StackDescriptionStorage); + // We create a string with a description of the stack allocation and + // pass it into __msan_set_alloca_origin. + // It will be printed by the run-time if stack-originated UMR is found. + // The first 4 bytes of the string are set to '----' and will be replaced + // by __msan_va_arg_overflow_size_tls at the first call. + StackDescription << "----" << I.getName() << "@" << F.getName(); + return createPrivateNonConstGlobalForString(*F.getParent(), + StackDescription.str()); + } + + void instrumentAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) { if (PoisonStack && ClPoisonStackWithCall) { IRB.CreateCall(MS.MsanPoisonStackFn, {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len}); } else { - Value *ShadowBase = getShadowOriginPtr(&I, IRB, IRB.getInt8Ty(), - I.getAlignment(), /*isStore*/ true) - .first; + Value *ShadowBase, *OriginBase; + std::tie(ShadowBase, OriginBase) = + getShadowOriginPtr(&I, IRB, IRB.getInt8Ty(), 1, /*isStore*/ true); Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0); IRB.CreateMemSet(ShadowBase, PoisonValue, Len, I.getAlignment()); } if (PoisonStack && MS.TrackOrigins) { - SmallString<2048> StackDescriptionStorage; - raw_svector_ostream StackDescription(StackDescriptionStorage); - // We create a string with a description of the stack allocation and - // pass it into __msan_set_alloca_origin. - // It will be printed by the run-time if stack-originated UMR is found. - // The first 4 bytes of the string are set to '----' and will be replaced - // by __msan_va_arg_overflow_size_tls at the first call. - StackDescription << "----" << I.getName() << "@" << F.getName(); - Value *Descr = - createPrivateNonConstGlobalForString(*F.getParent(), - StackDescription.str()); - + Value *Descr = getLocalVarDescription(I); IRB.CreateCall(MS.MsanSetAllocaOrigin4Fn, {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len, IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()), @@ -3074,6 +3288,34 @@ } } + void instrumentAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) { + Value *Descr = getLocalVarDescription(I); + if (PoisonStack) { + IRB.CreateCall(MS.MsanPoisonAllocaFn, + {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len, + IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy())}); + } else { + IRB.CreateCall(MS.MsanUnpoisonAllocaFn, + {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len}); + } + } + + void visitAllocaInst(AllocaInst &I) { + setShadow(&I, getCleanShadow(&I)); + setOrigin(&I, getCleanOrigin()); + IRBuilder<> IRB(I.getNextNode()); + const DataLayout &DL = F.getParent()->getDataLayout(); + uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType()); + Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize); + if (I.isArrayAllocation()) + Len = IRB.CreateMul(Len, I.getArraySize()); + + if (MS.CompileKernel) + instrumentAllocaKmsan(I, IRB, Len); + else + instrumentAllocaUserspace(I, IRB, Len); + } + void visitSelectInst(SelectInst& I) { IRBuilder<> IRB(&I); // a = select b, c, d @@ -3258,6 +3500,7 @@ MemorySanitizer &MS; MemorySanitizerVisitor &MSV; Value *VAArgTLSCopy = nullptr; + Value *VAArgTLSOriginCopy = nullptr; Value *VAArgOverflowSize = nullptr; SmallVector VAStartInstrumentationList; @@ -3320,6 +3563,10 @@ uint64_t ArgSize = DL.getTypeAllocSize(RealTy); Value *ShadowBase = getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset); + Value *OriginBase = nullptr; + if (MS.CompileKernel) { + OriginBase = getOriginPtrForVAArgument(RealTy, IRB, OverflowOffset); + } OverflowOffset += alignTo(ArgSize, 8); Value *ShadowPtr, *OriginPtr; std::tie(ShadowPtr, OriginPtr) = @@ -3328,20 +3575,30 @@ IRB.CreateMemCpy(ShadowBase, kShadowTLSAlignment, ShadowPtr, kShadowTLSAlignment, ArgSize); + if (MS.CompileKernel) { + IRB.CreateMemCpy(OriginBase, kShadowTLSAlignment, OriginPtr, + kShadowTLSAlignment, ArgSize); + } } else { ArgKind AK = classifyArgument(A); if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset) AK = AK_Memory; if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset) AK = AK_Memory; - Value *ShadowBase; + Value *ShadowBase, *OriginBase; switch (AK) { case AK_GeneralPurpose: ShadowBase = getShadowPtrForVAArgument(A->getType(), IRB, GpOffset); + if (MS.CompileKernel) + OriginBase = + getOriginPtrForVAArgument(A->getType(), IRB, GpOffset); GpOffset += 8; break; case AK_FloatingPoint: ShadowBase = getShadowPtrForVAArgument(A->getType(), IRB, FpOffset); + if (MS.CompileKernel) + OriginBase = + getOriginPtrForVAArgument(A->getType(), IRB, FpOffset); FpOffset += 16; break; case AK_Memory: @@ -3350,14 +3607,24 @@ uint64_t ArgSize = DL.getTypeAllocSize(A->getType()); ShadowBase = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset); + if (MS.CompileKernel) + OriginBase = + getOriginPtrForVAArgument(A->getType(), IRB, OverflowOffset); OverflowOffset += alignTo(ArgSize, 8); } // Take fixed arguments into account for GpOffset and FpOffset, // but don't actually store shadows for them. - if (IsFixed) + // TODO(glider): don't call get*PtrForVAArgument() for them. + if (IsFixed) { continue; - IRB.CreateAlignedStore(MSV.getShadow(A), ShadowBase, - kShadowTLSAlignment); + } + Value *Shadow = MSV.getShadow(A); + IRB.CreateAlignedStore(Shadow, ShadowBase, kShadowTLSAlignment); + if (MS.CompileKernel) { + unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType()); + MSV.paintOrigin(IRB, MSV.getOrigin(A), OriginBase, StoreSize, + std::max(kShadowTLSAlignment, kMinOriginAlignment)); + } } } Constant *OverflowSize = @@ -3370,8 +3637,15 @@ int ArgOffset) { Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy); Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); + return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0)); + } + + /// Compute the shadow address for a given va_arg. + Value *getOriginPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, int ArgOffset) { + Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy); + Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0), - "_msarg"); + "_msarg_va_o"); } void unpoisonVAListTagForInst(IntrinsicInst &I) { @@ -3416,6 +3690,10 @@ VAArgOverflowSize); VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize); + if (MS.CompileKernel) { + VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); + IRB.CreateMemCpy(VAArgTLSOriginCopy, 8, MS.VAArgOriginTLS, 8, CopySize); + } } // Instrument va_start. @@ -3437,6 +3715,10 @@ Alignment, /*isStore*/ true); IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment, AMD64FpEndOffset); + if (MS.CompileKernel) { + IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy, + Alignment, AMD64FpEndOffset); + } Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr( IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), ConstantInt::get(MS.IntptrTy, 8)), @@ -3450,6 +3732,13 @@ AMD64FpEndOffset); IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment, VAArgOverflowSize); + + if (MS.CompileKernel) { + SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy, + AMD64FpEndOffset); + IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment, + VAArgOverflowSize); + } } } }; Index: test/Instrumentation/MemorySanitizer/alloca.ll =================================================================== --- test/Instrumentation/MemorySanitizer/alloca.ll +++ test/Instrumentation/MemorySanitizer/alloca.ll @@ -2,6 +2,7 @@ ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-poison-stack-with-call=1 -S | FileCheck %s --check-prefixes=CHECK,CALL ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=1 -S | FileCheck %s --check-prefixes=CHECK,ORIGIN ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=2 -S | FileCheck %s --check-prefixes=CHECK,ORIGIN +; RUN: opt < %s -msan -msan-kernel=1 -S | FileCheck %s --check-prefixes=CHECK,KMSAN target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" @@ -16,6 +17,7 @@ ; INLINE: call void @llvm.memset.p0i8.i64(i8* align 4 {{.*}}, i8 -1, i64 4, i1 false) ; CALL: call void @__msan_poison_stack(i8* {{.*}}, i64 4) ; ORIGIN: call void @__msan_set_alloca_origin4(i8* {{.*}}, i64 4, +; KMSAN: call void @__msan_poison_alloca(i8* {{.*}}, i64 4, ; CHECK: ret void @@ -31,6 +33,7 @@ ; INLINE: call void @llvm.memset.p0i8.i64(i8* align 4 {{.*}}, i8 -1, i64 4, i1 false) ; CALL: call void @__msan_poison_stack(i8* {{.*}}, i64 4) ; ORIGIN: call void @__msan_set_alloca_origin4(i8* {{.*}}, i64 4, +; KMSAN: call void @__msan_poison_alloca(i8* {{.*}}, i64 4, ; CHECK: ret void define void @array() sanitize_memory { @@ -43,6 +46,7 @@ ; INLINE: call void @llvm.memset.p0i8.i64(i8* align 4 {{.*}}, i8 -1, i64 20, i1 false) ; CALL: call void @__msan_poison_stack(i8* {{.*}}, i64 20) ; ORIGIN: call void @__msan_set_alloca_origin4(i8* {{.*}}, i64 20, +; KMSAN: call void @__msan_poison_alloca(i8* {{.*}}, i64 20, ; CHECK: ret void define void @array_non_const(i64 %cnt) sanitize_memory { @@ -56,4 +60,20 @@ ; INLINE: call void @llvm.memset.p0i8.i64(i8* align 4 {{.*}}, i8 -1, i64 %[[A]], i1 false) ; CALL: call void @__msan_poison_stack(i8* {{.*}}, i64 %[[A]]) ; ORIGIN: call void @__msan_set_alloca_origin4(i8* {{.*}}, i64 %[[A]], +; KMSAN: call void @__msan_poison_alloca(i8* {{.*}}, i64 %[[A]], ; CHECK: ret void + +; Check that the local is unpoisoned in the absence of sanitize_memory +define void @unpoison_local() { +entry: + %x = alloca i32, i64 5, align 4 + ret void +} + +; CHECK-LABEL: define void @unpoison_local( +; INLINE: call void @llvm.memset.p0i8.i64(i8* align 4 {{.*}}, i8 0, i64 20, i1 false) +; CALL: call void @llvm.memset.p0i8.i64(i8* align 4 {{.*}}, i8 0, i64 20, i1 false) +; ORIGIN-NOT: call void @__msan_set_alloca_origin4(i8* {{.*}}, i64 20, +; KMSAN: call void @__msan_unpoison_alloca(i8* {{.*}}, i64 20) +; CHECK: ret void + Index: test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll =================================================================== --- test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll +++ test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll @@ -0,0 +1,404 @@ +; KMSAN instrumentation tests +; RUN: opt < %s -msan -msan-kernel=1 -S | FileCheck %s -check-prefixes=CHECK + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Check the instrumentation prologue. +define void @Empty() nounwind uwtable sanitize_memory { +entry: + ret void +} + +; CHECK-LABEL: @Empty +; CHECK: entry: +; CHECK: @__msan_get_context_state() +; %param_shadow: +; CHECK: getelementptr {{.*}} i32 0, i32 0 +; %retval_shadow: +; CHECK: getelementptr {{.*}} i32 0, i32 1 +; %va_arg_shadow: +; CHECK: getelementptr {{.*}} i32 0, i32 2 +; %va_arg_origin: +; CHECK: getelementptr {{.*}} i32 0, i32 3 +; %va_arg_overflow_size: +; CHECK: getelementptr {{.*}} i32 0, i32 4 +; %param_origin: +; CHECK: getelementptr {{.*}} i32 0, i32 5 +; %retval_origin: +; CHECK: getelementptr {{.*}} i32 0, i32 6 +; CHECK: entry.split: + +; Check instrumentation of stores + +define void @Store1(i8* nocapture %p, i8 %x) nounwind uwtable sanitize_memory { +entry: + store i8 %x, i8* %p + ret void +} + +; CHECK-LABEL: @Store1 +; CHECK-LABEL: entry: +; CHECK: @__msan_get_context_state() +; CHECK: [[PARAM_SHADOW:%[a-z0-9_]+]] = getelementptr {{.*}} i32 0, i32 0 +; CHECK-LABEL: entry.split: +; CHECK: [[BASE2:%[0-9]+]] = ptrtoint {{.*}} [[PARAM_SHADOW]] +; CHECK: [[BASE:%[0-9]+]] = ptrtoint {{.*}} [[PARAM_SHADOW]] +; CHECK: [[SHADOW:%[a-z0-9_]+]] = inttoptr {{.*}} [[BASE]] +; Load the shadow of %p and check it +; CHECK: load i64, i64* [[SHADOW]] +; CHECK: icmp +; CHECK: br i1 +; CHECK-LABEL: