Index: include/llvm/Transforms/Instrumentation.h =================================================================== --- include/llvm/Transforms/Instrumentation.h +++ include/llvm/Transforms/Instrumentation.h @@ -131,7 +131,8 @@ // Insert MemorySanitizer instrumentation (detection of uninitialized reads) FunctionPass *createMemorySanitizerPass(int TrackOrigins = 0, - bool Recover = false); + bool Recover = false, + bool EnableKmsan = false); FunctionPass *createHWAddressSanitizerPass(bool CompileKernel = false, bool Recover = false); Index: lib/Transforms/Instrumentation/MemorySanitizer.cpp =================================================================== --- lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -89,7 +89,35 @@ /// implementation ignores the load aspect of CAS/RMW, always returning a clean /// value. It implements the store part as a simple atomic store by storing a /// clean shadow. -// +/// +/// KernelMemorySanitizer (KMSAN) implementation. +/// +/// The major differences between KMSAN and MSan instrumentation are: +/// - KMSAN always tracks the origins and implies msan-keep-going=true; +/// - KMSAN allocates shadow and origin memory for each page separately, so +/// there are no explicit accesses to shadow and origin in the +/// instrumentation. +/// Shadow and origin values for a particular X-byte memory location +/// (X=1,2,4,8) are accessed through pointers obtained via the +/// __msan_metadata_ptr_for_load_X(ptr) +/// __msan_metadata_ptr_for_store_X(ptr) +/// functions. The corresponding functions check that the X-byte accesses +/// are possible and returns the pointers to shadow and origin memory. +/// Arbitrary sized accesses are handled with: +/// __msan_metadata_ptr_for_load_n(ptr, size) +/// __msan_metadata_ptr_for_store_n(ptr, size); +/// - TLS variables are stored in a single per-task struct. A call to a +/// function __msan_get_context_state() returning a pointer to that struct +/// is inserted into every instrumented function before the entry block; +/// - __msan_warning() takes a 32-bit origin parameter; +/// - the pass doesn't declare any global variables or add global constructors +/// to the translation unit. +/// +/// Also, KMSAN currently ignores uninitialized memory passed into inline asm +/// calls, making sure we're on the safe side wrt. possible false positives. +/// +/// KernelMemorySanitizer only supports X86_64 at the moment. +/// //===----------------------------------------------------------------------===// #include "llvm/ADT/APInt.h" @@ -233,6 +261,11 @@ "inline checks (-1 means never use callbacks)."), cl::Hidden, cl::init(3500)); +static cl::opt + ClEnableKmsan("msan-kernel", + cl::desc("Enable KernelMemorySanitizer instrumentation"), + cl::Hidden, cl::init(false)); + // This is an experiment to enable handling of cases where shadow is a non-zero // compile-time constant. For some unexplainable reason they were silently // ignored in the instrumentation. @@ -400,11 +433,19 @@ // Pass identification, replacement for typeid. static char ID; - MemorySanitizer(int TrackOrigins = 0, bool Recover = false) - : FunctionPass(ID), - TrackOrigins(std::max(TrackOrigins, (int)ClTrackOrigins)), - Recover(Recover || ClKeepGoing) {} - + MemorySanitizer(int TrackOrigins = 0, bool Recover = false, + bool EnableKmsan = false) + : FunctionPass(ID) { + this->CompileKernel = + ClEnableKmsan.getNumOccurrences() > 0 ? ClEnableKmsan : EnableKmsan; + if (ClTrackOrigins.getNumOccurrences() > 0) + this->TrackOrigins = ClTrackOrigins; + else + this->TrackOrigins = this->CompileKernel ? 2 : TrackOrigins; + this->Recover = ClKeepGoing.getNumOccurrences() > 0 + ? ClKeepGoing + : (this->CompileKernel | Recover); + } StringRef getPassName() const override { return "MemorySanitizer"; } void getAnalysisUsage(AnalysisUsage &AU) const override { @@ -422,8 +463,12 @@ friend struct VarArgPowerPC64Helper; void initializeCallbacks(Module &M); + void createKernelApi(Module &M); void createUserspaceApi(Module &M); + /// True if we're compiling the Linux kernel. + bool CompileKernel; + /// Track origins (allocation points) of uninitialized values. int TrackOrigins; bool Recover; @@ -432,29 +477,39 @@ Type *IntptrTy; Type *OriginTy; + // XxxTLS variables represent the per-thread state in MSan and per-task state + // in KMSAN. + // For the userspace these point to thread-local globals. In the kernel land + // they point to the members of a per-task struct obtained via a call to + // __msan_get_context_state(). + /// Thread-local shadow storage for function parameters. - GlobalVariable *ParamTLS; + Value *ParamTLS; /// Thread-local origin storage for function parameters. - GlobalVariable *ParamOriginTLS; + Value *ParamOriginTLS; /// Thread-local shadow storage for function return value. - GlobalVariable *RetvalTLS; + Value *RetvalTLS; /// Thread-local origin storage for function return value. - GlobalVariable *RetvalOriginTLS; + Value *RetvalOriginTLS; /// Thread-local shadow storage for in-register va_arg function /// parameters (x86_64-specific). - GlobalVariable *VAArgTLS; + Value *VAArgTLS; + + /// Thread-local shadow storage for in-register va_arg function + /// parameters (x86_64-specific, KMSAN only). + Value *VAArgOriginTLS; /// Thread-local shadow storage for va_arg overflow area /// (x86_64-specific). - GlobalVariable *VAArgOverflowSizeTLS; + Value *VAArgOverflowSizeTLS; /// Thread-local space used to pass origin value to the UMR reporting /// function. - GlobalVariable *OriginTLS; + Value *OriginTLS; /// Are the instrumentation callbacks set up? bool CallbacksInitialized = false; @@ -480,6 +535,21 @@ /// MSan runtime replacements for memmove, memcpy and memset. Value *MemmoveFn, *MemcpyFn, *MemsetFn; + /// KMSAN callback for task-local function argument shadow. + Value *MsanGetContextStateFn; + + /// Functions for poisoning/unpoisoning local variables + Value *MsanPoisonAllocaFn, *MsanUnpoisonAllocaFn; + + /// Each of the MsanMetadataPtrXxx functions returns a pair of shadow/origin + /// pointers. + Value *MsanMetadataPtrForLoadN, *MsanMetadataPtrForStoreN; + Value *MsanMetadataPtrForLoad_1_8[4]; + Value *MsanMetadataPtrForStore_1_8[4]; + + /// Helper to choose between different MsanMetadataPtrXxx(). + Value *getKmsanShadowOriginAccessFn(bool isStore, int size); + /// Memory map parameters used in application-to-shadow calculation. const MemoryMapParams *MapParams; @@ -510,8 +580,9 @@ MemorySanitizer, "msan", "MemorySanitizer: detects uninitialized reads.", false, false) -FunctionPass *llvm::createMemorySanitizerPass(int TrackOrigins, bool Recover) { - return new MemorySanitizer(TrackOrigins, Recover); +FunctionPass *llvm::createMemorySanitizerPass(int TrackOrigins, bool Recover, + bool CompileKernel) { + return new MemorySanitizer(TrackOrigins, Recover, CompileKernel); } /// Create a non-const global initialized with the given string. @@ -526,6 +597,68 @@ GlobalValue::PrivateLinkage, StrConst, ""); } +/// Create KMSAN API callbacks. +void MemorySanitizer::createKernelApi(Module &M) { + IRBuilder<> IRB(*C); + + // These will be initialized in insertKmsanPrologue(). + RetvalTLS = nullptr; + RetvalOriginTLS = nullptr; + ParamTLS = nullptr; + ParamOriginTLS = nullptr; + VAArgTLS = nullptr; + VAArgOriginTLS = nullptr; + VAArgOverflowSizeTLS = nullptr; + // OriginTLS is unused in the kernel. + OriginTLS = nullptr; + + // __msan_warning() in the kernel takes an origin. + WarningFn = M.getOrInsertFunction("__msan_warning", IRB.getVoidTy(), + IRB.getInt32Ty()); + // Requests the per-task context state (kmsan_context_state*) from the + // runtime library. + MsanGetContextStateFn = M.getOrInsertFunction( + "__msan_get_context_state", + PointerType::get( + StructType::get(ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), + ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), + ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), + ArrayType::get(IRB.getInt64Ty(), + kParamTLSSize / 8), /* va_arg_origin */ + IRB.getInt64Ty(), + ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy, + OriginTy), + 0)); + + Type *RetTy = StructType::get(PointerType::get(IRB.getInt8Ty(), 0), + PointerType::get(IRB.getInt32Ty(), 0)); + + for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) { + std::string name_load = + "__msan_metadata_ptr_for_load_" + std::to_string(size); + std::string name_store = + "__msan_metadata_ptr_for_store_" + std::to_string(size); + MsanMetadataPtrForLoad_1_8[ind] = M.getOrInsertFunction( + name_load, RetTy, PointerType::get(IRB.getInt8Ty(), 0)); + MsanMetadataPtrForStore_1_8[ind] = M.getOrInsertFunction( + name_store, RetTy, PointerType::get(IRB.getInt8Ty(), 0)); + } + + MsanMetadataPtrForLoadN = M.getOrInsertFunction( + "__msan_metadata_ptr_for_load_n", RetTy, + PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty()); + MsanMetadataPtrForStoreN = M.getOrInsertFunction( + "__msan_metadata_ptr_for_store_n", RetTy, + PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty()); + + // Functions for poisoning and unpoisoning memory. + MsanPoisonAllocaFn = M.getOrInsertFunction( + "__msan_poison_alloca", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy, + IRB.getInt8PtrTy(), IntptrTy); + MsanUnpoisonAllocaFn = M.getOrInsertFunction( + "__msan_unpoison_alloca", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy); +} + /// Insert declarations for userspace-specific functions and globals. void MemorySanitizer::createUserspaceApi(Module &M) { IRBuilder<> IRB(*C); @@ -615,10 +748,31 @@ StringRef(""), StringRef(""), /*hasSideEffects=*/true); - createUserspaceApi(M); + if (!CompileKernel) { + createUserspaceApi(M); + } else { + createKernelApi(M); + } CallbacksInitialized = true; } +Value *MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore, int size) { + Value **Fns = + isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8; + switch (size) { + case 1: + return Fns[0]; + case 2: + return Fns[1]; + case 4: + return Fns[2]; + case 8: + return Fns[3]; + default: + return nullptr; + } +} + /// Module-level initialization. /// /// inserts a call to __msan_init to the module's constructor list. @@ -695,27 +849,28 @@ ColdCallWeights = MDBuilder(*C).createBranchWeights(1, 1000); OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000); - std::tie(MsanCtorFunction, std::ignore) = - createSanitizerCtorAndInitFunctions(M, kMsanModuleCtorName, kMsanInitName, - /*InitArgTypes=*/{}, - /*InitArgs=*/{}); - if (ClWithComdat) { - Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName); - MsanCtorFunction->setComdat(MsanCtorComdat); - appendToGlobalCtors(M, MsanCtorFunction, 0, MsanCtorFunction); - } else { - appendToGlobalCtors(M, MsanCtorFunction, 0); - } - - - if (TrackOrigins) - new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage, - IRB.getInt32(TrackOrigins), "__msan_track_origins"); + if (!CompileKernel) { + std::tie(MsanCtorFunction, std::ignore) = + createSanitizerCtorAndInitFunctions(M, kMsanModuleCtorName, + kMsanInitName, + /*InitArgTypes=*/{}, + /*InitArgs=*/{}); + if (ClWithComdat) { + Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName); + MsanCtorFunction->setComdat(MsanCtorComdat); + appendToGlobalCtors(M, MsanCtorFunction, 0, MsanCtorFunction); + } else { + appendToGlobalCtors(M, MsanCtorFunction, 0); + } - if (Recover) - new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage, - IRB.getInt32(Recover), "__msan_keep_going"); + if (TrackOrigins) + new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage, + IRB.getInt32(TrackOrigins), "__msan_track_origins"); + if (Recover) + new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage, + IRB.getInt32(Recover), "__msan_keep_going"); + } return true; } @@ -809,7 +964,10 @@ TLI = &MS.getAnalysis().getTLI(); MS.initializeCallbacks(*F.getParent()); - ActualFnStart = &F.getEntryBlock(); + if (MS.CompileKernel) + ActualFnStart = insertKmsanPrologue(F); + else + ActualFnStart = &F.getEntryBlock(); LLVM_DEBUG(if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '" @@ -883,7 +1041,7 @@ unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType()); unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits); - if (AsCall && SizeIndex < kNumberOfAccessSizes) { + if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) { Value *Fn = MS.MaybeStoreOriginFn[SizeIndex]; Value *ConvertedShadow2 = IRB.CreateZExt( ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex))); @@ -918,9 +1076,6 @@ StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment); LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n"); - if (ClCheckAccessAddress) - insertShadowCheck(Addr, NewSI); - if (SI->isAtomic()) SI->setOrdering(addReleaseOrdering(SI->getOrdering())); @@ -934,10 +1089,14 @@ void insertWarningFn(IRBuilder<> &IRB, Value *Origin) { if (!Origin) Origin = (Value *)IRB.getInt32(0); - if (MS.TrackOrigins) { - IRB.CreateStore(Origin, MS.OriginTLS); + if (!MS.CompileKernel) { + if (MS.TrackOrigins) { + IRB.CreateStore(Origin, MS.OriginTLS); + } + IRB.CreateCall(MS.WarningFn, {}); + } else { + IRB.CreateCall(MS.WarningFn, Origin); } - IRB.CreateCall(MS.WarningFn, {}); IRB.CreateCall(MS.EmptyAsm, {}); // FIXME: Insert UnreachableInst if !MS.Recover? // This may invalidate some of the following checks and needs to be done @@ -963,7 +1122,7 @@ unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType()); unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits); - if (AsCall && SizeIndex < kNumberOfAccessSizes) { + if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) { Value *Fn = MS.MaybeWarningFn[SizeIndex]; Value *ConvertedShadow2 = IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex))); @@ -993,6 +1152,29 @@ LLVM_DEBUG(dbgs() << "DONE:\n" << F); } + BasicBlock *insertKmsanPrologue(Function &F) { + BasicBlock *ret = + SplitBlock(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHI()); + IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); + Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {}); + Constant *Zero = IRB.getInt32(0); + MS.ParamTLS = + IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(0)}, "param_shadow"); + MS.RetvalTLS = + IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(1)}, "retval_shadow"); + MS.VAArgTLS = + IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(2)}, "va_arg_shadow"); + MS.VAArgOriginTLS = + IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(3)}, "va_arg_origin"); + MS.VAArgOverflowSizeTLS = IRB.CreateGEP( + ContextState, {Zero, IRB.getInt32(4)}, "va_arg_overflow_size"); + MS.ParamOriginTLS = + IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(5)}, "param_origin"); + MS.RetvalOriginTLS = + IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(6)}, "retval_origin"); + return ret; + } + /// Add MemorySanitizer instrumentation to a function. bool runOnFunction() { // In the presence of unreachable blocks, we may see Phi nodes with @@ -1024,13 +1206,13 @@ InstrumentationList.size() + StoreList.size() > (unsigned)ClInstrumentationWithCallThreshold; + // Insert shadow value checks. + materializeChecks(InstrumentWithCalls); + // Delayed instrumentation of StoreInst. // This may add new checks to be inserted later. materializeStores(InstrumentWithCalls); - // Insert shadow value checks. - materializeChecks(InstrumentWithCalls); - return true; } @@ -1141,12 +1323,43 @@ return std::make_pair(ShadowPtr, OriginPtr); } + std::pair + getShadowOriginPtrKernel(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy, + unsigned Alignment, bool isStore) { + Value *ShadowOriginPtrs; + int BitWidth = VectorOrPrimitiveTypeSizeInBits(ShadowTy); + int Size = BitWidth / 8; + // Make sure Size is at least 1 if the operand is i1. + if (Size * 8 < BitWidth) + Size++; + Value *Getter = MS.getKmsanShadowOriginAccessFn(isStore, Size); + Value *AddrCast = + IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0)); + if (Getter) { + ShadowOriginPtrs = IRB.CreateCall(Getter, AddrCast); + } else { + Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size); + ShadowOriginPtrs = IRB.CreateCall(isStore ? MS.MsanMetadataPtrForStoreN + : MS.MsanMetadataPtrForLoadN, + {AddrCast, SizeVal}); + } + Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0); + ShadowPtr = IRB.CreatePointerCast(ShadowPtr, PointerType::get(ShadowTy, 0)); + Value *OriginPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 1); + + return std::make_pair(ShadowPtr, OriginPtr); + } + std::pair getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy, unsigned Alignment, bool isStore) { - std::pair ret = - getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment); + std::pair ret; + if (!MS.CompileKernel) { + ret = getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment); + } else { + ret = getShadowOriginPtrKernel(Addr, IRB, ShadowTy, Alignment, isStore); + } return ret; } @@ -1165,7 +1378,8 @@ /// Compute the origin address for a given function argument. Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB, int ArgOffset) { - if (!MS.TrackOrigins) return nullptr; + if (!MS.TrackOrigins) + return nullptr; Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy); if (ArgOffset) Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); @@ -1301,10 +1515,11 @@ Type *EltType = A->getType()->getPointerElementType(); ArgAlign = DL.getABITypeAlignment(EltType); } - Value *CpShadowPtr = + Value *CpShadowPtr, *CpOriginPtr; + std::tie(CpShadowPtr, CpOriginPtr) = getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign, - /*isStore*/ true) - .first; + /*isStore*/ true); + // TODO(glider): need to copy origins. if (Overflow) { // ParamTLS overflow. EntryIRB.CreateMemSet( @@ -1490,6 +1705,8 @@ /// Optionally, checks that the store address is fully defined. void visitStoreInst(StoreInst &I) { StoreList.push_back(&I); + if (ClCheckAccessAddress) + insertShadowCheck(I.getPointerOperand(), &I); } void handleCASOrRMW(Instruction &I) { @@ -2921,12 +3138,14 @@ if (ArgOffset + Size > kParamTLSSize) break; unsigned ParamAlignment = CS.getParamAlignment(i); unsigned Alignment = std::min(ParamAlignment, kShadowTLSAlignment); - Value *AShadowPtr = getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), - Alignment, /*isStore*/ false) - .first; + Value *AShadowPtr = + getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), Alignment, + /*isStore*/ false) + .first; Store = IRB.CreateMemCpy(ArgShadowBase, Alignment, AShadowPtr, Alignment, Size); + // TODO(glider): need to copy origins. } else { Size = DL.getTypeAllocSize(A->getType()); if (ArgOffset + Size > kParamTLSSize) break; @@ -3033,40 +3252,34 @@ "_msphi_o")); } - void visitAllocaInst(AllocaInst &I) { - setShadow(&I, getCleanShadow(&I)); - setOrigin(&I, getCleanOrigin()); - IRBuilder<> IRB(I.getNextNode()); - const DataLayout &DL = F.getParent()->getDataLayout(); - uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType()); - Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize); - if (I.isArrayAllocation()) - Len = IRB.CreateMul(Len, I.getArraySize()); + Value *getLocalVarDescription(AllocaInst &I) { + SmallString<2048> StackDescriptionStorage; + raw_svector_ostream StackDescription(StackDescriptionStorage); + // We create a string with a description of the stack allocation and + // pass it into __msan_set_alloca_origin. + // It will be printed by the run-time if stack-originated UMR is found. + // The first 4 bytes of the string are set to '----' and will be replaced + // by __msan_va_arg_overflow_size_tls at the first call. + StackDescription << "----" << I.getName() << "@" << F.getName(); + return createPrivateNonConstGlobalForString(*F.getParent(), + StackDescription.str()); + } + + void instrumentAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) { if (PoisonStack && ClPoisonStackWithCall) { IRB.CreateCall(MS.MsanPoisonStackFn, {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len}); } else { - Value *ShadowBase = getShadowOriginPtr(&I, IRB, IRB.getInt8Ty(), - I.getAlignment(), /*isStore*/ true) - .first; + Value *ShadowBase, *OriginBase; + std::tie(ShadowBase, OriginBase) = + getShadowOriginPtr(&I, IRB, IRB.getInt8Ty(), 1, /*isStore*/ true); Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0); IRB.CreateMemSet(ShadowBase, PoisonValue, Len, I.getAlignment()); } if (PoisonStack && MS.TrackOrigins) { - SmallString<2048> StackDescriptionStorage; - raw_svector_ostream StackDescription(StackDescriptionStorage); - // We create a string with a description of the stack allocation and - // pass it into __msan_set_alloca_origin. - // It will be printed by the run-time if stack-originated UMR is found. - // The first 4 bytes of the string are set to '----' and will be replaced - // by __msan_va_arg_overflow_size_tls at the first call. - StackDescription << "----" << I.getName() << "@" << F.getName(); - Value *Descr = - createPrivateNonConstGlobalForString(*F.getParent(), - StackDescription.str()); - + Value *Descr = getLocalVarDescription(I); IRB.CreateCall(MS.MsanSetAllocaOrigin4Fn, {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len, IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()), @@ -3074,6 +3287,38 @@ } } + void instrumentAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) { + Value *Descr = getLocalVarDescription(I); + Value *Pc = IRB.CreateCall( + Intrinsic::getDeclaration(F.getParent(), Intrinsic::returnaddress), + IRB.getInt32(0)); + if (PoisonStack) { + IRB.CreateCall(MS.MsanPoisonAllocaFn, + {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len, + IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()), + IRB.CreatePointerCast(Pc, MS.IntptrTy)}); + } else { + IRB.CreateCall(MS.MsanUnpoisonAllocaFn, + {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len}); + } + } + + void visitAllocaInst(AllocaInst &I) { + setShadow(&I, getCleanShadow(&I)); + setOrigin(&I, getCleanOrigin()); + IRBuilder<> IRB(I.getNextNode()); + const DataLayout &DL = F.getParent()->getDataLayout(); + uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType()); + Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize); + if (I.isArrayAllocation()) + Len = IRB.CreateMul(Len, I.getArraySize()); + + if (!MS.CompileKernel) + instrumentAllocaUserspace(I, IRB, Len); + else + instrumentAllocaKmsan(I, IRB, Len); + } + void visitSelectInst(SelectInst& I) { IRBuilder<> IRB(&I); // a = select b, c, d @@ -3249,12 +3494,16 @@ // An unfortunate workaround for asymmetric lowering of va_arg stuff. // See a comment in visitCallSite for more details. static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7 - static const unsigned AMD64FpEndOffset = 176; + static const unsigned AMD64FpEndOffsetUserspace = 176; + // In the kernel builds the register save area is only 48 bytes, most + // certainly due to -mno-sse. + static const unsigned AMD64FpEndOffsetKernel = 48; Function &F; MemorySanitizer &MS; MemorySanitizerVisitor &MSV; Value *VAArgTLSCopy = nullptr; + Value *VAArgTLSOriginCopy = nullptr; Value *VAArgOverflowSize = nullptr; SmallVector VAStartInstrumentationList; @@ -3287,7 +3536,9 @@ void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override { unsigned GpOffset = 0; unsigned FpOffset = AMD64GpEndOffset; - unsigned OverflowOffset = AMD64FpEndOffset; + unsigned PlatformOffset = + MS.CompileKernel ? AMD64FpEndOffsetKernel : AMD64FpEndOffsetUserspace; + unsigned OverflowOffset = PlatformOffset; const DataLayout &DL = F.getParent()->getDataLayout(); for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end(); ArgIt != End; ++ArgIt) { @@ -3306,6 +3557,10 @@ uint64_t ArgSize = DL.getTypeAllocSize(RealTy); Value *ShadowBase = getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset); + Value *OriginBase = nullptr; + if (MS.CompileKernel) { + OriginBase = getOriginPtrForVAArgument(RealTy, IRB, OverflowOffset); + } OverflowOffset += alignTo(ArgSize, 8); Value *ShadowPtr, *OriginPtr; std::tie(ShadowPtr, OriginPtr) = @@ -3314,20 +3569,30 @@ IRB.CreateMemCpy(ShadowBase, kShadowTLSAlignment, ShadowPtr, kShadowTLSAlignment, ArgSize); + if (MS.CompileKernel) { + IRB.CreateMemCpy(OriginBase, kShadowTLSAlignment, OriginPtr, + kShadowTLSAlignment, ArgSize); + } } else { ArgKind AK = classifyArgument(A); if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset) AK = AK_Memory; - if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset) + if (AK == AK_FloatingPoint && FpOffset >= PlatformOffset) AK = AK_Memory; - Value *ShadowBase; + Value *ShadowBase, *OriginBase; switch (AK) { case AK_GeneralPurpose: ShadowBase = getShadowPtrForVAArgument(A->getType(), IRB, GpOffset); + if (MS.CompileKernel) + OriginBase = + getOriginPtrForVAArgument(A->getType(), IRB, GpOffset); GpOffset += 8; break; case AK_FloatingPoint: ShadowBase = getShadowPtrForVAArgument(A->getType(), IRB, FpOffset); + if (MS.CompileKernel) + OriginBase = + getOriginPtrForVAArgument(A->getType(), IRB, FpOffset); FpOffset += 16; break; case AK_Memory: @@ -3336,18 +3601,28 @@ uint64_t ArgSize = DL.getTypeAllocSize(A->getType()); ShadowBase = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset); + if (MS.CompileKernel) + OriginBase = + getOriginPtrForVAArgument(A->getType(), IRB, OverflowOffset); OverflowOffset += alignTo(ArgSize, 8); } // Take fixed arguments into account for GpOffset and FpOffset, // but don't actually store shadows for them. - if (IsFixed) + // TODO(glider): don't call get*PtrForVAArgument() for them. + if (IsFixed) { continue; - IRB.CreateAlignedStore(MSV.getShadow(A), ShadowBase, - kShadowTLSAlignment); + } + Value *Shadow = MSV.getShadow(A); + IRB.CreateAlignedStore(Shadow, ShadowBase, kShadowTLSAlignment); + if (MS.CompileKernel) { + unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType()); + MSV.paintOrigin(IRB, MSV.getOrigin(A), OriginBase, StoreSize, + std::max(kShadowTLSAlignment, kMinOriginAlignment)); + } } } Constant *OverflowSize = - ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset); + ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - PlatformOffset); IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS); } @@ -3356,8 +3631,15 @@ int ArgOffset) { Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy); Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); + return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0)); + } + + /// Compute the shadow address for a given va_arg. + Value *getOriginPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, int ArgOffset) { + Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy); + Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0), - "_msarg"); + "_msarg_va_o"); } void unpoisonVAListTagForInst(IntrinsicInst &I) { @@ -3390,6 +3672,8 @@ } void finalizeInstrumentation() override { + unsigned PlatformOffset = + MS.CompileKernel ? AMD64FpEndOffsetKernel : AMD64FpEndOffsetUserspace; assert(!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"); if (!VAStartInstrumentationList.empty()) { @@ -3397,11 +3681,14 @@ // va_arg_tls somewhere in the function entry block. IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS); - Value *CopySize = - IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), - VAArgOverflowSize); + Value *CopySize = IRB.CreateAdd( + ConstantInt::get(MS.IntptrTy, PlatformOffset), VAArgOverflowSize); VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize); + if (MS.CompileKernel) { + VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); + IRB.CreateMemCpy(VAArgTLSOriginCopy, 8, MS.VAArgOriginTLS, 8, CopySize); + } } // Instrument va_start. @@ -3422,7 +3709,11 @@ MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment, - AMD64FpEndOffset); + PlatformOffset); + if (MS.CompileKernel) { + IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy, + Alignment, PlatformOffset); + } Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr( IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), ConstantInt::get(MS.IntptrTy, 8)), @@ -3432,10 +3723,17 @@ std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) = MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); - Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy, - AMD64FpEndOffset); + Value *SrcPtr = + IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy, PlatformOffset); IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment, VAArgOverflowSize); + + if (MS.CompileKernel) { + SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy, + PlatformOffset); + IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment, + VAArgOverflowSize); + } } } }; Index: test/Instrumentation/MemorySanitizer/alloca.ll =================================================================== --- test/Instrumentation/MemorySanitizer/alloca.ll +++ test/Instrumentation/MemorySanitizer/alloca.ll @@ -2,6 +2,7 @@ ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-poison-stack-with-call=1 -S | FileCheck %s --check-prefixes=CHECK,CALL ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=1 -S | FileCheck %s --check-prefixes=CHECK,ORIGIN ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=2 -S | FileCheck %s --check-prefixes=CHECK,ORIGIN +; RUN: opt < %s -msan -msan-kernel=1 -S | FileCheck %s --check-prefixes=CHECK,KMSAN target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" @@ -16,6 +17,7 @@ ; INLINE: call void @llvm.memset.p0i8.i64(i8* align 4 {{.*}}, i8 -1, i64 4, i1 false) ; CALL: call void @__msan_poison_stack(i8* {{.*}}, i64 4) ; ORIGIN: call void @__msan_set_alloca_origin4(i8* {{.*}}, i64 4, +; KMSAN: call void @__msan_poison_alloca(i8* {{.*}}, i64 4, ; CHECK: ret void @@ -31,6 +33,7 @@ ; INLINE: call void @llvm.memset.p0i8.i64(i8* align 4 {{.*}}, i8 -1, i64 4, i1 false) ; CALL: call void @__msan_poison_stack(i8* {{.*}}, i64 4) ; ORIGIN: call void @__msan_set_alloca_origin4(i8* {{.*}}, i64 4, +; KMSAN: call void @__msan_poison_alloca(i8* {{.*}}, i64 4, ; CHECK: ret void define void @array() sanitize_memory { @@ -43,6 +46,7 @@ ; INLINE: call void @llvm.memset.p0i8.i64(i8* align 4 {{.*}}, i8 -1, i64 20, i1 false) ; CALL: call void @__msan_poison_stack(i8* {{.*}}, i64 20) ; ORIGIN: call void @__msan_set_alloca_origin4(i8* {{.*}}, i64 20, +; KMSAN: call void @__msan_poison_alloca(i8* {{.*}}, i64 20, ; CHECK: ret void define void @array_non_const(i64 %cnt) sanitize_memory { @@ -56,4 +60,20 @@ ; INLINE: call void @llvm.memset.p0i8.i64(i8* align 4 {{.*}}, i8 -1, i64 %[[A]], i1 false) ; CALL: call void @__msan_poison_stack(i8* {{.*}}, i64 %[[A]]) ; ORIGIN: call void @__msan_set_alloca_origin4(i8* {{.*}}, i64 %[[A]], +; KMSAN: call void @__msan_poison_alloca(i8* {{.*}}, i64 %[[A]], ; CHECK: ret void + +; Check that the local is unpoisoned in the absence of sanitize_memory +define void @unpoison_local() { +entry: + %x = alloca i32, i64 5, align 4 + ret void +} + +; CHECK-LABEL: define void @unpoison_local( +; INLINE: call void @llvm.memset.p0i8.i64(i8* align 4 {{.*}}, i8 0, i64 20, i1 false) +; CALL: call void @llvm.memset.p0i8.i64(i8* align 4 {{.*}}, i8 0, i64 20, i1 false) +; ORIGIN-NOT: call void @__msan_set_alloca_origin4(i8* {{.*}}, i64 20, +; KMSAN: call void @__msan_unpoison_alloca(i8* {{.*}}, i64 20) +; CHECK: ret void + Index: test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll =================================================================== --- test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll +++ test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll @@ -0,0 +1,368 @@ +; KMSAN instrumentation tests +; RUN: opt < %s -msan -msan-kernel=1 -S | FileCheck %s -check-prefixes=CHECK + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Check the instrumentation prologue. +define void @Empty() nounwind uwtable sanitize_memory { +entry: + ret void +} + +; CHECK-LABEL: @Empty +; CHECK: entry: +; CHECK: @__msan_get_context_state() +; CHECK: %param_shadow +; CHECK: %retval_shadow +; CHECK: %va_arg_shadow +; CHECK: %va_arg_overflow_size +; CHECK: %param_origin +; CHECK: %retval_origin +; CHECK: entry.split: + +; Check instrumentation of stores + +define void @Store1(i8* nocapture %p, i8 %x) nounwind uwtable sanitize_memory { +entry: + store i8 %x, i8* %p + ret void +} + +; CHECK-LABEL: @Store1 +; CHECK-LABEL: entry: +; CHECK: @__msan_get_context_state() +; CHECK-LABEL: entry.split: +; CHECK: %param_shadow +; Load the shadow of %p and check it +; CHECK: load i64 +; CHECK: icmp +; CHECK: br i1 +; CHECK-LABEL: