Index: lib/Transforms/Instrumentation/MemorySanitizer.cpp =================================================================== --- lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -161,6 +161,11 @@ cl::desc("exact handling of relational integer ICmp"), cl::Hidden, cl::init(false)); +static cl::opt + ClKmsanEnable("msan-kernel", + cl::desc("Enable KernelMemorySanitizer instrumentation"), + cl::Hidden, cl::init(false)); + // This flag controls whether we check the shadow of the address // operand of load or store. Such bugs are very rare, since load from // a garbage address typically results in SEGV, but still happen @@ -312,14 +317,15 @@ /// uninitialized reads. class MemorySanitizer : public FunctionPass { public: - MemorySanitizer(int TrackOrigins = 0, bool Recover = false) - : FunctionPass(ID), - TrackOrigins(std::max(TrackOrigins, (int)ClTrackOrigins)), - Recover(Recover || ClKeepGoing), - WarningFn(nullptr) {} - StringRef getPassName() const override { return "MemorySanitizer"; } - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.addRequired(); + MemorySanitizer(int TrackOrigins = 0, bool Recover = false, + bool CompileKernel = false) + : FunctionPass(ID), CompileKernel(CompileKernel || ClKmsanEnable), + TrackOrigins( + CompileKernel ? 2 : std::max(TrackOrigins, (int)ClTrackOrigins)), + Recover(Recover || ClKeepGoing || CompileKernel), WarningFn(nullptr) {} + StringRef getPassName() const override { return "MemorySanitizer"; } + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); } bool runOnFunction(Function &F) override; bool doInitialization(Module &M) override; @@ -327,34 +333,40 @@ private: void initializeCallbacks(Module &M); + void initializeKmsanCallbacks(Module &M); + void createUserspaceGlobals(Module &M); /// \brief Track origins (allocation points) of uninitialized values. int TrackOrigins; bool Recover; + /// \brief True if we're compiling the Linux kernel. + bool CompileKernel; + LLVMContext *C; Type *IntptrTy; Type *OriginTy; /// \brief Thread-local shadow storage for function parameters. - GlobalVariable *ParamTLS; + Value *ParamTLS; /// \brief Thread-local origin storage for function parameters. - GlobalVariable *ParamOriginTLS; + Value *ParamOriginTLS; /// \brief Thread-local shadow storage for function return value. - GlobalVariable *RetvalTLS; + Value *RetvalTLS; /// \brief Thread-local origin storage for function return value. - GlobalVariable *RetvalOriginTLS; + Value *RetvalOriginTLS; /// \brief Thread-local shadow storage for in-register va_arg function /// parameters (x86_64-specific). - GlobalVariable *VAArgTLS; + Value *VAArgTLS; /// \brief Thread-local shadow storage for va_arg overflow area /// (x86_64-specific). - GlobalVariable *VAArgOverflowSizeTLS; + Value *VAArgOverflowSizeTLS; /// \brief Thread-local space used to pass origin value to the UMR reporting /// function. - GlobalVariable *OriginTLS; + Value *OriginTLS; /// \brief The run-time callback to print a warning. Value *WarningFn; + Value *KmsanWarning32Fn; // These arrays are indexed by log2(AccessSize). Value *MaybeWarningFn[kNumberOfAccessSizes]; Value *MaybeStoreOriginFn[kNumberOfAccessSizes]; @@ -362,6 +374,16 @@ /// \brief Run-time helper that generates a new origin value for a stack /// allocation. Value *MsanSetAllocaOrigin4Fn; + + Value *getKmsanShadowOriginAccessFn(bool isStore, int size); + + Value *KmsanPoisonAllocaFn; + Value *KmsanUnpoisonFn; + Value *KmsanLoadArgShadowFn; + Value *KmsanStoreArgShadowFn; + Value *KmsanLoadOverflowArgShadowFn; + Value *KmsanStoreOverflowArgShadowFn; + Value *KmsanRestoreVaArgShadowFn; /// \brief Run-time helper that poisons stack on function entry. Value *MsanPoisonStackFn; /// \brief Run-time helper that records a store (or any event) of an @@ -370,6 +392,13 @@ /// \brief MSan runtime replacements for memmove, memcpy and memset. Value *MemmoveFn, *MemcpyFn, *MemsetFn; + /// \brief KMSAN callbacks for task-local function argument shadow. + Value *GetContextStateFn; + Value *LoadShadowOrigin_1_8_Fn[4]; + Value *LoadShadowOrigin_n_8_Fn; + Value *StoreShadowOrigin_1_8_Fn[4]; + Value *StoreShadowOrigin_n_8_Fn; + /// \brief Memory map parameters used in application-to-shadow calculation. const MemoryMapParams *MapParams; @@ -397,8 +426,9 @@ MemorySanitizer, "msan", "MemorySanitizer: detects uninitialized reads.", false, false) -FunctionPass *llvm::createMemorySanitizerPass(int TrackOrigins, bool Recover) { - return new MemorySanitizer(TrackOrigins, Recover); +FunctionPass *llvm::createMemorySanitizerPass(int TrackOrigins, bool Recover, + bool CompileKernel) { + return new MemorySanitizer(TrackOrigins, Recover, CompileKernel); } /// \brief Create a non-const global initialized with the given string. @@ -414,6 +444,110 @@ } /// \brief Insert extern declaration of runtime-provided functions and globals. +void MemorySanitizer::initializeKmsanCallbacks(Module &M) { + IRBuilder<> IRB(*C); + + RetvalTLS = nullptr; + RetvalOriginTLS = nullptr; + ParamTLS = nullptr; + ParamOriginTLS = nullptr; + VAArgTLS = nullptr; + VAArgOverflowSizeTLS = nullptr; + OriginTLS = nullptr; + + KmsanWarning32Fn = M.getOrInsertFunction("__kmsan_warning_32", + IRB.getVoidTy(), IRB.getInt32Ty()); + GetContextStateFn = M.getOrInsertFunction( + "__kmsan_get_context_state", + PointerType::get( + StructType::get(ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), + ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), + ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), + IRB.getInt64Ty(), + ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy, + OriginTy), + 0)); + + for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) { + std::string name_load = + "__kmsan_load_shadow_origin_" + std::to_string(size); + std::string name_store = + "__kmsan_store_shadow_origin_" + std::to_string(size); + // TODO(glider): for i64 loads we should return an i64 origin, but that's + // not supported in the propagation code yet. + LoadShadowOrigin_1_8_Fn[ind] = M.getOrInsertFunction( + name_load, StructType::get(IRB.getInt64Ty(), IRB.getInt32Ty()), + PointerType::get(IRB.getInt8Ty(), 0)); + StoreShadowOrigin_1_8_Fn[ind] = M.getOrInsertFunction( + name_store, IRB.getVoidTy(), PointerType::get(IRB.getInt8Ty(), 0), + IRB.getInt64Ty(), IRB.getInt64Ty()); + } + LoadShadowOrigin_n_8_Fn = M.getOrInsertFunction( + "__kmsan_load_shadow_origin_n_8", + StructType::get(IRB.getInt64Ty(), IRB.getInt32Ty()), + PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty()); + StoreShadowOrigin_n_8_Fn = M.getOrInsertFunction( + "__kmsan_store_shadow_origin_n_8", IRB.getVoidTy(), + PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty(), IRB.getInt64Ty(), + IRB.getInt64Ty()); + + KmsanPoisonAllocaFn = M.getOrInsertFunction( + "__kmsan_poison_alloca", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy, + IRB.getInt8PtrTy(), IntptrTy); + + KmsanUnpoisonFn = M.getOrInsertFunction("__kmsan_unpoison", IRB.getVoidTy(), + IRB.getInt8PtrTy(), IntptrTy); + KmsanLoadOverflowArgShadowFn = + M.getOrInsertFunction("__kmsan_load_overflow_arg_shadow", IRB.getVoidTy(), + IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy); + KmsanLoadArgShadowFn = + M.getOrInsertFunction("__kmsan_load_arg_shadow", IRB.getVoidTy(), + IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy); + KmsanStoreArgShadowFn = + M.getOrInsertFunction("__kmsan_store_arg_shadow", IRB.getVoidTy(), + IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy); + KmsanStoreOverflowArgShadowFn = M.getOrInsertFunction( + "__kmsan_store_overflow_arg_shadow", IRB.getVoidTy(), IRB.getInt8PtrTy(), + IRB.getInt8PtrTy(), IntptrTy); + KmsanRestoreVaArgShadowFn = + M.getOrInsertFunction("__kmsan_restore_va_arg_shadow", IRB.getVoidTy(), + IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy); +} + +void MemorySanitizer::createUserspaceGlobals(Module &M) { + IRBuilder<> IRB(*C); + RetvalTLS = new GlobalVariable( + M, ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), false, + GlobalVariable::ExternalLinkage, nullptr, "__msan_retval_tls", nullptr, + GlobalVariable::InitialExecTLSModel); + + RetvalOriginTLS = new GlobalVariable( + M, OriginTy, false, GlobalVariable::ExternalLinkage, nullptr, + "__msan_retval_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel); + + ParamTLS = new GlobalVariable( + M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false, + GlobalVariable::ExternalLinkage, nullptr, "__msan_param_tls", nullptr, + GlobalVariable::InitialExecTLSModel); + + ParamOriginTLS = new GlobalVariable( + M, ArrayType::get(OriginTy, kParamTLSSize / 4), false, + GlobalVariable::ExternalLinkage, nullptr, "__msan_param_origin_tls", + nullptr, GlobalVariable::InitialExecTLSModel); + + VAArgTLS = new GlobalVariable( + M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false, + GlobalVariable::ExternalLinkage, nullptr, "__msan_va_arg_tls", nullptr, + GlobalVariable::InitialExecTLSModel); + VAArgOverflowSizeTLS = new GlobalVariable( + M, IRB.getInt64Ty(), false, GlobalVariable::ExternalLinkage, nullptr, + "__msan_va_arg_overflow_size_tls", nullptr, + GlobalVariable::InitialExecTLSModel); + OriginTLS = new GlobalVariable( + M, IRB.getInt32Ty(), false, GlobalVariable::ExternalLinkage, nullptr, + "__msan_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel); +} + void MemorySanitizer::initializeCallbacks(Module &M) { // Only do this once. if (WarningFn) @@ -459,35 +593,11 @@ "__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy); - // Create globals. - RetvalTLS = new GlobalVariable( - M, ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), false, - GlobalVariable::ExternalLinkage, nullptr, "__msan_retval_tls", nullptr, - GlobalVariable::InitialExecTLSModel); - RetvalOriginTLS = new GlobalVariable( - M, OriginTy, false, GlobalVariable::ExternalLinkage, nullptr, - "__msan_retval_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel); - - ParamTLS = new GlobalVariable( - M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false, - GlobalVariable::ExternalLinkage, nullptr, "__msan_param_tls", nullptr, - GlobalVariable::InitialExecTLSModel); - ParamOriginTLS = new GlobalVariable( - M, ArrayType::get(OriginTy, kParamTLSSize / 4), false, - GlobalVariable::ExternalLinkage, nullptr, "__msan_param_origin_tls", - nullptr, GlobalVariable::InitialExecTLSModel); - - VAArgTLS = new GlobalVariable( - M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false, - GlobalVariable::ExternalLinkage, nullptr, "__msan_va_arg_tls", nullptr, - GlobalVariable::InitialExecTLSModel); - VAArgOverflowSizeTLS = new GlobalVariable( - M, IRB.getInt64Ty(), false, GlobalVariable::ExternalLinkage, nullptr, - "__msan_va_arg_overflow_size_tls", nullptr, - GlobalVariable::InitialExecTLSModel); - OriginTLS = new GlobalVariable( - M, IRB.getInt32Ty(), false, GlobalVariable::ExternalLinkage, nullptr, - "__msan_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel); + if (!CompileKernel) { + createUserspaceGlobals(M); + } else { + initializeKmsanCallbacks(M); + } // We insert an empty inline asm after __msan_report* to avoid callback merge. EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false), @@ -495,6 +605,22 @@ /*hasSideEffects=*/true); } +Value *MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore, int size) { + Value **Fns = isStore ? StoreShadowOrigin_1_8_Fn : LoadShadowOrigin_1_8_Fn; + switch (size) { + case 1: + return Fns[0]; + case 2: + return Fns[1]; + case 4: + return Fns[2]; + case 8: + return Fns[3]; + default: + return nullptr; + } +} + /// \brief Module-level initialization. /// /// inserts a call to __msan_init to the module's constructor list. @@ -628,6 +754,7 @@ ValueMap ShadowMap, OriginMap; std::unique_ptr VAHelper; const TargetLibraryInfo *TLI; + BasicBlock *ActualFnStart; // The following flags disable parts of MSan instrumentation based on // blacklist contents and command-line options. @@ -637,6 +764,8 @@ bool PoisonUndef; bool CheckReturnValue; + SmallPtrSet SkippedFunctions; + struct ShadowOriginAndInsertPoint { Value *Shadow; Value *Origin; @@ -753,7 +882,7 @@ } } - void materializeStores(bool InstrumentWithCalls) { + void materializeStoresUserspace(bool InstrumentWithCalls) { for (StoreInst *SI : StoreList) { IRBuilder<> IRB(SI); Value *Val = SI->getValueOperand(); @@ -778,6 +907,80 @@ } } + bool setShadowOriginForStoreKmsan(IRBuilder<> &IRB, Value *Addr, + Value *Shadow, Value *Origin) { + Type *ShadowTy = Shadow->getType(); + int Size = 1; + int BitWidth = dyn_cast(ShadowTy)->getBitWidth(); + if (isa(ShadowTy)) { + // TODO(glider): visitAllocaInst() passes int8* as ShadowTy. + Size = BitWidth / 8; + // Make sure Size is at least 1 if the operand is i1. + if (Size * 8 < BitWidth) + Size++; + } + + Shadow = IRB.CreateIntCast(Shadow, IRB.getInt64Ty(), /* signed */ false); + Value *Setter = MS.getKmsanShadowOriginAccessFn(/*isStore*/ true, Size); + Origin = IRB.CreateIntCast(Origin, MS.IntptrTy, /* isSigned */ false); + Addr = IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0)); + if (Setter) { + IRB.CreateCall(Setter, {Addr, Shadow, Origin}); + return true; + } else { + Value *SizeVal = ConstantInt::get(IRB.getInt64Ty(), Size); + if (Size < 8) { + Setter = MS.StoreShadowOrigin_n_8_Fn; + IRB.CreateCall(Setter, {Addr, Shadow, Origin, SizeVal}); + return true; + } else { + Constant *ConstantShadow = dyn_cast_or_null(Shadow); + if (ConstantShadow && ConstantShadow->isZeroValue()) { + IRB.CreateCall(MS.KmsanUnpoisonFn, {Addr, SizeVal}); + return true; + } else { + // TODO(glider): unpoison the shadow if there's no callback for this + // access size. + IRB.CreateCall(MS.KmsanUnpoisonFn, {Addr, SizeVal}); + return false; + } + } + } + } + + void materializeStoresKmsan() { + for (StoreInst *SI : StoreList) { + IRBuilder<> IRB(SI); + Value *Val = SI->getValueOperand(); + Value *Addr = SI->getPointerOperand(); + Value *Shadow = SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val); + Value *Origin = getOrigin(Val); + + if (!setShadowOriginForStoreKmsan(IRB, Addr, Shadow, Origin)) { + // TODO(glider): Clang produces i80 stores for some structs with + // bitfields. Ignore them for now, we'll try to turn this behavior off + // in the future. + errs() << "Missing setter for the following instruction:\n" + << *SI << "\n"; + errs() << "Shadow: " << *Shadow << "\n"; + errs() << "Origin: " << *Origin << "\n"; + // assert(false); + } + if (ClCheckAccessAddress) + insertShadowCheck(Addr, SI); + + if (SI->isAtomic()) + SI->setOrdering(addReleaseOrdering(SI->getOrdering())); + } + } + + void materializeStores(bool InstrumentWithCalls) { + if (MS.CompileKernel) + materializeStoresKmsan(); + else + materializeStoresUserspace(InstrumentWithCalls); + } + void materializeOneCheck(Instruction *OrigIns, Value *Shadow, Value *Origin, bool AsCall) { IRBuilder<> IRB(OrigIns); @@ -820,11 +1023,24 @@ /* Unreachable */ !MS.Recover, MS.ColdCallWeights); IRB.SetInsertPoint(CheckTerm); - if (MS.TrackOrigins) { - IRB.CreateStore(Origin ? (Value *)Origin : (Value *)IRB.getInt32(0), - MS.OriginTLS); + if (!MS.CompileKernel) { + if (MS.TrackOrigins) { + Value *OriginTLS = MS.OriginTLS; + IRB.CreateStore(Origin ? Origin : (Value *)IRB.getInt32(0), + OriginTLS); + } + IRB.CreateCall(MS.WarningFn, {}); + } else { + int OrBitWidth = + Origin ? DL.getTypeStoreSizeInBits(Origin->getType()) : 32; + Value *ActualOrigin = 0; + assert(OrBitWidth <= 32); + ActualOrigin = Origin ? Origin : (Value *)IRB.getInt32(0); + if (OrBitWidth != 32) + ActualOrigin = + IRB.CreateIntCast(ActualOrigin, IRB.getInt32Ty(), false); + IRB.CreateCall(MS.KmsanWarning32Fn, ActualOrigin); } - IRB.CreateCall(MS.WarningFn, {}); IRB.CreateCall(MS.EmptyAsm, {}); DEBUG(dbgs() << " CHECK: " << *Cmp << "\n"); } @@ -840,9 +1056,30 @@ DEBUG(dbgs() << "DONE:\n" << F); } + BasicBlock *insertKmsanPrologue(Function &F) { + BasicBlock *ret = + SplitBlock(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHI()); + IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); + Value *ContextState = IRB.CreateCall(MS.GetContextStateFn, {}); + Constant *Zero = IRB.getInt32(0); + MS.ParamTLS = IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(0)}); + MS.RetvalTLS = IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(1)}); + MS.VAArgTLS = IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(2)}); + MS.VAArgOverflowSizeTLS = + IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(3)}); + MS.ParamOriginTLS = IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(4)}); + MS.RetvalOriginTLS = IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(5)}); + MS.OriginTLS = IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(6)}); + return ret; + } + /// \brief Add MemorySanitizer instrumentation to a function. bool runOnFunction() { MS.initializeCallbacks(*F.getParent()); + if (MS.CompileKernel) + ActualFnStart = insertKmsanPrologue(F); + else + ActualFnStart = &F.getEntryBlock(); // In the presence of unreachable blocks, we may see Phi nodes with // incoming nodes from such blocks. Since InstVisitor skips unreachable @@ -853,7 +1090,7 @@ // Iterate all BBs in depth-first order and create shadow instructions // for all instructions (where applicable). // For PHI nodes we create dummy shadow PHIs which will be finalized later. - for (BasicBlock *BB : depth_first(&F.getEntryBlock())) + for (BasicBlock *BB : depth_first(ActualFnStart)) visit(*BB); @@ -958,8 +1195,8 @@ /// address. /// /// Shadow = ShadowBase + Offset - Value *getShadowPtr(Value *Addr, Type *ShadowTy, - IRBuilder<> &IRB) { + Value *getShadowPtr(Value *Addr, Type *ShadowTy, IRBuilder<> &IRB) { + assert(!MS.CompileKernel); Value *ShadowLong = getShadowPtrOffset(Addr, IRB); uint64_t ShadowBase = MS.MapParams->ShadowBase; if (ShadowBase != 0) @@ -1112,7 +1349,7 @@ if (*ShadowPtr) return *ShadowPtr; Function *F = A->getParent(); - IRBuilder<> EntryIRB(F->getEntryBlock().getFirstNonPHI()); + IRBuilder<> EntryIRB(ActualFnStart->getFirstNonPHI()); unsigned ArgOffset = 0; const DataLayout &DL = F->getParent()->getDataLayout(); for (auto &FArg : F->args()) { @@ -1138,16 +1375,32 @@ } if (Overflow) { // ParamTLS overflow. - EntryIRB.CreateMemSet( - getShadowPtr(V, EntryIRB.getInt8Ty(), EntryIRB), - Constant::getNullValue(EntryIRB.getInt8Ty()), Size, ArgAlign); + if (!MS.CompileKernel) { + EntryIRB.CreateMemSet( + getShadowPtr(V, EntryIRB.getInt8Ty(), EntryIRB), + Constant::getNullValue(EntryIRB.getInt8Ty()), Size, + ArgAlign); + } else { + Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size); + EntryIRB.CreateCall(MS.KmsanUnpoisonFn, {V, SizeVal}); + } } else { - unsigned CopyAlign = std::min(ArgAlign, kShadowTLSAlignment); - Value *Cpy = EntryIRB.CreateMemCpy( - getShadowPtr(V, EntryIRB.getInt8Ty(), EntryIRB), Base, Size, - CopyAlign); - DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n"); - (void)Cpy; + if (!MS.CompileKernel) { + unsigned CopyAlign = std::min(ArgAlign, kShadowTLSAlignment); + Value *Cpy = EntryIRB.CreateMemCpy( + getShadowPtr(V, EntryIRB.getInt8Ty(), EntryIRB), Base, Size, + CopyAlign); + DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n"); + (void)Cpy; + } else { + // TODO(glider): do we need the above alignment? + Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size); + V = EntryIRB.CreatePointerCast(V, EntryIRB.getInt8PtrTy()); + Base = + EntryIRB.CreatePointerCast(Base, EntryIRB.getInt8PtrTy()); + EntryIRB.CreateCall(MS.KmsanLoadArgShadowFn, + {V, Base, SizeVal}); + } } *ShadowPtr = getCleanShadow(V); } else { @@ -1275,8 +1528,9 @@ /// /// Loads the corresponding shadow and (optionally) origin. /// Optionally, checks that the load address is fully defined. - void visitLoadInst(LoadInst &I) { + void visitLoadInstUserspace(LoadInst &I) { assert(I.getType()->isSized() && "Load type must have size"); + assert(!MS.CompileKernel); IRBuilder<> IRB(I.getNextNode()); Type *ShadowTy = getShadowTy(&I); Value *Addr = I.getPointerOperand(); @@ -1305,6 +1559,61 @@ } } } + void visitLoadInstKmsan(LoadInst &I) { + assert(I.getType()->isSized() && "Load type must have size"); + IRBuilder<> IRB(I.getNextNode()); + Type *ShadowTy = getShadowTy(&I); + Value *Addr = I.getPointerOperand(); + if (PropagateShadow && !I.getMetadata("nosanitize")) { + int Size = 1; + int BitWidth = dyn_cast(ShadowTy)->getBitWidth(); + if (isa(ShadowTy)) { + // TODO(glider): visitAllocaInst() passes int8* as ShadowTy. + Size = BitWidth / 8; + // Make sure Size is at least 1 if the operand is i1. + if (Size * 8 < BitWidth) + Size++; + } + Addr = IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0)); + Value *ShadowOrigin = nullptr; + Value *Getter = MS.getKmsanShadowOriginAccessFn(/*isStore*/ false, Size); + if (!Getter) { + if (Size > 8) { + errs() << "Missing getter for the following instruction:\n" + << I << "\n"; + assert(false); + } + Getter = MS.LoadShadowOrigin_n_8_Fn; + Value *SizeVal = ConstantInt::get(IRB.getInt64Ty(), Size); + ShadowOrigin = IRB.CreateCall(Getter, {Addr, SizeVal}); + } else { + ShadowOrigin = IRB.CreateCall(Getter, Addr); + } + Value *Shadow = IRB.CreateExtractValue(ShadowOrigin, 0); + Shadow = IRB.CreateIntCast(Shadow, ShadowTy, /* isSigned */ false); + Value *Origin = IRB.CreateExtractValue(ShadowOrigin, 1); + + setShadow(&I, Shadow); + setOrigin(&I, Origin); + } else { + setShadow(&I, getCleanShadow(&I)); + setOrigin(&I, getCleanOrigin()); + } + + if (ClCheckAccessAddress) + insertShadowCheck(I.getPointerOperand(), &I); + + // TODO(glider): do we need it for KMSAN? + if (I.isAtomic()) + I.setOrdering(addAcquireOrdering(I.getOrdering())); + } + + void visitLoadInst(LoadInst &I) { + if (MS.CompileKernel) + visitLoadInstKmsan(I); + else + visitLoadInstUserspace(I); + } /// \brief Instrument StoreInst /// @@ -1319,7 +1628,6 @@ IRBuilder<> IRB(&I); Value *Addr = I.getOperand(0); - Value *ShadowPtr = getShadowPtr(Addr, I.getType(), IRB); if (ClCheckAccessAddress) insertShadowCheck(Addr, &I); @@ -1330,8 +1638,13 @@ if (isa(I)) insertShadowCheck(I.getOperand(1), &I); - IRB.CreateStore(getCleanShadow(&I), ShadowPtr); - + if (!MS.CompileKernel) { + Value *ShadowPtr = getShadowPtr(Addr, I.getType(), IRB); + IRB.CreateStore(getCleanShadow(&I), ShadowPtr); + } else { + setShadowOriginForStoreKmsan(IRB, Addr, getCleanShadow(&I), + getCleanOrigin()); + } setShadow(&I, getCleanShadow(&I)); setOrigin(&I, getCleanOrigin()); } @@ -1937,6 +2250,8 @@ /// Instrument intrinsics that look like a simple SIMD store: writes memory, /// has 1 pointer argument and 1 vector argument, returns void. bool handleVectorStoreIntrinsic(IntrinsicInst &I) { + // FIXME: KMSAN instrumentation doesn't handle vector instructions for now. + assert(!MS.CompileKernel); IRBuilder<> IRB(&I); Value* Addr = I.getArgOperand(0); Value *Shadow = getShadow(&I, 1); @@ -1960,6 +2275,8 @@ /// Instrument intrinsics that look like a simple SIMD load: reads memory, /// has 1 pointer argument, returns a vector. bool handleVectorLoadIntrinsic(IntrinsicInst &I) { + // FIXME: KMSAN instrumentation doesn't handle vector instructions for now. + assert(!MS.CompileKernel); IRBuilder<> IRB(&I); Value *Addr = I.getArgOperand(0); @@ -2607,6 +2924,9 @@ // prevent this code from being optimized out, mark that function // non-readonly in advance. if (Function *Func = Call->getCalledFunction()) { + // If the called function is one of those inserted by us, do nothing. + if (Func == MS.GetContextStateFn) + return; // Clear out readonly/readnone attributes. AttrBuilder B; B.addAttribute(Attribute::ReadOnly) @@ -2646,9 +2966,20 @@ if (ArgOffset + Size > kParamTLSSize) break; unsigned ParamAlignment = CS.getParamAlignment(i); unsigned Alignment = std::min(ParamAlignment, kShadowTLSAlignment); - Store = IRB.CreateMemCpy(ArgShadowBase, - getShadowPtr(A, Type::getInt8Ty(*MS.C), IRB), - Size, Alignment); + if (!MS.CompileKernel) { + Store = IRB.CreateMemCpy(ArgShadowBase, + getShadowPtr(A, Type::getInt8Ty(*MS.C), IRB), + Size, Alignment); + } else { + // TODO(glider): do we need the above alignment? + Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size); + ArgShadowBase = + IRB.CreatePointerCast(ArgShadowBase, IRB.getInt8PtrTy()); + A = IRB.CreatePointerCast(A, IRB.getInt8PtrTy()); + // TODO(glider): __kmsan_store_arg_shadow() should handle the origins + // as well. + IRB.CreateCall(MS.KmsanStoreArgShadowFn, {ArgShadowBase, A, SizeVal}); + } } else { Size = DL.getTypeAllocSize(A->getType()); if (ArgOffset + Size > kParamTLSSize) break; @@ -2657,11 +2988,12 @@ Constant *Cst = dyn_cast(ArgShadow); if (Cst && Cst->isNullValue()) ArgIsInitialized = true; } - if (MS.TrackOrigins && !ArgIsInitialized) + if (MS.TrackOrigins && !ArgIsInitialized && !MS.CompileKernel) { IRB.CreateStore(getOrigin(A), getOriginPtrForArgument(A, IRB, ArgOffset)); + } (void)Store; - assert(Size != 0 && Store != nullptr); + assert((Size != 0 && Store != nullptr) || MS.CompileKernel); DEBUG(dbgs() << " Param:" << *Store << "\n"); ArgOffset += alignTo(Size, 8); } @@ -2732,8 +3064,9 @@ } else { Value *Shadow = getShadow(RetVal); IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment); - if (MS.TrackOrigins) + if (MS.TrackOrigins) { IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB)); + } } } @@ -2753,15 +3086,20 @@ "_msphi_o")); } - void visitAllocaInst(AllocaInst &I) { - setShadow(&I, getCleanShadow(&I)); - setOrigin(&I, getCleanOrigin()); - IRBuilder<> IRB(I.getNextNode()); - const DataLayout &DL = F.getParent()->getDataLayout(); - uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType()); - Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize); - if (I.isArrayAllocation()) - Len = IRB.CreateMul(Len, I.getArraySize()); + Value *getLocalVarDescription(AllocaInst &I) { + SmallString<2048> StackDescriptionStorage; + raw_svector_ostream StackDescription(StackDescriptionStorage); + // We create a string with a description of the stack allocation and + // pass it into __msan_set_alloca_origin. + // It will be printed by the run-time if stack-originated UMR is found. + // The first 4 bytes of the string are set to '----' and will be replaced + // by __msan_va_arg_overflow_size_tls at the first call. + StackDescription << "----" << I.getName() << "@" << F.getName(); + return createPrivateNonConstGlobalForString(*F.getParent(), + StackDescription.str()); + } + + void instrumentAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) { if (PoisonStack && ClPoisonStackWithCall) { IRB.CreateCall(MS.MsanPoisonStackFn, {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len}); @@ -2772,18 +3110,7 @@ } if (PoisonStack && MS.TrackOrigins) { - SmallString<2048> StackDescriptionStorage; - raw_svector_ostream StackDescription(StackDescriptionStorage); - // We create a string with a description of the stack allocation and - // pass it into __msan_set_alloca_origin. - // It will be printed by the run-time if stack-originated UMR is found. - // The first 4 bytes of the string are set to '----' and will be replaced - // by __msan_va_arg_overflow_size_tls at the first call. - StackDescription << "----" << I.getName() << "@" << F.getName(); - Value *Descr = - createPrivateNonConstGlobalForString(*F.getParent(), - StackDescription.str()); - + Value *Descr = getLocalVarDescription(I); IRB.CreateCall(MS.MsanSetAllocaOrigin4Fn, {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len, IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()), @@ -2791,6 +3118,33 @@ } } + void instrumentAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) { + Value *Descr = getLocalVarDescription(I); + Value *Pc = IRB.CreateCall( + Intrinsic::getDeclaration(F.getParent(), Intrinsic::returnaddress), + IRB.getInt32(0)); + IRB.CreateCall(MS.KmsanPoisonAllocaFn, + {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len, + IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()), + IRB.CreatePointerCast(Pc, MS.IntptrTy)}); + } + + void visitAllocaInst(AllocaInst &I) { + setShadow(&I, getCleanShadow(&I)); + setOrigin(&I, getCleanOrigin()); + IRBuilder<> IRB(I.getNextNode()); + const DataLayout &DL = F.getParent()->getDataLayout(); + uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType()); + Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize); + if (I.isArrayAllocation()) + Len = IRB.CreateMul(Len, I.getArraySize()); + + if (!MS.CompileKernel) + instrumentAllocaUserspace(I, IRB, Len); + else + instrumentAllocaKmsan(I, IRB, Len); + } + void visitSelectInst(SelectInst& I) { IRBuilder<> IRB(&I); // a = select b, c, d @@ -2989,8 +3343,13 @@ uint64_t ArgSize = DL.getTypeAllocSize(RealTy); Value *Base = getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset); OverflowOffset += alignTo(ArgSize, 8); - IRB.CreateMemCpy(Base, MSV.getShadowPtr(A, IRB.getInt8Ty(), IRB), - ArgSize, kShadowTLSAlignment); + if (!MS.CompileKernel) { + IRB.CreateMemCpy(Base, MSV.getShadowPtr(A, IRB.getInt8Ty(), IRB), + ArgSize, kShadowTLSAlignment); + } else { + Value *ArgSizeV = ConstantInt::get(MS.IntptrTy, ArgSize); + IRB.CreateCall(MS.KmsanStoreOverflowArgShadowFn, {Base, A, ArgSizeV}); + } } else { ArgKind AK = classifyArgument(A); if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset) @@ -3035,31 +3394,41 @@ "_msarg"); } - void visitVAStartInst(VAStartInst &I) override { + void visitVAStartInst(VAStartInst &I) { if (F.getCallingConv() == CallingConv::X86_64_Win64) return; IRBuilder<> IRB(&I); VAStartInstrumentationList.push_back(&I); Value *VAListTag = I.getArgOperand(0); - Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB); // Unpoison the whole __va_list_tag. // FIXME: magic ABI constants. - IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), - /* size */24, /* alignment */8, false); + if (!MS.CompileKernel) { + Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB); + IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), + /* size */ 24, /* alignment */ 8, false); + } else { + Value *Size = ConstantInt::get(MS.IntptrTy, 24); + IRB.CreateCall(MS.KmsanUnpoisonFn, {VAListTag, Size}); + } } - void visitVACopyInst(VACopyInst &I) override { + void visitVACopyInst(VACopyInst &I) { if (F.getCallingConv() == CallingConv::X86_64_Win64) return; IRBuilder<> IRB(&I); Value *VAListTag = I.getArgOperand(0); - Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB); // Unpoison the whole __va_list_tag. // FIXME: magic ABI constants. - IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), - /* size */24, /* alignment */8, false); + if (!MS.CompileKernel) { + Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB); + IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), + /* size */ 24, /* alignment */ 8, false); + } else { + Value *Size = ConstantInt::get(MS.IntptrTy, 24); + IRB.CreateCall(MS.KmsanUnpoisonFn, {VAListTag, Size}); + } } void finalizeInstrumentation() override { @@ -3068,7 +3437,7 @@ if (!VAStartInstrumentationList.empty()) { // If there is a va_start in this function, make a backup copy of // va_arg_tls somewhere in the function entry block. - IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); + IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS); Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), @@ -3090,10 +3459,17 @@ ConstantInt::get(MS.IntptrTy, 16)), Type::getInt64PtrTy(*MS.C)); Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr); - Value *RegSaveAreaShadowPtr = - MSV.getShadowPtr(RegSaveAreaPtr, IRB.getInt8Ty(), IRB); - IRB.CreateMemCpy(RegSaveAreaShadowPtr, VAArgTLSCopy, - AMD64FpEndOffset, 16); + if (!MS.CompileKernel) { + Value *RegSaveAreaShadowPtr = + MSV.getShadowPtr(RegSaveAreaPtr, IRB.getInt8Ty(), IRB); + IRB.CreateMemCpy(RegSaveAreaShadowPtr, VAArgTLSCopy, AMD64FpEndOffset, + 16); + } else { + Value *Offset = ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset); + RegSaveAreaPtr = IRB.CreateIntToPtr(RegSaveAreaPtr, IRB.getInt8PtrTy()); + IRB.CreateCall(MS.KmsanRestoreVaArgShadowFn, + {RegSaveAreaPtr, VAArgTLSCopy, Offset}); + } Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr( @@ -3101,11 +3477,19 @@ ConstantInt::get(MS.IntptrTy, 8)), Type::getInt64PtrTy(*MS.C)); Value *OverflowArgAreaPtr = IRB.CreateLoad(OverflowArgAreaPtrPtr); - Value *OverflowArgAreaShadowPtr = - MSV.getShadowPtr(OverflowArgAreaPtr, IRB.getInt8Ty(), IRB); Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy, AMD64FpEndOffset); - IRB.CreateMemCpy(OverflowArgAreaShadowPtr, SrcPtr, VAArgOverflowSize, 16); + if (!MS.CompileKernel) { + Value *OverflowArgAreaShadowPtr = + MSV.getShadowPtr(OverflowArgAreaPtr, IRB.getInt8Ty(), IRB); + IRB.CreateMemCpy(OverflowArgAreaShadowPtr, SrcPtr, VAArgOverflowSize, + 16); + } else { + OverflowArgAreaPtr = + IRB.CreateIntToPtr(OverflowArgAreaPtr, IRB.getInt8PtrTy()); + IRB.CreateCall(MS.KmsanLoadOverflowArgShadowFn, + {OverflowArgAreaPtr, SrcPtr, VAArgOverflowSize}); + } } } }; @@ -3171,7 +3555,7 @@ /* size */8, /* alignment */8, false); } - void visitVACopyInst(VACopyInst &I) override { + void visitVACopyInst(VACopyInst &I) { IRBuilder<> IRB(&I); Value *VAListTag = I.getArgOperand(0); Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB); @@ -3184,7 +3568,7 @@ void finalizeInstrumentation() override { assert(!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"); - IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); + IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS); Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize); @@ -3365,7 +3749,7 @@ if (!VAStartInstrumentationList.empty()) { // If there is a va_start in this function, make a backup copy of // va_arg_tls somewhere in the function entry block. - IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); + IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS); Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset), @@ -3587,7 +3971,7 @@ void finalizeInstrumentation() override { assert(!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"); - IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); + IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS); Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);