Index: include/llvm/Transforms/Instrumentation.h =================================================================== --- include/llvm/Transforms/Instrumentation.h +++ include/llvm/Transforms/Instrumentation.h @@ -84,8 +84,8 @@ const InstrProfOptions &Options = InstrProfOptions()); // Insert AddressSanitizer (address sanity checking) instrumentation -FunctionPass *createAddressSanitizerFunctionPass(); -ModulePass *createAddressSanitizerModulePass(); +FunctionPass *createAddressSanitizerFunctionPass(bool CompileKernel = false); +ModulePass *createAddressSanitizerModulePass(bool CompileKernel = false); // Insert MemorySanitizer instrumentation (detection of uninitialized reads) FunctionPass *createMemorySanitizerPass(int TrackOrigins = 0); Index: lib/Transforms/Instrumentation/AddressSanitizer.cpp =================================================================== --- lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -67,6 +67,7 @@ static const uint64_t kIOSShadowOffset32 = 1ULL << 30; static const uint64_t kDefaultShadowOffset64 = 1ULL << 44; static const uint64_t kSmallX86_64ShadowOffset = 0x7FFF8000; // < 2G. +static const uint64_t kLinuxKasan_ShadowOffset64 = 0xdffffc0000000000; static const uint64_t kPPC64_ShadowOffset64 = 1ULL << 41; static const uint64_t kMIPS32_ShadowOffset32 = 0x0aaa0000; static const uint64_t kMIPS64_ShadowOffset64 = 1ULL << 37; @@ -315,7 +316,8 @@ bool OrShadowOffset; }; -static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize) { +static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize, + bool IsKasan) { bool IsAndroid = TargetTriple.getEnvironment() == llvm::Triple::Android; bool IsIOS = TargetTriple.isiOS(); bool IsFreeBSD = TargetTriple.isOSFreeBSD(); @@ -350,9 +352,12 @@ Mapping.Offset = kPPC64_ShadowOffset64; else if (IsFreeBSD) Mapping.Offset = kFreeBSD_ShadowOffset64; - else if (IsLinux && IsX86_64) - Mapping.Offset = kSmallX86_64ShadowOffset; - else if (IsMIPS64) + else if (IsLinux && IsX86_64) { + if (IsKasan) + Mapping.Offset = kLinuxKasan_ShadowOffset64; + else + Mapping.Offset = kSmallX86_64ShadowOffset; + } else if (IsMIPS64) Mapping.Offset = kMIPS64_ShadowOffset64; else if (IsAArch64) Mapping.Offset = kAArch64_ShadowOffset64; @@ -381,7 +386,8 @@ /// AddressSanitizer: instrument the code in module to find memory bugs. struct AddressSanitizer : public FunctionPass { - AddressSanitizer() : FunctionPass(ID) { + explicit AddressSanitizer(bool CompileKernel = false) + : FunctionPass(ID), CompileKernel(CompileKernel) { initializeAddressSanitizerPass(*PassRegistry::getPassRegistry()); } const char *getPassName() const override { @@ -444,6 +450,7 @@ LLVMContext *C; Triple TargetTriple; int LongSize; + bool CompileKernel; Type *IntptrTy; ShadowMapping Mapping; DominatorTree *DT; @@ -467,7 +474,8 @@ class AddressSanitizerModule : public ModulePass { public: - AddressSanitizerModule() : ModulePass(ID) {} + explicit AddressSanitizerModule(bool CompileKernel = false) + : ModulePass(ID), CompileKernel(CompileKernel) {} bool runOnModule(Module &M) override; static char ID; // Pass identification, replacement for typeid const char *getPassName() const override { return "AddressSanitizerModule"; } @@ -484,6 +492,7 @@ } GlobalsMetadata GlobalsMD; + bool CompileKernel; Type *IntptrTy; LLVMContext *C; Triple TargetTriple; @@ -689,8 +698,8 @@ AddressSanitizer, "asan", "AddressSanitizer: detects use-after-free and out-of-bounds bugs.", false, false) -FunctionPass *llvm::createAddressSanitizerFunctionPass() { - return new AddressSanitizer(); +FunctionPass *llvm::createAddressSanitizerFunctionPass(bool CompileKernel) { + return new AddressSanitizer(CompileKernel); } char AddressSanitizerModule::ID = 0; @@ -699,8 +708,8 @@ "AddressSanitizer: detects use-after-free and out-of-bounds bugs." "ModulePass", false, false) -ModulePass *llvm::createAddressSanitizerModulePass() { - return new AddressSanitizerModule(); +ModulePass *llvm::createAddressSanitizerModulePass(bool CompileKernel) { + return new AddressSanitizerModule(CompileKernel); } static size_t TypeSizeToSizeIndex(uint32_t TypeSize) { @@ -1344,16 +1353,17 @@ int LongSize = M.getDataLayout().getPointerSizeInBits(); IntptrTy = Type::getIntNTy(*C, LongSize); TargetTriple = Triple(M.getTargetTriple()); - Mapping = getShadowMapping(TargetTriple, LongSize); + Mapping = getShadowMapping(TargetTriple, LongSize, CompileKernel); initializeCallbacks(M); bool Changed = false; - Function *CtorFunc = M.getFunction(kAsanModuleCtorName); - assert(CtorFunc); - IRBuilder<> IRB(CtorFunc->getEntryBlock().getTerminator()); - - if (ClGlobals) Changed |= InstrumentGlobals(IRB, M); + if (ClGlobals && !CompileKernel) { + Function *CtorFunc = M.getFunction(kAsanModuleCtorName); + assert(CtorFunc); + IRBuilder<> IRB(CtorFunc->getEntryBlock().getTerminator()); + Changed |= InstrumentGlobals(IRB, M); + } return Changed; } @@ -1366,10 +1376,11 @@ for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) { const std::string TypeStr = AccessIsWrite ? "store" : "load"; const std::string ExpStr = Exp ? "exp_" : ""; + const std::string SuffixStr = CompileKernel ? "N" : "_n"; const Type *ExpType = Exp ? Type::getInt32Ty(*C) : nullptr; AsanErrorCallbackSized[AccessIsWrite][Exp] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - kAsanReportErrorTemplate + ExpStr + TypeStr + "_n", + kAsanReportErrorTemplate + ExpStr + TypeStr + SuffixStr, IRB.getVoidTy(), IntptrTy, IntptrTy, ExpType, nullptr)); AsanMemoryAccessCallbackSized[AccessIsWrite][Exp] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( @@ -1390,14 +1401,16 @@ } } + const std::string MemIntrinCallbackPrefix = + CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix; AsanMemmove = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - ClMemoryAccessCallbackPrefix + "memmove", IRB.getInt8PtrTy(), + MemIntrinCallbackPrefix + "memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy, nullptr)); AsanMemcpy = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - ClMemoryAccessCallbackPrefix + "memcpy", IRB.getInt8PtrTy(), + MemIntrinCallbackPrefix + "memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy, nullptr)); AsanMemset = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - ClMemoryAccessCallbackPrefix + "memset", IRB.getInt8PtrTy(), + MemIntrinCallbackPrefix + "memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy, nullptr)); AsanHandleNoReturnFunc = checkSanitizerInterfaceFunction( @@ -1424,14 +1437,15 @@ IntptrTy = Type::getIntNTy(*C, LongSize); TargetTriple = Triple(M.getTargetTriple()); - std::tie(AsanCtorFunction, AsanInitFunction) = - createSanitizerCtorAndInitFunctions(M, kAsanModuleCtorName, kAsanInitName, - /*InitArgTypes=*/{}, - /*InitArgs=*/{}); - - Mapping = getShadowMapping(TargetTriple, LongSize); - - appendToGlobalCtors(M, AsanCtorFunction, kAsanCtorAndDtorPriority); + if (!CompileKernel) { + std::tie(AsanCtorFunction, AsanInitFunction) = + createSanitizerCtorAndInitFunctions(M, kAsanModuleCtorName, + kAsanInitName, + /*InitArgTypes=*/{}, + /*InitArgs=*/{}); + appendToGlobalCtors(M, AsanCtorFunction, kAsanCtorAndDtorPriority); + } + Mapping = getShadowMapping(TargetTriple, LongSize, CompileKernel); return true; } @@ -1513,7 +1527,7 @@ } } - bool UseCalls = false; + bool UseCalls = CompileKernel; if (ClInstrumentationWithCallsThreshold >= 0 && ToInstrument.size() > (unsigned)ClInstrumentationWithCallsThreshold) UseCalls = true; @@ -1732,8 +1746,8 @@ ComputeASanStackFrameLayout(SVD, 1UL << Mapping.Scale, MinHeaderSize, &L); DEBUG(dbgs() << L.DescriptionString << " --- " << L.FrameSize << "\n"); uint64_t LocalStackSize = L.FrameSize; - bool DoStackMalloc = - ClUseAfterReturn && LocalStackSize <= kMaxStackMallocSize; + bool DoStackMalloc = ClUseAfterReturn && !ASan.CompileKernel && + LocalStackSize <= kMaxStackMallocSize; // Don't do dynamic alloca in presence of inline asm: too often it makes // assumptions on which registers are available. Don't do stack malloc in the // presence of inline asm on 32-bit platforms for the same reason. Index: tools/clang/include/clang/Basic/Sanitizers.def =================================================================== --- tools/clang/include/clang/Basic/Sanitizers.def +++ tools/clang/include/clang/Basic/Sanitizers.def @@ -41,6 +41,9 @@ // AddressSanitizer SANITIZER("address", Address) +// Kernel AddressSanitizer (KASAN) +SANITIZER("kernel-address", KernelAddress) + // MemorySanitizer SANITIZER("memory", Memory) Index: tools/clang/lib/AST/Decl.cpp =================================================================== --- tools/clang/lib/AST/Decl.cpp +++ tools/clang/lib/AST/Decl.cpp @@ -3681,7 +3681,8 @@ bool RecordDecl::mayInsertExtraPadding(bool EmitRemark) const { ASTContext &Context = getASTContext(); - if (!Context.getLangOpts().Sanitize.has(SanitizerKind::Address) || + if (!(Context.getLangOpts().Sanitize.has(SanitizerKind::Address) || + Context.getLangOpts().Sanitize.has(SanitizerKind::KernelAddress)) || !Context.getLangOpts().SanitizeAddressFieldPadding) return false; const auto &Blacklist = Context.getSanitizerBlacklist(); Index: tools/clang/lib/CodeGen/BackendUtil.cpp =================================================================== --- tools/clang/lib/CodeGen/BackendUtil.cpp +++ tools/clang/lib/CodeGen/BackendUtil.cpp @@ -205,6 +205,12 @@ PM.add(createAddressSanitizerModulePass()); } +static void addKernelAddressSanitizerPasses(const PassManagerBuilder &Builder, + legacy::PassManagerBase &PM) { + PM.add(createAddressSanitizerFunctionPass(/*CompileKernel*/true)); + PM.add(createAddressSanitizerModulePass(/*CompileKernel*/true)); +} + static void addMemorySanitizerPass(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { const PassManagerBuilderWrapper &BuilderWrapper = @@ -329,6 +335,13 @@ addAddressSanitizerPasses); } + if (LangOpts.Sanitize.has(SanitizerKind::KernelAddress)) { + PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, + addKernelAddressSanitizerPasses); + PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, + addKernelAddressSanitizerPasses); + } + if (LangOpts.Sanitize.has(SanitizerKind::Memory)) { PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, addMemorySanitizerPass); Index: tools/clang/lib/CodeGen/CGDeclCXX.cpp =================================================================== --- tools/clang/lib/CodeGen/CGDeclCXX.cpp +++ tools/clang/lib/CodeGen/CGDeclCXX.cpp @@ -267,7 +267,8 @@ Fn->setDoesNotThrow(); if (!isInSanitizerBlacklist(Fn, Loc)) { - if (getLangOpts().Sanitize.has(SanitizerKind::Address)) + if (getLangOpts().Sanitize.has(SanitizerKind::Address) || + getLangOpts().Sanitize.has(SanitizerKind::KernelAddress)) Fn->addFnAttr(llvm::Attribute::SanitizeAddress); if (getLangOpts().Sanitize.has(SanitizerKind::Thread)) Fn->addFnAttr(llvm::Attribute::SanitizeThread); Index: tools/clang/lib/CodeGen/CodeGenFunction.cpp =================================================================== --- tools/clang/lib/CodeGen/CodeGenFunction.cpp +++ tools/clang/lib/CodeGen/CodeGenFunction.cpp @@ -615,7 +615,7 @@ } // Apply sanitizer attributes to the function. - if (SanOpts.has(SanitizerKind::Address)) + if (SanOpts.has(SanitizerKind::Address) || SanOpts.has(SanitizerKind::KernelAddress)) Fn->addFnAttr(llvm::Attribute::SanitizeAddress); if (SanOpts.has(SanitizerKind::Thread)) Fn->addFnAttr(llvm::Attribute::SanitizeThread); Index: tools/clang/lib/CodeGen/CodeGenModule.cpp =================================================================== --- tools/clang/lib/CodeGen/CodeGenModule.cpp +++ tools/clang/lib/CodeGen/CodeGenModule.cpp @@ -1218,8 +1218,9 @@ bool CodeGenModule::isInSanitizerBlacklist(llvm::GlobalVariable *GV, SourceLocation Loc, QualType Ty, StringRef Category) const { - // For now globals can be blacklisted only in ASan. - if (!LangOpts.Sanitize.has(SanitizerKind::Address)) + // For now globals can be blacklisted only in ASan and KASAN. + if (!LangOpts.Sanitize.has(SanitizerKind::Address) && + !LangOpts.Sanitize.has(SanitizerKind::KernelAddress)) return false; const auto &SanitizerBL = getContext().getSanitizerBlacklist(); if (SanitizerBL.isBlacklistedGlobal(GV->getName(), Category)) Index: tools/clang/lib/CodeGen/SanitizerMetadata.cpp =================================================================== --- tools/clang/lib/CodeGen/SanitizerMetadata.cpp +++ tools/clang/lib/CodeGen/SanitizerMetadata.cpp @@ -25,7 +25,8 @@ SourceLocation Loc, StringRef Name, QualType Ty, bool IsDynInit, bool IsBlacklisted) { - if (!CGM.getLangOpts().Sanitize.has(SanitizerKind::Address)) + if (!CGM.getLangOpts().Sanitize.has(SanitizerKind::Address) && + !CGM.getLangOpts().Sanitize.has(SanitizerKind::KernelAddress)) return; IsDynInit &= !CGM.isInSanitizerBlacklist(GV, Loc, Ty, "init"); IsBlacklisted |= CGM.isInSanitizerBlacklist(GV, Loc, Ty); @@ -56,7 +57,8 @@ void SanitizerMetadata::reportGlobalToASan(llvm::GlobalVariable *GV, const VarDecl &D, bool IsDynInit) { - if (!CGM.getLangOpts().Sanitize.has(SanitizerKind::Address)) + if (!CGM.getLangOpts().Sanitize.has(SanitizerKind::Address) && + !CGM.getLangOpts().Sanitize.has(SanitizerKind::KernelAddress)) return; std::string QualName; llvm::raw_string_ostream OS(QualName); @@ -67,7 +69,8 @@ void SanitizerMetadata::disableSanitizerForGlobal(llvm::GlobalVariable *GV) { // For now, just make sure the global is not modified by the ASan // instrumentation. - if (CGM.getLangOpts().Sanitize.has(SanitizerKind::Address)) + if (CGM.getLangOpts().Sanitize.has(SanitizerKind::Address) || + CGM.getLangOpts().Sanitize.has(SanitizerKind::KernelAddress)) reportGlobalToASan(GV, SourceLocation(), "", QualType(), false, true); } Index: tools/clang/lib/Lex/PPMacroExpansion.cpp =================================================================== --- tools/clang/lib/Lex/PPMacroExpansion.cpp +++ tools/clang/lib/Lex/PPMacroExpansion.cpp @@ -1050,6 +1050,8 @@ return llvm::StringSwitch(Feature) .Case("address_sanitizer", LangOpts.Sanitize.has(SanitizerKind::Address)) + .Case("kernel_address_sanitizer", + LangOpts.Sanitize.has(SanitizerKind::KernelAddress)) .Case("attribute_analyzer_noreturn", true) .Case("attribute_availability", true) .Case("attribute_availability_with_message", true)