Index: clang/include/clang/Basic/Sanitizers.def =================================================================== --- clang/include/clang/Basic/Sanitizers.def +++ clang/include/clang/Basic/Sanitizers.def @@ -44,8 +44,12 @@ // Kernel AddressSanitizer (KASan) SANITIZER("kernel-address", KernelAddress) +// Hardware-assisted AddressSanitizer SANITIZER("hwaddress", HWAddress) +// Kernel Hardware-assisted AddressSanitizer (KHWASan) +SANITIZER("kernel-hwaddress", KernelHWAddress) + // MemorySanitizer SANITIZER("memory", Memory) Index: clang/lib/CodeGen/BackendUtil.cpp =================================================================== --- clang/lib/CodeGen/BackendUtil.cpp +++ clang/lib/CodeGen/BackendUtil.cpp @@ -233,10 +233,9 @@ static void addKernelAddressSanitizerPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { PM.add(createAddressSanitizerFunctionPass( - /*CompileKernel*/ true, - /*Recover*/ true, /*UseAfterScope*/ false)); - PM.add(createAddressSanitizerModulePass(/*CompileKernel*/true, - /*Recover*/true)); + /*CompileKernel*/ true, /*Recover*/ true, /*UseAfterScope*/ false)); + PM.add(createAddressSanitizerModulePass( + /*CompileKernel*/ true, /*Recover*/ true)); } static void addHWAddressSanitizerPasses(const PassManagerBuilder &Builder, @@ -245,7 +244,13 @@ static_cast(Builder); const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts(); bool Recover = CGOpts.SanitizeRecover.has(SanitizerKind::HWAddress); - PM.add(createHWAddressSanitizerPass(Recover)); + PM.add(createHWAddressSanitizerPass(/*CompileKernel*/ false, Recover)); +} + +static void addKernelHWAddressSanitizerPasses(const PassManagerBuilder &Builder, + legacy::PassManagerBase &PM) { + PM.add(createHWAddressSanitizerPass( + /*CompileKernel*/ true, /*Recover*/ true)); } static void addMemorySanitizerPass(const PassManagerBuilder &Builder, @@ -578,6 +583,13 @@ addHWAddressSanitizerPasses); } + if (LangOpts.Sanitize.has(SanitizerKind::KernelHWAddress)) { + PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, + addKernelHWAddressSanitizerPasses); + PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, + addKernelHWAddressSanitizerPasses); + } + if (LangOpts.Sanitize.has(SanitizerKind::Memory)) { PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, addMemorySanitizerPass); Index: clang/lib/CodeGen/CGDeclCXX.cpp =================================================================== --- clang/lib/CodeGen/CGDeclCXX.cpp +++ clang/lib/CodeGen/CGDeclCXX.cpp @@ -331,6 +331,10 @@ !isInSanitizerBlacklist(SanitizerKind::HWAddress, Fn, Loc)) Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress); + if (getLangOpts().Sanitize.has(SanitizerKind::KernelHWAddress) && + !isInSanitizerBlacklist(SanitizerKind::KernelHWAddress, Fn, Loc)) + Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress); + if (getLangOpts().Sanitize.has(SanitizerKind::Thread) && !isInSanitizerBlacklist(SanitizerKind::Thread, Fn, Loc)) Fn->addFnAttr(llvm::Attribute::SanitizeThread); Index: clang/lib/CodeGen/CodeGenFunction.cpp =================================================================== --- clang/lib/CodeGen/CodeGenFunction.cpp +++ clang/lib/CodeGen/CodeGenFunction.cpp @@ -853,13 +853,17 @@ SanOpts.set(SanitizerKind::KernelAddress, false); if (mask & SanitizerKind::KernelAddress) SanOpts.set(SanitizerKind::Address, false); + if (mask & SanitizerKind::HWAddress) + SanOpts.set(SanitizerKind::KernelHWAddress, false); + if (mask & SanitizerKind::KernelHWAddress) + SanOpts.set(SanitizerKind::HWAddress, false); } } // Apply sanitizer attributes to the function. if (SanOpts.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress)) Fn->addFnAttr(llvm::Attribute::SanitizeAddress); - if (SanOpts.hasOneOf(SanitizerKind::HWAddress)) + if (SanOpts.hasOneOf(SanitizerKind::HWAddress | SanitizerKind::KernelHWAddress)) Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress); if (SanOpts.has(SanitizerKind::Thread)) Fn->addFnAttr(llvm::Attribute::SanitizeThread); Index: clang/lib/CodeGen/CodeGenModule.cpp =================================================================== --- clang/lib/CodeGen/CodeGenModule.cpp +++ clang/lib/CodeGen/CodeGenModule.cpp @@ -1815,7 +1815,8 @@ StringRef Category) const { // For now globals can be blacklisted only in ASan and KASan. const SanitizerMask EnabledAsanMask = LangOpts.Sanitize.Mask & - (SanitizerKind::Address | SanitizerKind::KernelAddress | SanitizerKind::HWAddress); + (SanitizerKind::Address | SanitizerKind::KernelAddress | + SanitizerKind::HWAddress | SanitizerKind::KernelHWAddress); if (!EnabledAsanMask) return false; const auto &SanitizerBL = getContext().getSanitizerBlacklist(); Index: clang/lib/CodeGen/SanitizerMetadata.cpp =================================================================== --- clang/lib/CodeGen/SanitizerMetadata.cpp +++ clang/lib/CodeGen/SanitizerMetadata.cpp @@ -27,7 +27,8 @@ bool IsBlacklisted) { if (!CGM.getLangOpts().Sanitize.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress | - SanitizerKind::HWAddress)) + SanitizerKind::HWAddress | + SanitizerKind::KernelHWAddress)) return; IsDynInit &= !CGM.isInSanitizerBlacklist(GV, Loc, Ty, "init"); IsBlacklisted |= CGM.isInSanitizerBlacklist(GV, Loc, Ty); @@ -60,7 +61,8 @@ const VarDecl &D, bool IsDynInit) { if (!CGM.getLangOpts().Sanitize.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress | - SanitizerKind::HWAddress)) + SanitizerKind::HWAddress | + SanitizerKind::KernelHWAddress)) return; std::string QualName; llvm::raw_string_ostream OS(QualName); @@ -79,7 +81,8 @@ // instrumentation. if (CGM.getLangOpts().Sanitize.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress | - SanitizerKind::HWAddress)) + SanitizerKind::HWAddress | + SanitizerKind::KernelHWAddress)) reportGlobalToASan(GV, SourceLocation(), "", QualType(), false, true); } Index: clang/lib/Driver/SanitizerArgs.cpp =================================================================== --- clang/lib/Driver/SanitizerArgs.cpp +++ clang/lib/Driver/SanitizerArgs.cpp @@ -32,9 +32,9 @@ NotAllowedWithMinimalRuntime = Vptr, RequiresPIE = DataFlow | HWAddress | Scudo, NeedsUnwindTables = Address | HWAddress | Thread | Memory | DataFlow, - SupportsCoverage = Address | HWAddress | KernelAddress | Memory | Leak | - Undefined | Integer | Nullability | DataFlow | Fuzzer | - FuzzerNoLink, + SupportsCoverage = Address | HWAddress | KernelAddress | KernelHWAddress | + Memory | Leak | Undefined | Integer | Nullability | + DataFlow | Fuzzer | FuzzerNoLink, RecoverableByDefault = Undefined | Integer | Nullability, Unrecoverable = Unreachable | Return, LegacyFsanitizeRecoverMask = Undefined | Integer, @@ -343,7 +343,9 @@ std::make_pair(Scudo, Address | HWAddress | Leak | Thread | Memory | KernelAddress | Efficiency), std::make_pair(SafeStack, Address | HWAddress | Leak | Thread | Memory | - KernelAddress | Efficiency)}; + KernelAddress | Efficiency), + std::make_pair(KernelHWAddress, Address | HWAddress | Leak | Thread | + Memory | KernelAddress | Efficiency | SafeStack)}; // Enable toolchain specific default sanitizers if not explicitly disabled. SanitizerMask Default = TC.getDefaultSanitizers() & ~AllRemove; Index: clang/lib/Driver/ToolChains/Linux.cpp =================================================================== --- clang/lib/Driver/ToolChains/Linux.cpp +++ clang/lib/Driver/ToolChains/Linux.cpp @@ -879,8 +879,10 @@ Res |= SanitizerKind::Function; if (IsX86_64 || IsMIPS64 || IsAArch64 || IsX86 || IsMIPS || IsArmArch) Res |= SanitizerKind::Scudo; - if (IsX86_64 || IsAArch64) + if (IsX86_64 || IsAArch64) { Res |= SanitizerKind::HWAddress; + Res |= SanitizerKind::KernelHWAddress; + } return Res; } Index: clang/lib/Lex/PPMacroExpansion.cpp =================================================================== --- clang/lib/Lex/PPMacroExpansion.cpp +++ clang/lib/Lex/PPMacroExpansion.cpp @@ -1105,7 +1105,8 @@ LangOpts.Sanitize.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress)) .Case("hwaddress_sanitizer", - LangOpts.Sanitize.hasOneOf(SanitizerKind::HWAddress)) + LangOpts.Sanitize.hasOneOf(SanitizerKind::HWAddress | + SanitizerKind::KernelHWAddress)) .Case("assume_nonnull", true) .Case("attribute_analyzer_noreturn", true) .Case("attribute_availability", true) Index: clang/test/CodeGen/address-safety-attr-flavors.cpp =================================================================== --- clang/test/CodeGen/address-safety-attr-flavors.cpp +++ clang/test/CodeGen/address-safety-attr-flavors.cpp @@ -1,10 +1,12 @@ -// Make sure the sanitize_address attribute is emitted when using ASan, KASan or HWASan. +// Make sure the sanitize_address attribute is emitted when using ASan, KASan, HWASan or KHWASan. // Either __attribute__((no_sanitize("address")) or __attribute__((no_sanitize("kernel-address")) disables both ASan and KASan instrumentation. +// Same for __attribute__((no_sanitize("hwaddress")) and __attribute__((no_sanitize("kernel-hwddress")) and HWASan and KHWASan. /// RUN: %clang_cc1 -triple i386-unknown-linux -disable-O0-optnone -emit-llvm -o - %s | FileCheck -check-prefix=CHECK-NOASAN %s /// RUN: %clang_cc1 -triple i386-unknown-linux -fsanitize=address -disable-O0-optnone -emit-llvm -o - %s | FileCheck -check-prefix=CHECK-ASAN %s /// RUN: %clang_cc1 -triple i386-unknown-linux -fsanitize=kernel-address -disable-O0-optnone -emit-llvm -o - %s | FileCheck -check-prefix=CHECK-KASAN %s /// RUN: %clang_cc1 -triple i386-unknown-linux -fsanitize=hwaddress -disable-O0-optnone -emit-llvm -o - %s | FileCheck -check-prefix=CHECK-HWASAN %s +/// RUN: %clang_cc1 -triple i386-unknown-linux -fsanitize=kernel-hwaddress -disable-O0-optnone -emit-llvm -o - %s | FileCheck -check-prefix=CHECK-KHWASAN %s int HasSanitizeAddress() { return 1; @@ -13,6 +15,7 @@ // CHECK-ASAN: Function Attrs: noinline nounwind sanitize_address // CHECK-KASAN: Function Attrs: noinline nounwind sanitize_address // CHECK-HWASAN: Function Attrs: noinline nounwind sanitize_hwaddress +// CHECK-KHWASAN: Function Attrs: noinline nounwind sanitize_hwaddress __attribute__((no_sanitize("address"))) int NoSanitizeQuoteAddress() { @@ -22,6 +25,7 @@ // CHECK-ASAN: {{Function Attrs: noinline nounwind$}} // CHECK-KASAN: {{Function Attrs: noinline nounwind$}} // CHECK-HWASAN: {{Function Attrs: noinline nounwind sanitize_hwaddress$}} +// CHECK-KHWASAN: {{Function Attrs: noinline nounwind sanitize_hwaddress$}} __attribute__((no_sanitize_address)) int NoSanitizeAddress() { @@ -31,6 +35,7 @@ // CHECK-ASAN: {{Function Attrs: noinline nounwind$}} // CHECK-KASAN: {{Function Attrs: noinline nounwind$}} // CHECK-HWASAN: {{Function Attrs: noinline nounwind sanitize_hwaddress$}} +// CHECK-KHWASAN: {{Function Attrs: noinline nounwind sanitize_hwaddress$}} __attribute__((no_sanitize("kernel-address"))) int NoSanitizeKernelAddress() { @@ -40,6 +45,7 @@ // CHECK-ASAN: {{Function Attrs: noinline nounwind$}} // CHECK-KASAN: {{Function Attrs: noinline nounwind$}} // CHECK-HWASAN: {{Function Attrs: noinline nounwind sanitize_hwaddress$}} +// CHECK-KHWASAN: {{Function Attrs: noinline nounwind sanitize_hwaddress$}} __attribute__((no_sanitize("hwaddress"))) int NoSanitizeHWAddress() { @@ -49,3 +55,14 @@ // CHECK-ASAN: {{Function Attrs: noinline nounwind sanitize_address$}} // CHECK-KASAN: {{Function Attrs: noinline nounwind sanitize_address$}} // CHECK-HWASAN: {{Function Attrs: noinline nounwind$}} +// CHECK-KHWASAN: {{Function Attrs: noinline nounwind$}} + +__attribute__((no_sanitize("kernel-hwaddress"))) +int NoSanitizeKernelHWAddress() { + return 0; +} +// CHECK-NOASAN: {{Function Attrs: noinline nounwind$}} +// CHECK-ASAN: {{Function Attrs: noinline nounwind sanitize_address$}} +// CHECK-KASAN: {{Function Attrs: noinline nounwind sanitize_address$}} +// CHECK-HWASAN: {{Function Attrs: noinline nounwind$}} +// CHECK-KHWASAN: {{Function Attrs: noinline nounwind$}} Index: clang/test/Driver/asan.c =================================================================== --- clang/test/Driver/asan.c +++ clang/test/Driver/asan.c @@ -10,9 +10,14 @@ // RUN: %clang -O1 -target aarch64-unknown-linux -fsanitize=hwaddress %s -S -emit-llvm -o - | FileCheck %s --check-prefix=CHECK-HWASAN // RUN: %clang -O2 -target aarch64-unknown-linux -fsanitize=hwaddress %s -S -emit-llvm -o - | FileCheck %s --check-prefix=CHECK-HWASAN // RUN: %clang -O3 -target aarch64-unknown-linux -fsanitize=hwaddress %s -S -emit-llvm -o - | FileCheck %s --check-prefix=CHECK-HWASAN -// Verify that -fsanitize={address,kernel-address} invoke ASan and KASan instrumentation. +// RUN: %clang -target aarch64-unknown-linux -fsanitize=kernel-hwaddress %s -S -emit-llvm -o - | FileCheck %s --check-prefix=CHECK-KHWASAN +// RUN: %clang -O1 -target aarch64-unknown-linux -fsanitize=kernel-hwaddress %s -S -emit-llvm -o - | FileCheck %s --check-prefix=CHECK-KHWASAN +// RUN: %clang -O2 -target aarch64-unknown-linux -fsanitize=kernel-hwaddress %s -S -emit-llvm -o - | FileCheck %s --check-prefix=CHECK-KHWASAN +// RUN: %clang -O3 -target aarch64-unknown-linux -fsanitize=kernel-hwaddress %s -S -emit-llvm -o - | FileCheck %s --check-prefix=CHECK-KHWASAN +// Verify that -fsanitize={address,hwaddres,kernel-address,kernel-hwaddress} invokes ASan, HWAsan, KASan or KHWASan instrumentation. int foo(int *a) { return *a; } // CHECK-ASAN: __asan_init // CHECK-KASAN: __asan_load4_noabort // CHECK-HWASAN: __hwasan_init +// CHECK-KHWASAN: __hwasan_load4_noabort Index: clang/test/Driver/fsanitize-coverage.c =================================================================== --- clang/test/Driver/fsanitize-coverage.c +++ clang/test/Driver/fsanitize-coverage.c @@ -7,6 +7,7 @@ // RUN: %clang -target x86_64-linux-gnu -fsanitize=address -fsanitize-coverage=func,trace-pc %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANITIZE-COVERAGE-FUNC // RUN: %clang -target x86_64-linux-gnu -fsanitize=kernel-address -fsanitize-coverage=func,trace-pc %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANITIZE-COVERAGE-FUNC // RUN: %clang -target x86_64-linux-gnu -fsanitize=hwaddress -fsanitize-coverage=func,trace-pc %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANITIZE-COVERAGE-FUNC +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kernel-hwaddress -fsanitize-coverage=func,trace-pc %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANITIZE-COVERAGE-FUNC // RUN: %clang -target x86_64-linux-gnu -fsanitize=memory -fsanitize-coverage=func,trace-pc %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANITIZE-COVERAGE-FUNC // RUN: %clang -target x86_64-linux-gnu -fsanitize=leak -fsanitize-coverage=func,trace-pc %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANITIZE-COVERAGE-FUNC // RUN: %clang -target x86_64-linux-gnu -fsanitize=undefined -fsanitize-coverage=func,trace-pc %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANITIZE-COVERAGE-FUNC Index: clang/test/Driver/fsanitize.c =================================================================== --- clang/test/Driver/fsanitize.c +++ clang/test/Driver/fsanitize.c @@ -83,6 +83,27 @@ // RUN: %clang -target x86_64-linux-gnu -fsanitize=kernel-address,address -fno-rtti %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANKA-SANA // CHECK-SANKA-SANA: '-fsanitize=kernel-address' not allowed with '-fsanitize=address' +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kernel-address,leak -pie -fno-rtti %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANKA-SANL +// CHECK-SANKA-SANL: '-fsanitize=kernel-address' not allowed with '-fsanitize=leak' + +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kernel-hwaddress,thread -fno-rtti %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANKHA-SANT +// CHECK-SANKHA-SANT: '-fsanitize=kernel-hwaddress' not allowed with '-fsanitize=thread' + +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kernel-hwaddress,memory -pie -fno-rtti %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANKHA-SANM +// CHECK-SANKHA-SANM: '-fsanitize=kernel-hwaddress' not allowed with '-fsanitize=memory' + +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kernel-hwaddress,address -fno-rtti %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANKHA-SANA +// CHECK-SANKHA-SANA: '-fsanitize=kernel-hwaddress' not allowed with '-fsanitize=address' + +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kernel-hwaddress,leak -pie -fno-rtti %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANKHA-SANL +// CHECK-SANKHA-SANL: '-fsanitize=kernel-hwaddress' not allowed with '-fsanitize=leak' + +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kernel-hwaddress,hwaddress -fno-rtti %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANKHA-SANHA +// CHECK-SANKHA-SANHA: '-fsanitize=kernel-hwaddress' not allowed with '-fsanitize=hwaddress' + +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kernel-hwaddress,kernel-address -pie -fno-rtti %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANKHA-SANKA +// CHECK-SANKHA-SANKA: '-fsanitize=kernel-hwaddress' not allowed with '-fsanitize=kernel-address' + // RUN: %clang -target x86_64-linux-gnu -fsanitize=hwaddress,thread -fno-rtti %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANHA-SANT // CHECK-SANHA-SANT: '-fsanitize=hwaddress' not allowed with '-fsanitize=thread' @@ -92,9 +113,6 @@ // RUN: %clang -target x86_64-linux-gnu -fsanitize=hwaddress,address -fno-rtti %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANHA-SANA // CHECK-SANHA-SANA: '-fsanitize=hwaddress' not allowed with '-fsanitize=address' -// RUN: %clang -target x86_64-linux-gnu -fsanitize=kernel-address,leak -pie -fno-rtti %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANKA-SANL -// CHECK-SANKA-SANL: '-fsanitize=kernel-address' not allowed with '-fsanitize=leak' - // RUN: %clang -target x86_64-linux-gnu -fsanitize=efficiency-cache-frag,address -pie -fno-rtti %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANE-SANA // RUN: %clang -target x86_64-linux-gnu -fsanitize=efficiency-working-set,address -pie -fno-rtti %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SANE-SANA // CHECK-SANE-SANA: '-fsanitize=efficiency-{{.*}}' not allowed with '-fsanitize=address' Index: clang/test/Lexer/has_feature_address_sanitizer.cpp =================================================================== --- clang/test/Lexer/has_feature_address_sanitizer.cpp +++ clang/test/Lexer/has_feature_address_sanitizer.cpp @@ -1,6 +1,7 @@ // RUN: %clang_cc1 -E -fsanitize=address %s -o - | FileCheck --check-prefix=CHECK-ASAN %s // RUN: %clang_cc1 -E -fsanitize=kernel-address %s -o - | FileCheck --check-prefix=CHECK-ASAN %s // RUN: %clang_cc1 -E -fsanitize=hwaddress %s -o - | FileCheck --check-prefix=CHECK-HWASAN %s +// RUN: %clang_cc1 -E -fsanitize=kernel-hwaddress %s -o - | FileCheck --check-prefix=CHECK-HWASAN %s // RUN: %clang_cc1 -E %s -o - | FileCheck --check-prefix=CHECK-NO-ASAN %s #if __has_feature(address_sanitizer) Index: llvm/include/llvm/Transforms/Instrumentation.h =================================================================== --- llvm/include/llvm/Transforms/Instrumentation.h +++ llvm/include/llvm/Transforms/Instrumentation.h @@ -133,7 +133,8 @@ FunctionPass *createMemorySanitizerPass(int TrackOrigins = 0, bool Recover = false); -FunctionPass *createHWAddressSanitizerPass(bool Recover = false); +FunctionPass *createHWAddressSanitizerPass(bool CompileKernel = false, + bool Recover = false); // Insert ThreadSanitizer (race detection) instrumentation FunctionPass *createThreadSanitizerPass(); Index: llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp =================================================================== --- llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -589,9 +589,10 @@ explicit AddressSanitizer(bool CompileKernel = false, bool Recover = false, bool UseAfterScope = false) - : FunctionPass(ID), CompileKernel(CompileKernel || ClEnableKasan), - Recover(Recover || ClRecover), + : FunctionPass(ID), UseAfterScope(UseAfterScope || ClUseAfterScope) { + this->CompileKernel = CompileKernel || ClEnableKasan; + this->Recover = Recover || ClRecover || this->CompileKernel; initializeAddressSanitizerPass(*PassRegistry::getPassRegistry()); } @@ -717,8 +718,7 @@ explicit AddressSanitizerModule(bool CompileKernel = false, bool Recover = false, bool UseGlobalsGC = true) - : ModulePass(ID), CompileKernel(CompileKernel || ClEnableKasan), - Recover(Recover || ClRecover), + : ModulePass(ID), UseGlobalsGC(UseGlobalsGC && ClUseGlobalsGC), // Not a typo: ClWithComdat is almost completely pointless without // ClUseGlobalsGC (because then it only works on modules without @@ -727,7 +727,10 @@ // argument is designed as workaround. Therefore, disable both // ClWithComdat and ClUseGlobalsGC unless the frontend says it's ok to // do globals-gc. - UseCtorComdat(UseGlobalsGC && ClWithComdat) {} + UseCtorComdat(UseGlobalsGC && ClWithComdat) { + this->CompileKernel = CompileKernel || ClEnableKasan; + this->Recover = Recover || ClRecover || this->CompileKernel; + } bool runOnModule(Module &M) override; StringRef getPassName() const override { return "AddressSanitizerModule"; } Index: llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp =================================================================== --- llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -119,8 +119,11 @@ // Pass identification, replacement for typeid. static char ID; - HWAddressSanitizer(bool Recover = false) - : FunctionPass(ID), Recover(Recover || ClRecover) {} + explicit HWAddressSanitizer(bool CompileKernel = false, bool Recover = false) + : FunctionPass(ID) { + this->CompileKernel = CompileKernel || ClEnableKhwasan; + this->Recover = Recover || ClRecover || this->CompileKernel; + } StringRef getPassName() const override { return "HWAddressSanitizer"; } @@ -156,6 +159,7 @@ Type *IntptrTy; Type *Int8Ty; + bool CompileKernel; bool Recover; Function *HwasanCtorFunction; @@ -178,8 +182,9 @@ HWAddressSanitizer, "hwasan", "HWAddressSanitizer: detect memory bugs using tagged addressing.", false, false) -FunctionPass *llvm::createHWAddressSanitizerPass(bool Recover) { - return new HWAddressSanitizer(Recover); +FunctionPass *llvm::createHWAddressSanitizerPass(bool CompileKernel, + bool Recover) { + return new HWAddressSanitizer(CompileKernel, Recover); } /// \brief Module-level initialization. @@ -197,7 +202,7 @@ Int8Ty = IRB.getInt8Ty(); HwasanCtorFunction = nullptr; - if (!ClEnableKhwasan) { + if (!CompileKernel) { std::tie(HwasanCtorFunction, std::ignore) = createSanitizerCtorAndInitFunctions(M, kHwasanModuleCtorName, kHwasanInitName, @@ -335,9 +340,12 @@ IRB.CreateLoad(IRB.CreateIntToPtr(ShadowLong, IRB.getInt8PtrTy())); Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag); - if (ClMatchAllTag != -1) { + int matchAllTag = ClMatchAllTag; + if (ClMatchAllTag == -1 && CompileKernel) + matchAllTag = 0xFF; + if (matchAllTag != -1) { Value *TagNotIgnored = IRB.CreateICmpNE(PtrTag, - ConstantInt::get(PtrTag->getType(), ClMatchAllTag)); + ConstantInt::get(PtrTag->getType(), matchAllTag)); TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored); } @@ -502,7 +510,7 @@ Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag) { Value *TaggedPtrLong; - if (ClEnableKhwasan) { + if (CompileKernel) { // Kernel addresses have 0xFF in the most significant byte. Value *ShiftedTag = IRB.CreateOr( IRB.CreateShl(Tag, kPointerTagShift), @@ -519,7 +527,7 @@ // Remove tag from an address. Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) { Value *UntaggedPtrLong; - if (ClEnableKhwasan) { + if (CompileKernel) { // Kernel addresses have 0xFF in the most significant byte. UntaggedPtrLong = IRB.CreateOr(PtrLong, ConstantInt::get(PtrLong->getType(), 0xFFULL << kPointerTagShift)); Index: llvm/test/Instrumentation/HWAddressSanitizer/X86/kernel.ll =================================================================== --- llvm/test/Instrumentation/HWAddressSanitizer/X86/kernel.ll +++ llvm/test/Instrumentation/HWAddressSanitizer/X86/kernel.ll @@ -1,12 +1,10 @@ -; Test kernel hwasan instrumentation. +; Test KHWASan instrumentation. ; Generic code is covered by ../kernel.ll, only the x86_64 specific code is ; tested here. ; ; RUN: opt < %s -hwasan -hwasan-kernel=1 -S | FileCheck %s --allow-empty --check-prefixes=INIT ; RUN: opt < %s -hwasan -hwasan-kernel=1 -S | FileCheck %s ; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-mapping-offset=12345678 -S | FileCheck %s -; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=0 -S | FileCheck %s --check-prefixes=CHECK,ABORT -; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -S | FileCheck %s --check-prefixes=CHECK,RECOVER target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "x86_64-unknown-linux-gnu" @@ -15,10 +13,8 @@ ; CHECK-LABEL: @test_load( ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64 -; ABORT: call void asm sideeffect "int3\0Anopl 64(%rax)", "{rdi}"(i64 %[[A]]) -; ABORT: unreachable -; RECOVER: call void asm sideeffect "int3\0Anopl 96(%rax)", "{rdi}"(i64 %[[A]]) -; RECOVER: br label +; CHECK: call void asm sideeffect "int3\0Anopl 96(%rax)", "{rdi}"(i64 %[[A]]) +; CHECK: br label ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64 ; CHECK: %[[UNTAGGED:[^ ]*]] = or i64 %[[A]], -72057594037927936 Index: llvm/test/Instrumentation/HWAddressSanitizer/kernel.ll =================================================================== --- llvm/test/Instrumentation/HWAddressSanitizer/kernel.ll +++ llvm/test/Instrumentation/HWAddressSanitizer/kernel.ll @@ -1,11 +1,9 @@ -; Test kernel hwasan instrumentation. +; Test KHWASan instrumentation. +; Enabling -hwasan-kernel=1 enables -hwasan-recover=1 and -hwasan-match-all-tag=0xff. ; ; RUN: opt < %s -hwasan -hwasan-kernel=1 -S | FileCheck %s --allow-empty --check-prefixes=INIT ; RUN: opt < %s -hwasan -hwasan-kernel=1 -S | FileCheck %s --check-prefixes=CHECK,NOOFFSET,NO-MATCH-ALL -; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-mapping-offset=12345678 -S | FileCheck %s --check-prefixes=CHECK,OFFSET,NO-MATCH-ALL -; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=0 -S | FileCheck %s --check-prefixes=CHECK,NOOFFSET,ABORT,NO-MATCH-ALL -; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -S | FileCheck %s --check-prefixes=CHECK,NOOFFSET,RECOVER,NO-MATCH-ALL -; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -hwasan-match-all-tag=0xff -S | FileCheck %s --check-prefixes=CHECK,NOOFFSET,RECOVER,MATCH-ALL +; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-mapping-offset=12345678 -S | FileCheck %s --check-prefixes=CHECK,OFFSET target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-android" @@ -26,16 +24,12 @@ ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]] ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]] -; MATCH-ALL: %[[G:[^ ]*]] = icmp ne i8 %[[PTRTAG]], -1 -; MATCH-ALL: %[[H:[^ ]*]] = and i1 %[[F]], %[[G]] -; MATCH-ALL: br i1 %[[H]], label {{.*}}, label {{.*}}, !prof {{.*}} +; CHECK: %[[G:[^ ]*]] = icmp ne i8 %[[PTRTAG]], -1 +; CHECK: %[[H:[^ ]*]] = and i1 %[[F]], %[[G]] +; CHECK: br i1 %[[H]], label {{.*}}, label {{.*}}, !prof {{.*}} -; NO-MATCH-ALL: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}} - -; ABORT: call void asm sideeffect "brk #2304", "{x0}"(i64 %[[A]]) -; ABORT: unreachable -; RECOVER: call void asm sideeffect "brk #2336", "{x0}"(i64 %[[A]]) -; RECOVER: br label +; CHECK: call void asm sideeffect "brk #2336", "{x0}"(i64 %[[A]]) +; CHECK: br label ; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4 ; CHECK: ret i8 %[[G]]