[hwasan] LLVM-level flags for linux kernel-compatible hwasan instrumentation.

[eugenis]

Summary

-hwasan-mapping-offset defines the non-zero shadow base address.
-hwasan-kernel disables calls to __hwasan_init in module constructors.
Unlike ASan, -hwasan-kernel does not force callback instrumentation.
This is controlled separately with -hwasan-instrument-with-calls.

[kcc]

LGTM

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

static cl::opt<bool> ClInstrumentStack("hwasan-instrument-stack",
                                        cl::desc("instrument stack (allocas)"),
                                        cl::Hidden, cl::init(true));
 
 static cl::opt<bool> ClGenerateTagsWithCalls(
     "hwasan-generate-tags-with-calls",
     cl::desc("generate new tags with runtime library calls"), cl::Hidden,
     cl::init(false));
 
+static cl::opt<unsigned long long> ClMappingOffset(
+    "hwasan-mapping-offset",
+    cl::desc("offset of hwasan shadow mapping [EXPERIMENTAL]"), cl::Hidden,
+    cl::init(0));
+
+static cl::opt<bool> ClEnableKhwasan(
+    "hwasan-kernel", cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
+    cl::Hidden, cl::init(false));
+
 namespace {
 
 /// \brief An instrumentation pass implementing detection of addressability bugs
 /// using tagged pointers.
 class HWAddressSanitizer : public FunctionPass {
 public:
   // Pass identification, replacement for typeid.
   static char ID;
bool HWAddressSanitizer::doInitialization(Module &M) {
 
   Triple TargetTriple(M.getTargetTriple());
 
   C = &(M.getContext());
   IRBuilder<> IRB(*C);
   IntptrTy = IRB.getIntPtrTy(DL);
   Int8Ty = IRB.getInt8Ty();
 
+  if (!ClEnableKhwasan) {
     std::tie(HwasanCtorFunction, std::ignore) =
         createSanitizerCtorAndInitFunctions(M, kHwasanModuleCtorName,
                                             kHwasanInitName,
                                             /*InitArgTypes=*/{},
                                             /*InitArgs=*/{});
     appendToGlobalCtors(M, HwasanCtorFunction, 0);
+  }
   return true;
 }
 
 void HWAddressSanitizer::initializeCallbacks(Module &M) {
   IRBuilder<> IRB(*C);
   for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
     const std::string TypeStr = AccessIsWrite ? "store" : "load";
     const std::string EndingStr = Recover ? "_noabort" : "";
void HWAddressSanitizer::instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
                                                    unsigned AccessSizeIndex,
                                                    Instruction *InsertBefore) {
   IRBuilder<> IRB(InsertBefore);
   Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift), IRB.getInt8Ty());
   Value *AddrLong =
       IRB.CreateAnd(PtrLong, ConstantInt::get(PtrLong->getType(),
                                               ~(0xFFULL << kPointerTagShift)));
   Value *ShadowLong = IRB.CreateLShr(AddrLong, kShadowScale);
-  Value *MemTag = IRB.CreateLoad(IRB.CreateIntToPtr(ShadowLong, IRB.getInt8PtrTy()));
+  if (ClMappingOffset)
+    ShadowLong = IRB.CreateAdd(
+        ShadowLong, ConstantInt::get(PtrLong->getType(), ClMappingOffset,
+                                     /*isSigned=*/false));
+  Value *MemTag =
+      IRB.CreateLoad(IRB.CreateIntToPtr(ShadowLong, IRB.getInt8PtrTy()));
   Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
 
   TerminatorInst *CheckTerm =
       SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, !Recover,
                                 MDBuilder(*C).createBranchWeights(1, 100000));
 
   IRB.SetInsertPoint(CheckTerm);
   // The signal handler will find the data address in x0.

llvm/test/Instrumentation/HWAddressSanitizer/kernel.ll

+; Test kernel hwasan instrumentation.
+;
+; RUN: opt < %s -hwasan -hwasan-kernel=1 -S | FileCheck %s --allow-empty --check-prefixes=KERNEL
+; RUN: opt < %s -hwasan -hwasan-mapping-offset=12345678 -S | FileCheck %s  --check-prefixes=OFFSET
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-android"
+
+define i8 @test_load(i8* %a) sanitize_hwaddress {
+; OFFSET-LABEL: @test_load(
+; OFFSET: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
+; OFFSET: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; OFFSET: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; OFFSET: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; OFFSET: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; OFFSET: %[[D1:[^ ]*]] = add i64 %[[D]], 12345678
+; OFFSET: %[[E:[^ ]*]] = inttoptr i64 %[[D1]] to i8*
+; OFFSET: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; OFFSET: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; OFFSET: br i1 %[[F]],
+
+entry:
+  %b = load i8, i8* %a, align 4
+  ret i8 %b
+}
+
+; KERNEL-NOT: call void @__hwasan_init