Index: compiler-rt/trunk/lib/hwasan/hwasan.h =================================================================== --- compiler-rt/trunk/lib/hwasan/hwasan.h +++ compiler-rt/trunk/lib/hwasan/hwasan.h @@ -40,7 +40,7 @@ #define MEM_TO_SHADOW(mem) ((uptr)(mem) >> kShadowScale) #define SHADOW_TO_MEM(shadow) ((uptr)(shadow) << kShadowScale) -#define MEM_IS_APP(mem) true +#define MEM_IS_APP(mem) MemIsApp((uptr)(mem)) // TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address // translation and can be used to store a tag. @@ -69,6 +69,8 @@ extern bool hwasan_init_is_running; extern int hwasan_report_count; +bool MemIsApp(uptr p); + bool ProtectRange(uptr beg, uptr end); bool InitShadow(); char *GetProcSelfMaps(); Index: compiler-rt/trunk/lib/hwasan/hwasan.cc =================================================================== --- compiler-rt/trunk/lib/hwasan/hwasan.cc +++ compiler-rt/trunk/lib/hwasan/hwasan.cc @@ -359,6 +359,10 @@ CheckAddress(p); } +void __hwasan_tag_memory(uptr p, u8 tag, uptr sz) { + TagMemoryAligned(p, sz, tag); +} + #if !SANITIZER_SUPPORTS_WEAK_HOOKS extern "C" { SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE Index: compiler-rt/trunk/lib/hwasan/hwasan_interceptors.cc =================================================================== --- compiler-rt/trunk/lib/hwasan/hwasan_interceptors.cc +++ compiler-rt/trunk/lib/hwasan/hwasan_interceptors.cc @@ -427,6 +427,15 @@ *begin = *end = 0; \ } +#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \ + { \ + COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); \ + if (common_flags()->intercept_intrin && \ + MEM_IS_APP(GetAddressFromPointer(dst))) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ + return REAL(memset)(dst, v, size); \ + } + #include "sanitizer_common/sanitizer_platform_interceptors.h" #include "sanitizer_common/sanitizer_common_interceptors.inc" #include "sanitizer_common/sanitizer_signal_interceptors.inc" Index: compiler-rt/trunk/lib/hwasan/hwasan_interface_internal.h =================================================================== --- compiler-rt/trunk/lib/hwasan/hwasan_interface_internal.h +++ compiler-rt/trunk/lib/hwasan/hwasan_interface_internal.h @@ -83,6 +83,9 @@ SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_store16_noabort(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_tag_memory(uptr p, u8 tag, uptr sz); + // Returns the offset of the first tag mismatch or -1 if the whole range is // good. SANITIZER_INTERFACE_ATTRIBUTE Index: compiler-rt/trunk/lib/hwasan/hwasan_linux.cc =================================================================== --- compiler-rt/trunk/lib/hwasan/hwasan_linux.cc +++ compiler-rt/trunk/lib/hwasan/hwasan_linux.cc @@ -74,20 +74,24 @@ Die(); } +// LowMem covers as much of the first 4GB as possible. +const uptr kLowMemEnd = 1UL<<32; +const uptr kLowShadowEnd = kLowMemEnd >> kShadowScale; +const uptr kLowShadowStart = kLowShadowEnd >> kShadowScale; +static uptr kHighShadowStart; +static uptr kHighShadowEnd; +static uptr kHighMemStart; + bool InitShadow() { const uptr maxVirtualAddress = GetMaxUserVirtualAddress(); - // LowMem covers as much of the first 4GB as possible. - const uptr kLowMemEnd = 1UL<<32; - const uptr kLowShadowEnd = kLowMemEnd >> kShadowScale; - const uptr kLowShadowStart = kLowShadowEnd >> kShadowScale; // HighMem covers the upper part of the address space. - const uptr kHighShadowEnd = (maxVirtualAddress >> kShadowScale) + 1; - const uptr kHighShadowStart = Max(kLowMemEnd, kHighShadowEnd >> kShadowScale); + kHighShadowEnd = (maxVirtualAddress >> kShadowScale) + 1; + kHighShadowStart = Max(kLowMemEnd, kHighShadowEnd >> kShadowScale); CHECK(kHighShadowStart < kHighShadowEnd); - const uptr kHighMemStart = kHighShadowStart << kShadowScale; + kHighMemStart = kHighShadowStart << kShadowScale; CHECK(kHighShadowEnd <= kHighMemStart); if (Verbosity()) { @@ -120,6 +124,11 @@ return true; } +bool MemIsApp(uptr p) { + CHECK(GetTagFromPointer(p) == 0); + return p >= kHighMemStart || (p >= kLowShadowEnd && p < kLowMemEnd); +} + static void HwasanAtExit(void) { if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0)) ReportStats(); Index: compiler-rt/trunk/test/hwasan/TestCases/stack-oob.cc =================================================================== --- compiler-rt/trunk/test/hwasan/TestCases/stack-oob.cc +++ compiler-rt/trunk/test/hwasan/TestCases/stack-oob.cc @@ -0,0 +1,25 @@ +// RUN: %clangxx_hwasan -DSIZE=16 -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s +// RUN: %clangxx_hwasan -DSIZE=64 -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s +// RUN: %clangxx_hwasan -DSIZE=0x1000 -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s + +// REQUIRES: stable-runtime + +#include +#include + +__attribute__((noinline)) +int f() { + char z[SIZE]; + char *volatile p = z; + return p[SIZE]; +} + +int main() { + return f(); + // CHECK: READ of size 1 at + // CHECK: #0 {{.*}} in f{{.*}}stack-oob.cc:14 + + // CHECK: HWAddressSanitizer can not describe address in more detail. + + // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in f +} Index: compiler-rt/trunk/test/hwasan/TestCases/stack-uar.cc =================================================================== --- compiler-rt/trunk/test/hwasan/TestCases/stack-uar.cc +++ compiler-rt/trunk/test/hwasan/TestCases/stack-uar.cc @@ -0,0 +1,23 @@ +// RUN: %clangxx_hwasan -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s + +// REQUIRES: stable-runtime + +#include +#include + +__attribute__((noinline)) +char *f() { + char z[0x1000]; + char *volatile p = z; + return p; +} + +int main() { + return *f(); + // CHECK: READ of size 1 at + // CHECK: #0 {{.*}} in main{{.*}}stack-uar.cc:16 + + // CHECK: HWAddressSanitizer can not describe address in more detail. + + // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in main +} Index: llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.h =================================================================== --- llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.h +++ llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.h @@ -545,6 +545,7 @@ SDValue getAddrLarge(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const; template SDValue getAddr(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const; + SDValue LowerADDROFRETURNADDR(SDValue Op, SelectionDAG &DAG) const; SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const; SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const; SDValue LowerDarwinGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const; Index: llvm/trunk/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp =================================================================== --- llvm/trunk/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ llvm/trunk/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -22,10 +22,7 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DerivedTypes.h" -#include "llvm/IR/MDBuilder.h" -#include "llvm/Support/raw_ostream.h" #include "llvm/IR/Function.h" -#include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/InstVisitor.h" @@ -34,6 +31,7 @@ #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/MDBuilder.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" @@ -41,8 +39,11 @@ #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Instrumentation.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/ModuleUtils.h" +#include "llvm/Transforms/Utils/PromoteMemToReg.h" using namespace llvm; @@ -55,6 +56,7 @@ static const size_t kNumberOfAccessSizes = 5; static const size_t kShadowScale = 4; +static const unsigned kAllocaAlignment = 1U << kShadowScale; static const unsigned kPointerTagShift = 56; static cl::opt ClMemoryAccessCallbackPrefix( @@ -85,6 +87,10 @@ cl::desc("Enable recovery mode (continue-after-error)."), cl::Hidden, cl::init(false)); +static cl::opt ClInstrumentStack("hwasan-instrument-stack", + cl::desc("instrument stack (allocas)"), + cl::Hidden, cl::init(true)); + namespace { /// \brief An instrumentation pass implementing detection of addressability bugs @@ -111,9 +117,15 @@ uint64_t *TypeSize, unsigned *Alignment, Value **MaybeMask); + bool isInterestingAlloca(const AllocaInst &AI); + bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag); + bool instrumentStack(SmallVectorImpl &Allocas, + SmallVectorImpl &RetVec); + private: LLVMContext *C; Type *IntptrTy; + Type *Int8Ty; bool Recover; @@ -121,6 +133,8 @@ Function *HwasanMemoryAccessCallback[2][kNumberOfAccessSizes]; Function *HwasanMemoryAccessCallbackSized[2]; + + Function *HwasanTagMemoryFunc; }; } // end anonymous namespace @@ -150,6 +164,7 @@ C = &(M.getContext()); IRBuilder<> IRB(*C); IntptrTy = IRB.getIntPtrTy(DL); + Int8Ty = IRB.getInt8Ty(); std::tie(HwasanCtorFunction, std::ignore) = createSanitizerCtorAndInitFunctions(M, kHwasanModuleCtorName, @@ -180,6 +195,9 @@ FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false))); } } + + HwasanTagMemoryFunc = checkSanitizerInterfaceFunction(M.getOrInsertFunction( + "__hwasan_tag_memory", IRB.getVoidTy(), IntptrTy, Int8Ty, IntptrTy)); } Value *HWAddressSanitizer::isInterestingMemoryAccess(Instruction *I, @@ -305,6 +323,133 @@ return true; } +static uint64_t getAllocaSizeInBytes(const AllocaInst &AI) { + uint64_t ArraySize = 1; + if (AI.isArrayAllocation()) { + const ConstantInt *CI = dyn_cast(AI.getArraySize()); + assert(CI && "non-constant array size"); + ArraySize = CI->getZExtValue(); + } + Type *Ty = AI.getAllocatedType(); + uint64_t SizeInBytes = AI.getModule()->getDataLayout().getTypeAllocSize(Ty); + return SizeInBytes * ArraySize; +} + +bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, + Value *Tag) { + size_t Size = (getAllocaSizeInBytes(*AI) + kAllocaAlignment - 1) & + ~(kAllocaAlignment - 1); + + Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty()); + if (ClInstrumentWithCalls) { + IRB.CreateCall(HwasanTagMemoryFunc, + {IRB.CreatePointerCast(AI, IntptrTy), JustTag, + ConstantInt::get(IntptrTy, Size)}); + } else { + size_t ShadowSize = Size >> kShadowScale; + Value *ShadowPtr = IRB.CreateIntToPtr( + IRB.CreateLShr(IRB.CreatePointerCast(AI, IntptrTy), kShadowScale), + IRB.getInt8PtrTy()); + // If this memset is not inlined, it will be intercepted in the hwasan + // runtime library. That's OK, because the interceptor skips the checks if + // the address is in the shadow region. + // FIXME: the interceptor is not as fast as real memset. Consider lowering + // llvm.memset right here into either a sequence of stores, or a call to + // hwasan_tag_memory. + IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, /*Align=*/1); + } + return true; +} + +static unsigned RetagMask(unsigned AllocaNo) { + // A list of 8-bit numbers that have at most one run of non-zero bits. + // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these + // masks. + // The list does not include the value 255, which is used for UAR. + static unsigned FastMasks[] = { + 0, 1, 2, 3, 4, 6, 7, 8, 12, 14, 15, 16, 24, + 28, 30, 31, 32, 48, 56, 60, 62, 63, 64, 96, 112, 120, + 124, 126, 127, 128, 192, 224, 240, 248, 252, 254}; + return FastMasks[AllocaNo % (sizeof(FastMasks) / sizeof(FastMasks[0]))]; +} + +bool HWAddressSanitizer::instrumentStack( + SmallVectorImpl &Allocas, + SmallVectorImpl &RetVec) { + Function *F = Allocas[0]->getParent()->getParent(); + Module *M = F->getParent(); + Instruction *InsertPt = &*F->getEntryBlock().begin(); + IRBuilder<> IRB(InsertPt); + + // FIXME: use addressofreturnaddress (but implement it in aarch64 backend + // first). + auto GetStackPointerFn = Intrinsic::getDeclaration(M, Intrinsic::frameaddress); + Value *StackPointer = IRB.CreateCall(GetStackPointerFn, {Constant::getNullValue(IRB.getInt32Ty())}); + + // Extract some entropy from the stack pointer for the tags. + // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ + // between functions). + Value *StackPointerLong = IRB.CreatePointerCast(StackPointer, IntptrTy); + Value *StackTag = + IRB.CreateXor(StackPointerLong, IRB.CreateLShr(StackPointerLong, 20), + "hwasan.stack.base.tag"); + + // Ideally, we want to calculate tagged stack base pointer, and rewrite all + // alloca addresses using that. Unfortunately, offsets are not known yet + // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a + // temp, shift-OR it into each alloca address and xor with the retag mask. + // This generates one extra instruction per alloca use. + for (unsigned N = 0; N < Allocas.size(); ++N) { + auto *AI = Allocas[N]; + IRB.SetInsertPoint(AI->getNextNode()); + + // Replace uses of the alloca with tagged address. + std::string Name = + AI->hasName() ? AI->getName().str() : "alloca." + itostr(N); + Value *Tag = + IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, RetagMask(N))); + Value *AILong = IRB.CreatePointerCast(AI, IntptrTy); + Value *Replacement = IRB.CreateIntToPtr( + IRB.CreateOr(AILong, IRB.CreateShl(Tag, kPointerTagShift)), + AI->getType(), Name + ".hwasan"); + + for (auto UI = AI->use_begin(), UE = AI->use_end(); + UI != UE;) { + Use &U = *UI++; + if (U.getUser() != AILong) + U.set(Replacement); + } + + tagAlloca(IRB, AI, Tag); + + for (auto RI : RetVec) { + IRB.SetInsertPoint(RI); + + // Re-tag alloca memory with the special UAR tag. + Value *Tag = IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, 0xFFU)); + tagAlloca(IRB, AI, Tag); + } + } + + return true; +} + +bool HWAddressSanitizer::isInterestingAlloca(const AllocaInst &AI) { + return (AI.getAllocatedType()->isSized() && + // FIXME: instrument dynamic allocas, too + AI.isStaticAlloca() && + // alloca() may be called with 0 size, ignore it. + getAllocaSizeInBytes(AI) > 0 && + // We are only interested in allocas not promotable to registers. + // Promotable allocas are common under -O0. + !isAllocaPromotable(&AI) && + // inalloca allocas are not treated as static, and we don't want + // dynamic alloca instrumentation for them as well. + !AI.isUsedWithInAlloca() && + // swifterror allocas are register promoted by ISel + !AI.isSwiftError()); +} + bool HWAddressSanitizer::runOnFunction(Function &F) { if (&F == HwasanCtorFunction) return false; @@ -318,8 +463,25 @@ bool Changed = false; SmallVector ToInstrument; + SmallVector AllocasToInstrument; + SmallVector RetVec; for (auto &BB : F) { for (auto &Inst : BB) { + if (ClInstrumentStack) + if (AllocaInst *AI = dyn_cast(&Inst)) { + // Realign all allocas. We don't want small uninteresting allocas to + // hide in instrumented alloca's padding. + if (AI->getAlignment() < kAllocaAlignment) + AI->setAlignment(kAllocaAlignment); + // Instrument some of them. + if (isInterestingAlloca(*AI)) + AllocasToInstrument.push_back(AI); + continue; + } + + if (isa(Inst) || isa(Inst) || isa(Inst)) + RetVec.push_back(&Inst); + Value *MaybeMask = nullptr; bool IsWrite; unsigned Alignment; @@ -331,6 +493,9 @@ } } + if (!AllocasToInstrument.empty()) + Changed |= instrumentStack(AllocasToInstrument, RetVec); + for (auto Inst : ToInstrument) Changed |= instrumentMemAccess(Inst); Index: llvm/trunk/test/Instrumentation/HWAddressSanitizer/alloca.ll =================================================================== --- llvm/trunk/test/Instrumentation/HWAddressSanitizer/alloca.ll +++ llvm/trunk/test/Instrumentation/HWAddressSanitizer/alloca.ll @@ -0,0 +1,45 @@ +; Test basic address sanitizer instrumentation. +; +; RUN: opt < %s -hwasan -S | FileCheck %s + +target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" +target triple = "aarch64--linux-android" + +declare void @use32(i32*) + +define void @test_alloca() sanitize_hwaddress { +; CHECK-LABEL: @test_alloca( +; CHECK: %[[FP:[^ ]*]] = call i8* @llvm.frameaddress(i32 0) +; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %[[FP]] to i64 +; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 20 +; CHECK: %[[BASE_TAG:[^ ]*]] = xor i64 %[[A]], %[[B]] + +; CHECK: %[[X:[^ ]*]] = alloca i32, align 16 +; CHECK: %[[X_TAG:[^ ]*]] = xor i64 %[[BASE_TAG]], 0 +; CHECK: %[[X1:[^ ]*]] = ptrtoint i32* %[[X]] to i64 +; CHECK: %[[C:[^ ]*]] = shl i64 %[[X_TAG]], 56 +; CHECK: %[[D:[^ ]*]] = or i64 %[[X1]], %[[C]] +; CHECK: %[[X_HWASAN:[^ ]*]] = inttoptr i64 %[[D]] to i32* + +; CHECK: %[[X_TAG2:[^ ]*]] = trunc i64 %[[X_TAG]] to i8 +; CHECK: %[[E:[^ ]*]] = ptrtoint i32* %[[X]] to i64 +; CHECK: %[[F:[^ ]*]] = lshr i64 %[[E]], 4 +; CHECK: %[[X_SHADOW:[^ ]*]] = inttoptr i64 %[[F]] to i8* +; CHECK: call void @llvm.memset.p0i8.i64(i8* %[[X_SHADOW]], i8 %[[X_TAG2]], i64 1, i32 1, i1 false) +; CHECK: call void @use32(i32* nonnull %[[X_HWASAN]]) + +; CHECK: %[[X_TAG_UAR:[^ ]*]] = xor i64 %[[BASE_TAG]], 255 +; CHECK: %[[X_TAG_UAR2:[^ ]*]] = trunc i64 %[[X_TAG_UAR]] to i8 +; CHECK: %[[E2:[^ ]*]] = ptrtoint i32* %[[X]] to i64 +; CHECK: %[[F2:[^ ]*]] = lshr i64 %[[E2]], 4 +; CHECK: %[[X_SHADOW2:[^ ]*]] = inttoptr i64 %[[F2]] to i8* +; CHECK: call void @llvm.memset.p0i8.i64(i8* %[[X_SHADOW2]], i8 %[[X_TAG_UAR2]], i64 1, i32 1, i1 false) +; CHECK: ret void + + +entry: + %x = alloca i32, align 4 + call void @use32(i32* nonnull %x) + ret void +} +