diff --git a/compiler-rt/lib/hwasan/hwasan.cpp b/compiler-rt/lib/hwasan/hwasan.cpp --- a/compiler-rt/lib/hwasan/hwasan.cpp +++ b/compiler-rt/lib/hwasan/hwasan.cpp @@ -576,6 +576,12 @@ return t->GenerateRandomTag(); } +void __hwasan_record_frame_record(u64 frame_record_info) { + Thread *t = GetCurrentThread(); + if (t) + t->stack_allocations()->push(frame_record_info); +} + #if !SANITIZER_SUPPORTS_WEAK_HOOKS extern "C" { SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE diff --git a/compiler-rt/lib/hwasan/hwasan_interface_internal.h b/compiler-rt/lib/hwasan/hwasan_interface_internal.h --- a/compiler-rt/lib/hwasan/hwasan_interface_internal.h +++ b/compiler-rt/lib/hwasan/hwasan_interface_internal.h @@ -168,6 +168,9 @@ SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_print_memory_usage(); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_record_frame_record(uint64_t frame_record_info); + SANITIZER_INTERFACE_ATTRIBUTE void *__hwasan_memcpy(void *dst, const void *src, uptr size); SANITIZER_INTERFACE_ATTRIBUTE diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -185,6 +185,13 @@ cl::desc("Record stack frames with tagged allocations " "in a thread-local ring buffer"), cl::Hidden, cl::init(true)); + +static cl::opt ClRecordStackHistoryWithCalls( + "hwasan-record-stack-history-with-calls", + cl::desc("If recording stack frames with tagged allocations, record them " + "with runtime library calls"), + cl::Hidden, cl::init(false)); + static cl::opt ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics", cl::desc("instrument memory intrinsics"), @@ -315,6 +322,9 @@ void instrumentPersonalityFunctions(); + Value *getPC(IRBuilder<> &IRB); + Value *getSP(IRBuilder<> &IRB); + private: LLVMContext *C; Module &M; @@ -377,6 +387,7 @@ FunctionCallee HwasanTagMemoryFunc; FunctionCallee HwasanGenerateTagFunc; + FunctionCallee HwasanRecordFrameRecordFunc; Constant *ShadowGlobal; @@ -635,6 +646,9 @@ HwasanGenerateTagFunc = M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty); + HwasanRecordFrameRecordFunc = M.getOrInsertFunction( + "__hwasan_record_frame_record", IRB.getVoidTy(), Int64Ty); + ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow", ArrayType::get(IRB.getInt8Ty(), 0)); @@ -1123,6 +1137,30 @@ return nullptr; } +Value *HWAddressSanitizer::getPC(IRBuilder<> &IRB) { + Value *PC; + if (TargetTriple.getArch() == Triple::aarch64) { + PC = readRegister(IRB, "pc"); + } else { + Function *F = IRB.GetInsertBlock()->getParent(); + PC = IRB.CreatePtrToInt(F, IntptrTy); + } + return PC; +} + +Value *HWAddressSanitizer::getSP(IRBuilder<> &IRB) { + Function *F = IRB.GetInsertBlock()->getParent(); + Module *M = F->getParent(); + auto GetStackPointerFn = Intrinsic::getDeclaration( + M, Intrinsic::frameaddress, + IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace())); + Value *SP = IRB.CreatePtrToInt( + IRB.CreateCall(GetStackPointerFn, + {Constant::getNullValue(IRB.getInt32Ty())}), + IntptrTy); + return SP; +} + void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) { if (!Mapping.InTls) ShadowBase = getShadowNonTls(IRB); @@ -1132,32 +1170,12 @@ if (!WithFrameRecord && ShadowBase) return; - Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy); - assert(SlotPtr); - - Value *ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr); - // Extract the address field from ThreadLong. Unnecessary on AArch64 with TBI. - Value *ThreadLongMaybeUntagged = - TargetTriple.isAArch64() ? ThreadLong : untagPointer(IRB, ThreadLong); - + Value *ThreadLongMaybeUntagged = nullptr; if (WithFrameRecord) { - Function *F = IRB.GetInsertBlock()->getParent(); - StackBaseTag = IRB.CreateAShr(ThreadLong, 3); - // Prepare ring buffer data. - Value *PC; - if (TargetTriple.getArch() == Triple::aarch64) - PC = readRegister(IRB, "pc"); - else - PC = IRB.CreatePtrToInt(F, IntptrTy); - Module *M = F->getParent(); - auto GetStackPointerFn = Intrinsic::getDeclaration( - M, Intrinsic::frameaddress, - IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace())); - Value *SP = IRB.CreatePtrToInt( - IRB.CreateCall(GetStackPointerFn, - {Constant::getNullValue(IRB.getInt32Ty())}), - IntptrTy); + Value *PC = getPC(IRB); + Value *SP = getSP(IRB); + // Mix SP and PC. // Assumptions: // PC is 0x0000PPPPPPPPPPPP (48 bits are meaningful, others are zero) @@ -1165,28 +1183,48 @@ // We only really need ~20 lower non-zero bits (SSSS), so we mix like this: // 0xSSSSPPPPPPPPPPPP SP = IRB.CreateShl(SP, 44); + Value *FrameRecordInfo = IRB.CreateOr(PC, SP); - // Store data to ring buffer. - Value *RecordPtr = - IRB.CreateIntToPtr(ThreadLongMaybeUntagged, IntptrTy->getPointerTo(0)); - IRB.CreateStore(IRB.CreateOr(PC, SP), RecordPtr); - - // Update the ring buffer. Top byte of ThreadLong defines the size of the - // buffer in pages, it must be a power of two, and the start of the buffer - // must be aligned by twice that much. Therefore wrap around of the ring - // buffer is simply Addr &= ~((ThreadLong >> 56) << 12). - // The use of AShr instead of LShr is due to - // https://bugs.llvm.org/show_bug.cgi?id=39030 - // Runtime library makes sure not to use the highest bit. - Value *WrapMask = IRB.CreateXor( - IRB.CreateShl(IRB.CreateAShr(ThreadLong, 56), 12, "", true, true), - ConstantInt::get(IntptrTy, (uint64_t)-1)); - Value *ThreadLongNew = IRB.CreateAnd( - IRB.CreateAdd(ThreadLong, ConstantInt::get(IntptrTy, 8)), WrapMask); - IRB.CreateStore(ThreadLongNew, SlotPtr); + if (ClRecordStackHistoryWithCalls) { + // Emit a runtime call into hwasan rather than emitting instructions for + // recording stack history. + IRB.CreateCall(HwasanRecordFrameRecordFunc, {FrameRecordInfo}); + } else { + Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy); + assert(SlotPtr); + + Value *ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr); + // Extract the address field from ThreadLong. Unnecessary on AArch64 with + // TBI. + ThreadLongMaybeUntagged = + TargetTriple.isAArch64() ? ThreadLong : untagPointer(IRB, ThreadLong); + + if (WithFrameRecord) { + StackBaseTag = IRB.CreateAShr(ThreadLong, 3); + + // Store data to ring buffer. + Value *RecordPtr = IRB.CreateIntToPtr(ThreadLongMaybeUntagged, + IntptrTy->getPointerTo(0)); + IRB.CreateStore(FrameRecordInfo, RecordPtr); + + // Update the ring buffer. Top byte of ThreadLong defines the size of + // the buffer in pages, it must be a power of two, and the start of the + // buffer must be aligned by twice that much. Therefore wrap around of + // the ring buffer is simply Addr &= ~((ThreadLong >> 56) << 12). The + // use of AShr instead of LShr is due to + // https://bugs.llvm.org/show_bug.cgi?id=39030 + // Runtime library makes sure not to use the highest bit. + Value *WrapMask = IRB.CreateXor( + IRB.CreateShl(IRB.CreateAShr(ThreadLong, 56), 12, "", true, true), + ConstantInt::get(IntptrTy, (uint64_t)-1)); + Value *ThreadLongNew = IRB.CreateAnd( + IRB.CreateAdd(ThreadLong, ConstantInt::get(IntptrTy, 8)), WrapMask); + IRB.CreateStore(ThreadLongNew, SlotPtr); + } + } } - if (!ShadowBase) { + if (!ShadowBase && ThreadLongMaybeUntagged) { // Get shadow base address by aligning RecordPtr up. // Note: this is not correct if the pointer is already aligned. // Runtime library will make sure this never happens.