Index: include/llvm/Bitcode/LLVMBitCodes.h =================================================================== --- include/llvm/Bitcode/LLVMBitCodes.h +++ include/llvm/Bitcode/LLVMBitCodes.h @@ -516,7 +516,8 @@ ATTR_KIND_INACCESSIBLEMEM_ONLY = 49, ATTR_KIND_INACCESSIBLEMEM_OR_ARGMEMONLY = 50, ATTR_KIND_ALLOC_SIZE = 51, - ATTR_KIND_WRITEONLY = 52 + ATTR_KIND_WRITEONLY = 52, + ATTR_KIND_RUNTIME_INIT = 53 }; enum ComdatSelectionKindCodes { Index: include/llvm/IR/Attributes.td =================================================================== --- include/llvm/IR/Attributes.td +++ include/llvm/IR/Attributes.td @@ -127,6 +127,12 @@ /// Function can return twice. def ReturnsTwice : EnumAttr<"returns_twice">; +/// Function may be used during initialization of dynamic linker or language +/// runtime. SafeStack will compile function to be usable both before and after +/// the default (e.g. thread-local) unsafe stack pointer is configured, assuming +/// that appropriate runtime support is available. +def RuntimeInit : EnumAttr<"runtime_init">; + /// Safe Stack protection. def SafeStack : EnumAttr<"safestack">; Index: include/llvm/Target/TargetLowering.h =================================================================== --- include/llvm/Target/TargetLowering.h +++ include/llvm/Target/TargetLowering.h @@ -1122,6 +1122,9 @@ virtual Value *getSSPStackGuardCheck(const Module &M) const; protected: + GlobalVariable *getOrCreateStackPtrVar(Module &M, const char *Name, + GlobalValue::ThreadLocalMode TlsModel) const; + Value *getDefaultSafeStackPointerLocation(IRBuilder<> &IRB, bool UseTLS) const; @@ -1129,6 +1132,10 @@ /// Returns the target-specific address of the unsafe stack pointer. virtual Value *getSafeStackPointerLocation(IRBuilder<> &IRB) const; + /// Returns the address of the unsafe stack pointer used during runtime + /// initialization. + Value *getSafeStackInitPointerLocation(IRBuilder<> &IRB) const; + /// Returns true if a cast between SrcAS and DestAS is a noop. virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const { return false; Index: lib/AsmParser/LLLexer.cpp =================================================================== --- lib/AsmParser/LLLexer.cpp +++ lib/AsmParser/LLLexer.cpp @@ -643,6 +643,7 @@ KEYWORD(readonly); KEYWORD(returned); KEYWORD(returns_twice); + KEYWORD(runtime_init); KEYWORD(signext); KEYWORD(sret); KEYWORD(ssp); Index: lib/AsmParser/LLParser.cpp =================================================================== --- lib/AsmParser/LLParser.cpp +++ lib/AsmParser/LLParser.cpp @@ -1107,6 +1107,7 @@ case lltok::kw_readonly: B.addAttribute(Attribute::ReadOnly); break; case lltok::kw_returns_twice: B.addAttribute(Attribute::ReturnsTwice); break; + case lltok::kw_runtime_init: B.addAttribute(Attribute::RuntimeInit); break; case lltok::kw_ssp: B.addAttribute(Attribute::StackProtect); break; case lltok::kw_sspreq: B.addAttribute(Attribute::StackProtectReq); break; case lltok::kw_sspstrong: @@ -1436,6 +1437,7 @@ case lltok::kw_optnone: case lltok::kw_optsize: case lltok::kw_returns_twice: + case lltok::kw_runtime_init: case lltok::kw_sanitize_address: case lltok::kw_sanitize_memory: case lltok::kw_sanitize_thread: @@ -1527,6 +1529,7 @@ case lltok::kw_optnone: case lltok::kw_optsize: case lltok::kw_returns_twice: + case lltok::kw_runtime_init: case lltok::kw_sanitize_address: case lltok::kw_sanitize_memory: case lltok::kw_sanitize_thread: Index: lib/AsmParser/LLToken.h =================================================================== --- lib/AsmParser/LLToken.h +++ lib/AsmParser/LLToken.h @@ -195,6 +195,7 @@ kw_readonly, kw_returned, kw_returns_twice, + kw_runtime_init, kw_signext, kw_ssp, kw_sspreq, Index: lib/Bitcode/Reader/BitcodeReader.cpp =================================================================== --- lib/Bitcode/Reader/BitcodeReader.cpp +++ lib/Bitcode/Reader/BitcodeReader.cpp @@ -1556,6 +1556,8 @@ return Attribute::Returned; case bitc::ATTR_KIND_RETURNS_TWICE: return Attribute::ReturnsTwice; + case bitc::ATTR_KIND_RUNTIME_INIT: + return Attribute::RuntimeInit; case bitc::ATTR_KIND_S_EXT: return Attribute::SExt; case bitc::ATTR_KIND_STACK_ALIGNMENT: Index: lib/Bitcode/Writer/BitcodeWriter.cpp =================================================================== --- lib/Bitcode/Writer/BitcodeWriter.cpp +++ lib/Bitcode/Writer/BitcodeWriter.cpp @@ -688,6 +688,8 @@ return bitc::ATTR_KIND_RETURNED; case Attribute::ReturnsTwice: return bitc::ATTR_KIND_RETURNS_TWICE; + case Attribute::RuntimeInit: + return bitc::ATTR_KIND_RUNTIME_INIT; case Attribute::SExt: return bitc::ATTR_KIND_S_EXT; case Attribute::StackAlignment: Index: lib/CodeGen/SafeStack.cpp =================================================================== --- lib/CodeGen/SafeStack.cpp +++ lib/CodeGen/SafeStack.cpp @@ -103,7 +103,13 @@ Type *Int32Ty; Type *Int8Ty; + /// Pointer to default unsafe stack pointer Value *UnsafeStackPtr = nullptr; + /// Pointer to single-threaded unsafe stack pointer to be used during program + /// initialization + Value *UnsafeStackPtrInit = nullptr; + /// Load of init unsafe stack pointer at beginning of function + Value *UnsafeStackPtrInitLoad = nullptr; /// Unsafe stack alignment. Each stack frame must ensure that the stack is /// aligned to this value. We need to re-align the unsafe stack if the @@ -113,6 +119,16 @@ /// might expect to appear on the stack on most common targets. enum { StackAlignment = 16 }; + /// \brief Load the unsafe stack pointer. + /// + /// \param Top Set to true if this load is at the beginning/top of the + /// function. + Instruction *loadUnsafeStackPtr(IRBuilder<> &IRB, bool Top = false); + + /// \brief Store the updated value of the unsafe stack pointer. + void storeUnsafeStackPtr(IRBuilder<> &IRB, Value *UpdatedUSP, + Value *NameSource = nullptr); + /// \brief Return the value of the stack canary. Value *getStackGuard(IRBuilder<> &IRB, Function &F); @@ -159,8 +175,7 @@ /// \brief Replace all allocas in \p DynamicAllocas with code to allocate /// space dynamically on the unsafe stack and store the dynamic unsafe stack /// top to \p DynamicTop if non-null. - void moveDynamicAllocasToUnsafeStack(Function &F, Value *UnsafeStackPtr, - AllocaInst *DynamicTop, + void moveDynamicAllocasToUnsafeStack(Function &F, AllocaInst *DynamicTop, ArrayRef DynamicAllocas); bool IsSafeStackAlloca(const Value *AllocaPtr, uint64_t AllocaSize); @@ -347,6 +362,60 @@ return true; } +Instruction *SafeStack::loadUnsafeStackPtr(IRBuilder<> &IRB, bool Top) { + if (UnsafeStackPtrInit == nullptr) + return IRB.CreateLoad(UnsafeStackPtr, false, "unsafe_stack_ptr"); + + auto IsInit = IRB.CreateIsNotNull(UnsafeStackPtrInitLoad); + + auto PHI = IRB.CreatePHI(StackPtrTy, 2); + TerminatorInst *ThenTerm, *ElseTerm; + SplitBlockAndInsertIfThenElse(IsInit, PHI, &ThenTerm, &ElseTerm); + IRBuilder<> InitIRB(ThenTerm); + IRBuilder<> DefaultIRB(ElseTerm); + auto Tail = ThenTerm->getSuccessor(0); + IRB.SetInsertPoint(Tail, Tail->getFirstInsertionPt()); + + auto InitLoad = Top? + // There is no need to load this twice at the top of the function. + UnsafeStackPtrInitLoad : + InitIRB.CreateLoad(UnsafeStackPtrInit, false, "unsafe_stack_ptr_init"); + auto DefaultLoad = DefaultIRB.CreateLoad(UnsafeStackPtr, false, "unsafe_stack_ptr"); + + PHI->addIncoming(InitLoad, ThenTerm->getParent()); + PHI->addIncoming(DefaultLoad, ElseTerm->getParent()); + + return PHI; +} + +void SafeStack::storeUnsafeStackPtr(IRBuilder<> &IRB, Value *UpdatedUSP, + Value *NameSource) { + if (UnsafeStackPtrInit == nullptr) { + auto DefaultStore = IRB.CreateStore(UpdatedUSP, UnsafeStackPtr); + if (NameSource != nullptr) + DefaultStore->takeName(NameSource); + return; + } + + auto IsInit = IRB.CreateIsNotNull(UnsafeStackPtrInitLoad); + TerminatorInst *ThenTerm, *ElseTerm; + SplitBlockAndInsertIfThenElse(IsInit, &*IRB.GetInsertPoint(), + &ThenTerm, &ElseTerm); + IRBuilder<> InitIRB(ThenTerm); + IRBuilder<> DefaultIRB(ElseTerm); + auto Tail = ThenTerm->getSuccessor(0); + IRB.SetInsertPoint(Tail, Tail->getFirstInsertionPt()); + + auto InitStore = InitIRB.CreateStore(UpdatedUSP, UnsafeStackPtrInit); + auto DefaultStore = DefaultIRB.CreateStore(UpdatedUSP, UnsafeStackPtr); + + if (NameSource == nullptr) + return; + + InitStore->takeName(NameSource); + DefaultStore->takeName(NameSource); +} + Value *SafeStack::getStackGuard(IRBuilder<> &IRB, Function &F) { Value *StackGuardVar = TL->getIRStackGuard(IRB); if (!StackGuardVar) @@ -434,7 +503,7 @@ IRB.SetInsertPoint(I->getNextNode()); Value *CurrentTop = DynamicTop ? IRB.CreateLoad(DynamicTop) : StaticTop; - IRB.CreateStore(CurrentTop, UnsafeStackPtr); + storeUnsafeStackPtr(IRB, CurrentTop); } return DynamicTop; @@ -617,12 +686,12 @@ Value *StaticTop = IRB.CreateGEP(BasePointer, ConstantInt::get(Int32Ty, -FrameSize), "unsafe_stack_static_top"); - IRB.CreateStore(StaticTop, UnsafeStackPtr); + storeUnsafeStackPtr(IRB, StaticTop); return StaticTop; } void SafeStack::moveDynamicAllocasToUnsafeStack( - Function &F, Value *UnsafeStackPtr, AllocaInst *DynamicTop, + Function &F, AllocaInst *DynamicTop, ArrayRef DynamicAllocas) { DIBuilder DIB(*F.getParent()); @@ -638,7 +707,7 @@ uint64_t TySize = DL->getTypeAllocSize(Ty); Value *Size = IRB.CreateMul(ArraySize, ConstantInt::get(IntPtrTy, TySize)); - Value *SP = IRB.CreatePtrToInt(IRB.CreateLoad(UnsafeStackPtr), IntPtrTy); + Value *SP = IRB.CreatePtrToInt(loadUnsafeStackPtr(IRB), IntPtrTy); SP = IRB.CreateSub(SP, Size); // Align the SP value to satisfy the AllocaInst, type and stack alignments. @@ -652,7 +721,7 @@ StackPtrTy); // Save the stack pointer. - IRB.CreateStore(NewTop, UnsafeStackPtr); + storeUnsafeStackPtr(IRB, NewTop); if (DynamicTop) IRB.CreateStore(NewTop, DynamicTop); @@ -667,25 +736,32 @@ if (!DynamicAllocas.empty()) { // Now go through the instructions again, replacing stacksave/stackrestore. + SmallPtrSet StackSaves, StackRestores; for (inst_iterator It = inst_begin(&F), Ie = inst_end(&F); It != Ie;) { Instruction *I = &*(It++); auto II = dyn_cast(I); if (!II) continue; - if (II->getIntrinsicID() == Intrinsic::stacksave) { - IRBuilder<> IRB(II); - Instruction *LI = IRB.CreateLoad(UnsafeStackPtr); - LI->takeName(II); - II->replaceAllUsesWith(LI); - II->eraseFromParent(); - } else if (II->getIntrinsicID() == Intrinsic::stackrestore) { - IRBuilder<> IRB(II); - Instruction *SI = IRB.CreateStore(II->getArgOperand(0), UnsafeStackPtr); - SI->takeName(II); - assert(II->use_empty()); - II->eraseFromParent(); - } + if (II->getIntrinsicID() == Intrinsic::stacksave) + StackSaves.insert(II); + else if (II->getIntrinsicID() == Intrinsic::stackrestore) + StackRestores.insert(II); + } + + for (auto II : StackSaves) { + IRBuilder<> IRB(II); + Instruction *LI = loadUnsafeStackPtr(IRB); + LI->takeName(II); + II->replaceAllUsesWith(LI); + II->eraseFromParent(); + } + + for (auto II : StackRestores) { + IRBuilder<> IRB(II); + storeUnsafeStackPtr(IRB, II->getArgOperand(0), II); + assert(II->use_empty()); + II->eraseFromParent(); } } } @@ -742,11 +818,24 @@ IRBuilder<> IRB(&F.front(), F.begin()->getFirstInsertionPt()); UnsafeStackPtr = TL->getSafeStackPointerLocation(IRB); + UnsafeStackPtrInit = nullptr; + if (F.hasFnAttribute(Attribute::RuntimeInit)) { + // Initialize UnsafeStackPtrInit with a value representing a pointer to the + // single-threaded unsafe stack pointer to be used during program + // initialization. + UnsafeStackPtrInit = TL->getSafeStackInitPointerLocation(IRB); + // Initialize UnsafeStackPtrInitLoad with an instruction that loads + // UnsafeStackPtrInit. That loaded value is initially used to select + // between that init pointer and the default pointer, depending on whether + // the init pointer is null. That loaded value is also used to determine + // where to later store the updated unsafe stack pointer. + UnsafeStackPtrInitLoad = + IRB.CreateLoad(UnsafeStackPtrInit, false, "unsafe_stack_ptr_init"); + } // Load the current stack pointer (we'll also use it as a base pointer). // FIXME: use a dedicated register for it ? - Instruction *BasePointer = - IRB.CreateLoad(UnsafeStackPtr, false, "unsafe_stack_ptr"); + Instruction *BasePointer = loadUnsafeStackPtr(IRB, true); assert(BasePointer->getType() == StackPtrTy); AllocaInst *StackGuardSlot = nullptr; @@ -780,13 +869,12 @@ IRB, F, StackRestorePoints, StaticTop, !DynamicAllocas.empty()); // Handle dynamic allocas. - moveDynamicAllocasToUnsafeStack(F, UnsafeStackPtr, DynamicTop, - DynamicAllocas); + moveDynamicAllocasToUnsafeStack(F, DynamicTop, DynamicAllocas); // Restore the unsafe stack pointer before each return. for (ReturnInst *RI : Returns) { IRB.SetInsertPoint(RI); - IRB.CreateStore(BasePointer, UnsafeStackPtr); + storeUnsafeStackPtr(IRB, BasePointer); } DEBUG(dbgs() << "[SafeStack] safestack applied\n"); Index: lib/CodeGen/TargetLoweringBase.cpp =================================================================== --- lib/CodeGen/TargetLoweringBase.cpp +++ lib/CodeGen/TargetLoweringBase.cpp @@ -1767,36 +1767,43 @@ } } +GlobalVariable *TargetLoweringBase::getOrCreateStackPtrVar(Module &M, + const char *Name, + GlobalValue::ThreadLocalMode TlsModel) const { + GlobalVariable *V = dyn_cast_or_null(M.getNamedValue(Name)); + Type *StackPtrTy = Type::getInt8PtrTy(M.getContext()); + if (!V) { + // The global variable is not defined yet, define it ourselves. + V = new GlobalVariable( + M, StackPtrTy, false, GlobalValue::ExternalLinkage, nullptr, Name, + nullptr, TlsModel); + } else { + // The variable exists, check its type and attributes. + if (V->getValueType() != StackPtrTy) + report_fatal_error(Twine(Name) + " must have void* type"); + if (TlsModel == GlobalValue::NotThreadLocal) { + if (V->isThreadLocal()) + report_fatal_error(Twine(Name) + " must not be thread-local"); + } else + if (!V->isThreadLocal()) + report_fatal_error(Twine(Name) + " must be thread-local"); + } + + return V; +} + Value *TargetLoweringBase::getDefaultSafeStackPointerLocation(IRBuilder<> &IRB, bool UseTLS) const { // compiler-rt provides a variable with a magic name. Targets that do not // link with compiler-rt may also provide such a variable. Module *M = IRB.GetInsertBlock()->getParent()->getParent(); - const char *UnsafeStackPtrVar = "__safestack_unsafe_stack_ptr"; - auto UnsafeStackPtr = - dyn_cast_or_null(M->getNamedValue(UnsafeStackPtrVar)); - - Type *StackPtrTy = Type::getInt8PtrTy(M->getContext()); - - if (!UnsafeStackPtr) { - auto TLSModel = UseTLS ? - GlobalValue::InitialExecTLSModel : - GlobalValue::NotThreadLocal; - // The global variable is not defined yet, define it ourselves. - // We use the initial-exec TLS model because we do not support the - // variable living anywhere other than in the main executable. - UnsafeStackPtr = new GlobalVariable( - *M, StackPtrTy, false, GlobalValue::ExternalLinkage, nullptr, - UnsafeStackPtrVar, nullptr, TLSModel); - } else { - // The variable exists, check its type and attributes. - if (UnsafeStackPtr->getValueType() != StackPtrTy) - report_fatal_error(Twine(UnsafeStackPtrVar) + " must have void* type"); - if (UseTLS != UnsafeStackPtr->isThreadLocal()) - report_fatal_error(Twine(UnsafeStackPtrVar) + " must " + - (UseTLS ? "" : "not ") + "be thread-local"); - } - return UnsafeStackPtr; + const char *UnsafeStackPtrName = "__safestack_unsafe_stack_ptr"; + auto TLSModel = UseTLS ? + // We use the initial-exec TLS model because we do not support the + // variable living anywhere other than in the main executable. + GlobalValue::InitialExecTLSModel : + GlobalValue::NotThreadLocal; + return getOrCreateStackPtrVar(*M, UnsafeStackPtrName, TLSModel); } Value *TargetLoweringBase::getSafeStackPointerLocation(IRBuilder<> &IRB) const { @@ -1812,6 +1819,13 @@ return IRB.CreateCall(Fn); } +Value *TargetLoweringBase::getSafeStackInitPointerLocation(IRBuilder<> &IRB) const { + const char *UnsafeStackPtrInitName = "__safestack_unsafe_stack_ptr_init"; + Module *M = IRB.GetInsertBlock()->getParent()->getParent(); + return getOrCreateStackPtrVar( + *M, UnsafeStackPtrInitName, GlobalValue::NotThreadLocal); +} + //===----------------------------------------------------------------------===// // Loop Strength Reduction hooks //===----------------------------------------------------------------------===// Index: lib/IR/Attributes.cpp =================================================================== --- lib/IR/Attributes.cpp +++ lib/IR/Attributes.cpp @@ -298,6 +298,8 @@ return "returned"; if (hasAttribute(Attribute::ReturnsTwice)) return "returns_twice"; + if (hasAttribute(Attribute::RuntimeInit)) + return "runtime_init"; if (hasAttribute(Attribute::SExt)) return "signext"; if (hasAttribute(Attribute::StackProtect)) @@ -527,6 +529,7 @@ case Attribute::SwiftSelf: return 1ULL << 51; case Attribute::SwiftError: return 1ULL << 52; case Attribute::WriteOnly: return 1ULL << 53; + case Attribute::RuntimeInit: return 1ULL << 54; case Attribute::Dereferenceable: llvm_unreachable("dereferenceable attribute not supported in raw format"); break; Index: lib/IR/Verifier.cpp =================================================================== --- lib/IR/Verifier.cpp +++ lib/IR/Verifier.cpp @@ -1312,6 +1312,7 @@ I->getKindAsEnum() == Attribute::UWTable || I->getKindAsEnum() == Attribute::NonLazyBind || I->getKindAsEnum() == Attribute::ReturnsTwice || + I->getKindAsEnum() == Attribute::RuntimeInit || I->getKindAsEnum() == Attribute::SanitizeAddress || I->getKindAsEnum() == Attribute::SanitizeThread || I->getKindAsEnum() == Attribute::SanitizeMemory || Index: lib/Transforms/IPO/ForceFunctionAttrs.cpp =================================================================== --- lib/Transforms/IPO/ForceFunctionAttrs.cpp +++ lib/Transforms/IPO/ForceFunctionAttrs.cpp @@ -50,6 +50,7 @@ .Case("readonly", Attribute::ReadOnly) .Case("argmemonly", Attribute::ArgMemOnly) .Case("returns_twice", Attribute::ReturnsTwice) + .Case("runtime_init", Attribute::RuntimeInit) .Case("safestack", Attribute::SafeStack) .Case("sanitize_address", Attribute::SanitizeAddress) .Case("sanitize_memory", Attribute::SanitizeMemory) Index: test/Transforms/SafeStack/runtime-init.ll =================================================================== --- /dev/null +++ test/Transforms/SafeStack/runtime-init.ll @@ -0,0 +1,22 @@ +; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s +; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s + +; CHECK: @__safestack_unsafe_stack_ptr = external thread_local(initialexec) global i8* +; CHECK: @__safestack_unsafe_stack_ptr_init = external global i8* + +; Load from an array at a fixed offset with overflow. +define i8 @StaticArrayFixedUnsafe() nounwind uwtable safestack runtime_init { +entry: + ; CHECK-LABEL: define i8 @StaticArrayFixedUnsafe( + ; CHECK: %unsafe_stack_ptr_init = load i8*, i8** @__safestack_unsafe_stack_ptr_init + ; CHECK-NEXT: [[REG0:%[0-9]+]] = icmp ne i8* %unsafe_stack_ptr_init, null + ; CHECK-NEXT: br i1 [[REG0]] + ; CHECK: %unsafe_stack_ptr = load i8*, i8** @__safestack_unsafe_stack_ptr + ; CHECK: store i8* [[REG1:.*]], i8** @__safestack_unsafe_stack_ptr_init + ; CHECK: store i8* [[REG1]], i8** @__safestack_unsafe_stack_ptr + + %buf = alloca i8, i32 4, align 1 + %gep = getelementptr inbounds i8, i8* %buf, i32 5 + %x = load i8, i8* %gep, align 1 + ret i8 %x +}