Index: include/llvm/Bitcode/LLVMBitCodes.h =================================================================== --- include/llvm/Bitcode/LLVMBitCodes.h +++ include/llvm/Bitcode/LLVMBitCodes.h @@ -520,7 +520,8 @@ ATTR_KIND_NO_RECURSE = 48, ATTR_KIND_INACCESSIBLEMEM_ONLY = 49, ATTR_KIND_INACCESSIBLEMEM_OR_ARGMEMONLY = 50, - ATTR_KIND_ALLOC_SIZE = 51 + ATTR_KIND_ALLOC_SIZE = 51, + ATTR_KIND_RUNTIME_INIT = 52 }; enum ComdatSelectionKindCodes { Index: include/llvm/IR/Attributes.td =================================================================== --- include/llvm/IR/Attributes.td +++ include/llvm/IR/Attributes.td @@ -127,6 +127,12 @@ /// Function can return twice. def ReturnsTwice : EnumAttr<"returns_twice">; +/// Function may be used during initialization of dynamic linker or language +/// runtime. SafeStack will compile function to be usable both before and after +/// the default (e.g. thread-local) unsafe stack pointer is configured, assuming +/// that appropriate runtime support is available. +def RuntimeInit : EnumAttr<"runtime_init">; + /// Safe Stack protection. def SafeStack : EnumAttr<"safestack">; Index: lib/AsmParser/LLLexer.cpp =================================================================== --- lib/AsmParser/LLLexer.cpp +++ lib/AsmParser/LLLexer.cpp @@ -646,6 +646,7 @@ KEYWORD(readonly); KEYWORD(returned); KEYWORD(returns_twice); + KEYWORD(runtime_init); KEYWORD(signext); KEYWORD(sret); KEYWORD(ssp); Index: lib/AsmParser/LLParser.cpp =================================================================== --- lib/AsmParser/LLParser.cpp +++ lib/AsmParser/LLParser.cpp @@ -1090,6 +1090,7 @@ case lltok::kw_readonly: B.addAttribute(Attribute::ReadOnly); break; case lltok::kw_returns_twice: B.addAttribute(Attribute::ReturnsTwice); break; + case lltok::kw_runtime_init: B.addAttribute(Attribute::RuntimeInit); break; case lltok::kw_ssp: B.addAttribute(Attribute::StackProtect); break; case lltok::kw_sspreq: B.addAttribute(Attribute::StackProtectReq); break; case lltok::kw_sspstrong: @@ -1419,6 +1420,7 @@ case lltok::kw_optnone: case lltok::kw_optsize: case lltok::kw_returns_twice: + case lltok::kw_runtime_init: case lltok::kw_sanitize_address: case lltok::kw_sanitize_memory: case lltok::kw_sanitize_thread: @@ -1510,6 +1512,7 @@ case lltok::kw_optnone: case lltok::kw_optsize: case lltok::kw_returns_twice: + case lltok::kw_runtime_init: case lltok::kw_sanitize_address: case lltok::kw_sanitize_memory: case lltok::kw_sanitize_thread: Index: lib/AsmParser/LLToken.h =================================================================== --- lib/AsmParser/LLToken.h +++ lib/AsmParser/LLToken.h @@ -151,6 +151,7 @@ kw_readonly, kw_returned, kw_returns_twice, + kw_runtime_init, kw_signext, kw_ssp, kw_sspreq, Index: lib/Bitcode/Reader/BitcodeReader.cpp =================================================================== --- lib/Bitcode/Reader/BitcodeReader.cpp +++ lib/Bitcode/Reader/BitcodeReader.cpp @@ -1446,6 +1446,8 @@ return Attribute::Returned; case bitc::ATTR_KIND_RETURNS_TWICE: return Attribute::ReturnsTwice; + case bitc::ATTR_KIND_RUNTIME_INIT: + return Attribute::RuntimeInit; case bitc::ATTR_KIND_S_EXT: return Attribute::SExt; case bitc::ATTR_KIND_STACK_ALIGNMENT: Index: lib/Bitcode/Writer/BitcodeWriter.cpp =================================================================== --- lib/Bitcode/Writer/BitcodeWriter.cpp +++ lib/Bitcode/Writer/BitcodeWriter.cpp @@ -483,6 +483,8 @@ return bitc::ATTR_KIND_RETURNED; case Attribute::ReturnsTwice: return bitc::ATTR_KIND_RETURNS_TWICE; + case Attribute::RuntimeInit: + return bitc::ATTR_KIND_RUNTIME_INIT; case Attribute::SExt: return bitc::ATTR_KIND_S_EXT; case Attribute::StackAlignment: Index: lib/CodeGen/SafeStack.cpp =================================================================== --- lib/CodeGen/SafeStack.cpp +++ lib/CodeGen/SafeStack.cpp @@ -101,7 +101,13 @@ Type *Int32Ty; Type *Int8Ty; + /// Pointer to default unsafe stack pointer Value *UnsafeStackPtr = nullptr; + /// Pointer to single-threaded unsafe stack pointer to be used during program + /// initialization + GlobalVariable *UnsafeStackPtrInit = nullptr; + /// Load of init unsafe stack pointer at beginning of function + Value *UnsafeStackPtrInitLoad = nullptr; /// Unsafe stack alignment. Each stack frame must ensure that the stack is /// aligned to this value. We need to re-align the unsafe stack if the @@ -111,8 +117,34 @@ /// might expect to appear on the stack on most common targets. enum { StackAlignment = 16 }; - /// \brief Build a value representing a pointer to the unsafe stack pointer. - Value *getOrCreateUnsafeStackPtr(IRBuilder<> &IRB, Function &F); + /// \brief Load the unsafe stack pointer. Assumes that + /// getOrCreateUnsafeStackPtr was invoked at the beginning of the function + /// being processed. + /// + /// \param Top Set to true if this load is at the beginning/top of the + /// function. + Instruction *loadUnsafeStackPtr(IRBuilder<> &IRB, bool Top = false); + + GlobalVariable *getOrCreateStackPtrVar(Module &M, const char *Name, + GlobalValue::ThreadLocalMode TlsMode); + + /// \brief Initialize UnsafeStackPtr with a value representing a pointer to + /// the default unsafe stack pointer. + /// If the function has the runtime_init attribute, then also initialize + /// UnsafeStackPtrInit with a value representing a pointer to the single- + /// threaded unsafe stack pointer to be used during program initialization. + /// Initialize UnsafeStackPtrInitLoad with an instruction that loads + /// UnsafeStackPtrInit. That loaded value is initially used to select + /// between that init pointer and the default pointer, depending on whether + /// the init pointer is null. That loaded value is also used to determine + /// where to later store the updated unsafe stack pointer. + /// + /// \returns instruction that loads the unsafe stack pointer. + Instruction *getOrCreateUnsafeStackPtr(IRBuilder<> &IRB, Function &F); + + /// \brief Store the updated value of the unsafe stack pointer. + void storeUnsafeStackPtr(IRBuilder<> &IRB, Value *UpdatedUSP, + Value *NameSource = nullptr); /// \brief Return the value of the stack canary. Value *getStackGuard(IRBuilder<> &IRB, Function &F); @@ -160,8 +192,7 @@ /// \brief Replace all allocas in \p DynamicAllocas with code to allocate /// space dynamically on the unsafe stack and store the dynamic unsafe stack /// top to \p DynamicTop if non-null. - void moveDynamicAllocasToUnsafeStack(Function &F, Value *UnsafeStackPtr, - AllocaInst *DynamicTop, + void moveDynamicAllocasToUnsafeStack(Function &F, AllocaInst *DynamicTop, ArrayRef DynamicAllocas); bool IsSafeStackAlloca(const Value *AllocaPtr, uint64_t AllocaSize); @@ -348,34 +379,110 @@ return true; } -Value *SafeStack::getOrCreateUnsafeStackPtr(IRBuilder<> &IRB, Function &F) { - // Check if there is a target-specific location for the unsafe stack pointer. - if (TL) - if (Value *V = TL->getSafeStackPointerLocation(IRB)) - return V; +Instruction *SafeStack::loadUnsafeStackPtr(IRBuilder<> &IRB, bool Top) { + if (UnsafeStackPtrInit == nullptr) + return IRB.CreateLoad(UnsafeStackPtr, false, "unsafe_stack_ptr"); - // Otherwise, assume the target links with compiler-rt, which provides a - // thread-local variable with a magic name. - Module &M = *F.getParent(); - const char *UnsafeStackPtrVar = "__safestack_unsafe_stack_ptr"; - auto UnsafeStackPtr = - dyn_cast_or_null(M.getNamedValue(UnsafeStackPtrVar)); + auto IsInit = IRB.CreateIsNotNull(UnsafeStackPtrInitLoad); + + auto PHI = IRB.CreatePHI(StackPtrTy, 2); + TerminatorInst *ThenTerm, *ElseTerm; + SplitBlockAndInsertIfThenElse(IsInit, PHI, &ThenTerm, &ElseTerm); + IRBuilder<> InitIRB(ThenTerm); + IRBuilder<> DefaultIRB(ElseTerm); + auto Tail = ThenTerm->getSuccessor(0); + IRB.SetInsertPoint(Tail, Tail->getFirstInsertionPt()); + + auto InitLoad = Top? + // There is no need to load this twice at the top of the function. + UnsafeStackPtrInitLoad : + InitIRB.CreateLoad(UnsafeStackPtrInit, false, "unsafe_stack_ptr_init"); + auto DefaultLoad = DefaultIRB.CreateLoad(UnsafeStackPtr, false, "unsafe_stack_ptr"); - if (!UnsafeStackPtr) { + PHI->addIncoming(InitLoad, ThenTerm->getParent()); + PHI->addIncoming(DefaultLoad, ElseTerm->getParent()); + + return PHI; +} + +GlobalVariable *SafeStack::getOrCreateStackPtrVar(Module &M, const char *Name, + GlobalValue::ThreadLocalMode TlsMode) { + GlobalVariable *V = dyn_cast_or_null(M.getNamedValue(Name)); + if (!V) { // The global variable is not defined yet, define it ourselves. - // We use the initial-exec TLS model because we do not support the - // variable living anywhere other than in the main executable. - UnsafeStackPtr = new GlobalVariable( - M, StackPtrTy, false, GlobalValue::ExternalLinkage, nullptr, - UnsafeStackPtrVar, nullptr, GlobalValue::InitialExecTLSModel); + V = new GlobalVariable( + M, StackPtrTy, false, GlobalValue::ExternalLinkage, nullptr, Name, + nullptr, TlsMode); } else { // The variable exists, check its type and attributes. - if (UnsafeStackPtr->getValueType() != StackPtrTy) - report_fatal_error(Twine(UnsafeStackPtrVar) + " must have void* type"); - if (!UnsafeStackPtr->isThreadLocal()) - report_fatal_error(Twine(UnsafeStackPtrVar) + " must be thread-local"); + if (V->getValueType() != StackPtrTy) + report_fatal_error(Twine(Name) + " must have void* type"); + if (TlsMode == GlobalValue::NotThreadLocal) { + if (V->isThreadLocal()) + report_fatal_error(Twine(Name) + " must not be thread-local"); + } else + if (!V->isThreadLocal()) + report_fatal_error(Twine(Name) + " must be thread-local"); } - return UnsafeStackPtr; + + return V; +} + +Instruction *SafeStack::getOrCreateUnsafeStackPtr(IRBuilder<> &IRB, Function &F) { + Module &M = *F.getParent(); + if (F.hasFnAttribute(Attribute::RuntimeInit)) { + const char *UnsafeStackPtrInitName = "__safestack_unsafe_stack_ptr_init"; + UnsafeStackPtrInit = getOrCreateStackPtrVar( + M, UnsafeStackPtrInitName, GlobalValue::NotThreadLocal); + + UnsafeStackPtrInitLoad = + IRB.CreateLoad(UnsafeStackPtrInit, false, "unsafe_stack_ptr_init"); + } + + // Check if there is a target-specific location for the unsafe stack pointer. + if (TL) { + UnsafeStackPtr = TL->getSafeStackPointerLocation(IRB); + if (UnsafeStackPtr) + return loadUnsafeStackPtr(IRB, true); + } + + // Otherwise, assume the target links with compiler-rt, which provides a + // thread-local variable with a magic name. + const char *UnsafeStackPtrName = "__safestack_unsafe_stack_ptr"; + // We use the initial-exec TLS model because we do not support the + // variable living anywhere other than in the main executable. + UnsafeStackPtr = getOrCreateStackPtrVar( + M, UnsafeStackPtrName, GlobalValue::InitialExecTLSModel); + + return loadUnsafeStackPtr(IRB, true); +} + +void SafeStack::storeUnsafeStackPtr(IRBuilder<> &IRB, Value *UpdatedUSP, + Value *NameSource) { + if (UnsafeStackPtrInit == nullptr) { + auto DefaultStore = IRB.CreateStore(UpdatedUSP, UnsafeStackPtr); + if (NameSource != nullptr) + DefaultStore->takeName(NameSource); + return; + } + + auto IsInit = IRB.CreateIsNotNull(UnsafeStackPtrInitLoad); + TerminatorInst *ThenTerm, *ElseTerm; + SplitBlockAndInsertIfThenElse(IsInit, (Instruction *)IRB.GetInsertPoint(), + &ThenTerm, &ElseTerm); + IRBuilder<> InitIRB(ThenTerm); + IRBuilder<> DefaultIRB(ElseTerm); + auto Tail = ThenTerm->getSuccessor(0); + IRB.SetInsertPoint(Tail, Tail->getFirstInsertionPt()); + + auto InitStore = InitIRB.CreateStore(UpdatedUSP, UnsafeStackPtrInit); + auto DefaultStore = DefaultIRB.CreateStore(UpdatedUSP, UnsafeStackPtr); + + if (NameSource == nullptr) + return; + + InitStore->takeName(NameSource); + DefaultStore->takeName(NameSource); } Value *SafeStack::getStackGuard(IRBuilder<> &IRB, Function &F) { @@ -467,7 +574,7 @@ IRB.SetInsertPoint(I->getNextNode()); Value *CurrentTop = DynamicTop ? IRB.CreateLoad(DynamicTop) : StaticTop; - IRB.CreateStore(CurrentTop, UnsafeStackPtr); + storeUnsafeStackPtr(IRB, CurrentTop); } return DynamicTop; @@ -617,12 +724,12 @@ Value *StaticTop = IRB.CreateGEP(BasePointer, ConstantInt::get(Int32Ty, -StaticOffset), "unsafe_stack_static_top"); - IRB.CreateStore(StaticTop, UnsafeStackPtr); + storeUnsafeStackPtr(IRB, StaticTop); return StaticTop; } void SafeStack::moveDynamicAllocasToUnsafeStack( - Function &F, Value *UnsafeStackPtr, AllocaInst *DynamicTop, + Function &F, AllocaInst *DynamicTop, ArrayRef DynamicAllocas) { DIBuilder DIB(*F.getParent()); @@ -638,7 +745,7 @@ uint64_t TySize = DL->getTypeAllocSize(Ty); Value *Size = IRB.CreateMul(ArraySize, ConstantInt::get(IntPtrTy, TySize)); - Value *SP = IRB.CreatePtrToInt(IRB.CreateLoad(UnsafeStackPtr), IntPtrTy); + Value *SP = IRB.CreatePtrToInt(loadUnsafeStackPtr(IRB), IntPtrTy); SP = IRB.CreateSub(SP, Size); // Align the SP value to satisfy the AllocaInst, type and stack alignments. @@ -652,7 +759,7 @@ StackPtrTy); // Save the stack pointer. - IRB.CreateStore(NewTop, UnsafeStackPtr); + storeUnsafeStackPtr(IRB, NewTop); if (DynamicTop) IRB.CreateStore(NewTop, DynamicTop); @@ -666,6 +773,7 @@ } if (!DynamicAllocas.empty()) { + std::vector StackSaves, StackRestores; // Now go through the instructions again, replacing stacksave/stackrestore. for (inst_iterator It = inst_begin(&F), Ie = inst_end(&F); It != Ie;) { Instruction *I = &*(It++); @@ -673,19 +781,25 @@ if (!II) continue; - if (II->getIntrinsicID() == Intrinsic::stacksave) { - IRBuilder<> IRB(II); - Instruction *LI = IRB.CreateLoad(UnsafeStackPtr); - LI->takeName(II); - II->replaceAllUsesWith(LI); - II->eraseFromParent(); - } else if (II->getIntrinsicID() == Intrinsic::stackrestore) { - IRBuilder<> IRB(II); - Instruction *SI = IRB.CreateStore(II->getArgOperand(0), UnsafeStackPtr); - SI->takeName(II); - assert(II->use_empty()); - II->eraseFromParent(); - } + if (II->getIntrinsicID() == Intrinsic::stacksave) + StackSaves.push_back(II); + else if (II->getIntrinsicID() == Intrinsic::stackrestore) + StackRestores.push_back(II); + } + + for (auto II : StackSaves) { + IRBuilder<> IRB(II); + Instruction *LI = loadUnsafeStackPtr(IRB); + LI->takeName(II); + II->replaceAllUsesWith(LI); + II->eraseFromParent(); + } + + for (auto II : StackRestores) { + IRBuilder<> IRB(II); + storeUnsafeStackPtr(IRB, II->getArgOperand(0), II); + assert(II->use_empty()); + II->eraseFromParent(); } } } @@ -693,6 +807,8 @@ bool SafeStack::runOnFunction(Function &F) { DEBUG(dbgs() << "[SafeStack] Function: " << F.getName() << "\n"); + UnsafeStackPtrInit = nullptr; + if (!F.hasFnAttribute(Attribute::SafeStack)) { DEBUG(dbgs() << "[SafeStack] safestack is not requested" " for this function\n"); @@ -739,12 +855,10 @@ ++NumUnsafeStackRestorePointsFunctions; IRBuilder<> IRB(&F.front(), F.begin()->getFirstInsertionPt()); - UnsafeStackPtr = getOrCreateUnsafeStackPtr(IRB, F); // Load the current stack pointer (we'll also use it as a base pointer). // FIXME: use a dedicated register for it ? - Instruction *BasePointer = - IRB.CreateLoad(UnsafeStackPtr, false, "unsafe_stack_ptr"); + Instruction *BasePointer = getOrCreateUnsafeStackPtr(IRB, F); assert(BasePointer->getType() == StackPtrTy); AllocaInst *StackGuardSlot = nullptr; @@ -778,13 +892,12 @@ IRB, F, StackRestorePoints, StaticTop, !DynamicAllocas.empty()); // Handle dynamic allocas. - moveDynamicAllocasToUnsafeStack(F, UnsafeStackPtr, DynamicTop, - DynamicAllocas); + moveDynamicAllocasToUnsafeStack(F, DynamicTop, DynamicAllocas); // Restore the unsafe stack pointer before each return. for (ReturnInst *RI : Returns) { IRB.SetInsertPoint(RI); - IRB.CreateStore(BasePointer, UnsafeStackPtr); + storeUnsafeStackPtr(IRB, BasePointer); } DEBUG(dbgs() << "[SafeStack] safestack applied\n"); Index: lib/IR/Attributes.cpp =================================================================== --- lib/IR/Attributes.cpp +++ lib/IR/Attributes.cpp @@ -296,6 +296,8 @@ return "returned"; if (hasAttribute(Attribute::ReturnsTwice)) return "returns_twice"; + if (hasAttribute(Attribute::RuntimeInit)) + return "runtime_init"; if (hasAttribute(Attribute::SExt)) return "signext"; if (hasAttribute(Attribute::StackProtect)) @@ -516,6 +518,7 @@ case Attribute::InaccessibleMemOrArgMemOnly: return 1ULL << 50; case Attribute::SwiftSelf: return 1ULL << 51; case Attribute::SwiftError: return 1ULL << 52; + case Attribute::RuntimeInit: return 1ULL << 53; case Attribute::Dereferenceable: llvm_unreachable("dereferenceable attribute not supported in raw format"); break; Index: lib/IR/Verifier.cpp =================================================================== --- lib/IR/Verifier.cpp +++ lib/IR/Verifier.cpp @@ -1265,6 +1265,7 @@ I->getKindAsEnum() == Attribute::UWTable || I->getKindAsEnum() == Attribute::NonLazyBind || I->getKindAsEnum() == Attribute::ReturnsTwice || + I->getKindAsEnum() == Attribute::RuntimeInit || I->getKindAsEnum() == Attribute::SanitizeAddress || I->getKindAsEnum() == Attribute::SanitizeThread || I->getKindAsEnum() == Attribute::SanitizeMemory || Index: lib/Transforms/IPO/ForceFunctionAttrs.cpp =================================================================== --- lib/Transforms/IPO/ForceFunctionAttrs.cpp +++ lib/Transforms/IPO/ForceFunctionAttrs.cpp @@ -50,6 +50,7 @@ .Case("readonly", Attribute::ReadOnly) .Case("argmemonly", Attribute::ArgMemOnly) .Case("returns_twice", Attribute::ReturnsTwice) + .Case("runtime_init", Attribute::RuntimeInit) .Case("safestack", Attribute::SafeStack) .Case("sanitize_address", Attribute::SanitizeAddress) .Case("sanitize_memory", Attribute::SanitizeMemory) Index: test/Transforms/SafeStack/runtime-init.ll =================================================================== --- /dev/null +++ test/Transforms/SafeStack/runtime-init.ll @@ -0,0 +1,22 @@ +; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s +; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s + +; CHECK: @__safestack_unsafe_stack_ptr_init = external global i8* +; CHECK: @__safestack_unsafe_stack_ptr = external thread_local(initialexec) global i8* + +; Load from an array at a fixed offset with overflow. +define i8 @StaticArrayFixedUnsafe() nounwind uwtable safestack runtime_init { +entry: + ; CHECK-LABEL: define i8 @StaticArrayFixedUnsafe( + ; CHECK: %unsafe_stack_ptr_init = load i8*, i8** @__safestack_unsafe_stack_ptr_init + ; CHECK-NEXT: [[REG0:%[0-9]+]] = icmp ne i8* %unsafe_stack_ptr_init, null + ; CHECK-NEXT: br i1 [[REG0]] + ; CHECK: %unsafe_stack_ptr = load i8*, i8** @__safestack_unsafe_stack_ptr + ; CHECK: store i8* [[REG1:.*]], i8** @__safestack_unsafe_stack_ptr_init + ; CHECK: store i8* [[REG1]], i8** @__safestack_unsafe_stack_ptr + + %buf = alloca i8, i32 4, align 1 + %gep = getelementptr inbounds i8, i8* %buf, i32 5 + %x = load i8, i8* %gep, align 1 + ret i8 %x +}