Index: lib/Transforms/IPO/LowerTypeTests.cpp =================================================================== --- lib/Transforms/IPO/LowerTypeTests.cpp +++ lib/Transforms/IPO/LowerTypeTests.cpp @@ -23,6 +23,7 @@ #include "llvm/IR/GlobalObject.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InlineAsm.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/Mangler.h" @@ -262,6 +263,7 @@ Mangler Mang; Function *WeakInitializerFn = nullptr; + Function *DummyJumpTableFn = nullptr; BitSetInfo buildBitSet(Metadata *TypeId, @@ -281,8 +283,10 @@ ArrayRef Globals); unsigned getJumpTableEntrySize(); Type *getJumpTableEntryType(); - void createJumpTableEntry(raw_ostream &OS, Function *Dest, unsigned Distance); - void createJumpTableAlias(raw_ostream &OS, Function *Dest, + void createJumpTableEntry(raw_ostream &AsmOS, raw_ostream &ConstraintOS, + SmallVectorImpl &AsmArgs, Function *Dest); + void createJumpTableAlias(raw_ostream &AsmOS, raw_ostream &ConstraintOS, + SmallVectorImpl &AsmArgs, Function *Dest, GlobalVariable *JumpTable, unsigned Distance); void verifyTypeMDNode(GlobalObject *GO, MDNode *Type); void buildBitSetsFromFunctions(ArrayRef TypeIds, @@ -299,6 +303,7 @@ void findGlobalVariableUsersOf(Constant *C, SmallSetVector &Out); + void createJumpTableFn(); public: LowerTypeTestsModule(Module &M); bool lower(); @@ -691,80 +696,52 @@ } } -static bool isValidAsmUnquotedName(StringRef Name) { - if (Name.empty()) - return false; - - for (char C : Name) { - if (!((C >= 'a' && C <= 'z') || (C >= 'A' && C <= 'Z') || - (C >= '0' && C <= '9') || C == '_' || C == '$' || C == '.' || - C == '@')) - return false; - } - - return true; -} - // Create a constant representing a jump table entry for the target. This // consists of an instruction sequence containing a relative branch to Dest. The // constant will be laid out at address Src+(Len*Distance) where Len is the // target-specific jump table entry size. -void LowerTypeTestsModule::createJumpTableEntry(raw_ostream &OS, Function *Dest, - unsigned Distance) { - // FIXME: replace IR Mangler with TargetLoweringObjectFile interface. - // A private instance of Mangler we use here can not deal with unnamed - // symbols, as it may create colliding labels. Thankfully(?), the use of - // inline asm requires us to give names to all affected functions anyway. - assert(Dest->hasName() && "jumptable targets can not be anonymous"); - SmallString<16> Name; - Mang.getNameWithPrefix(Name, Dest, /* CannotUsePrivateLabel */ false); - - if (!isValidAsmUnquotedName(Name)) { - // We are going to emit a function call as textual asm. Escaped strings - // in such expressions are not well supported. - report_fatal_error( - "CFI-ICall does not allow special characters in a function name."); - } +void LowerTypeTestsModule::createJumpTableEntry( + raw_ostream &AsmOS, raw_ostream &ConstraintOS, + SmallVectorImpl &AsmArgs, Function *Dest) { + unsigned ArgIndex = AsmArgs.size(); if (Arch == Triple::x86 || Arch == Triple::x86_64) { - OS << "jmp " << Name << "@plt\n"; - OS << "int3\nint3\nint3\n"; + AsmOS << "jmp ${" << ArgIndex << ":c}@plt\n"; + AsmOS << "int3\nint3\nint3\n"; } else if (Arch == Triple::arm || Arch == Triple::aarch64) { - OS << "b " << Name << "\n"; + AsmOS << "b ${" << ArgIndex << ":c}\n"; } else if (Arch == Triple::thumb) { - OS << "b.w " << Name << "\n"; + AsmOS << "b.w ${" << ArgIndex << ":c}\n"; } else { report_fatal_error("Unsupported architecture for jump tables"); } -} -void LowerTypeTestsModule::createJumpTableAlias(raw_ostream &OS, Function *Dest, - GlobalVariable *JumpTable, - unsigned Distance) { - assert(Dest->hasName() && "jumptable targets can not be anonymous"); - SmallString<16> Name; - Mang.getNameWithPrefix(Name, Dest, /* CannotUsePrivateLabel */ false); + ConstraintOS << (ArgIndex > 0 ? ",s" : "s"); + AsmArgs.push_back(Dest); +} - if (!isValidAsmUnquotedName(Name)) { - // We are going to emit a function alias as textual asm. Escaped strings - // in such expressions are not well supported. - report_fatal_error( - "CFI-ICall does not allow special characters in a function name."); - } +void LowerTypeTestsModule::createJumpTableAlias( + raw_ostream &AsmOS, raw_ostream &ConstraintOS, + SmallVectorImpl &AsmArgs, Function *Dest, + GlobalVariable *JumpTable, unsigned Distance) { + unsigned ArgIndex = AsmArgs.size(); if (Dest->isWeakForLinker()) - OS << ".weak " << Name << "\n"; + AsmOS << ".weak ${" << ArgIndex << ":c}\n"; else if (!Dest->hasLocalLinkage()) - OS << ".globl " << Name << "\n"; - OS << ".type " << Name << ", function\n"; + AsmOS << ".globl ${" << ArgIndex << ":c}\n"; + AsmOS << ".type ${" << ArgIndex << ":c}, function\n"; if (Arch == Triple::thumb) { - OS << ".thumb_set " << Name << ", " << JumpTable->getName() << " + " - << (getJumpTableEntrySize() * Distance) << "\n"; + AsmOS << ".thumb_set ${" << ArgIndex << ":c}, " << JumpTable->getName() + << " + " << (getJumpTableEntrySize() * Distance) << "\n"; } else { - OS << Name << " = " << JumpTable->getName() << " + " - << (getJumpTableEntrySize() * Distance) << "\n"; + AsmOS << "${" << ArgIndex << ":c} = " << JumpTable->getName() << " + " + << (getJumpTableEntrySize() * Distance) << "\n"; } - OS << ".size " << Name << ", " << getJumpTableEntrySize() << "\n"; + AsmOS << ".size ${" << ArgIndex << ":c}, " << getJumpTableEntrySize() << "\n"; + + ConstraintOS << (ArgIndex > 0 ? ",s" : "s"); + AsmArgs.push_back(Dest); } Type *LowerTypeTestsModule::getJumpTableEntryType() { @@ -844,6 +821,26 @@ PlaceholderFn->eraseFromParent(); } +void LowerTypeTestsModule::createJumpTableFn() { + // DummyJumpTableFn needs a name to be added to llvm.used. + DummyJumpTableFn = + Function::Create(FunctionType::get(Type::getVoidTy(M.getContext()), + /* IsVarArg */ false), + GlobalValue::PrivateLinkage, "__cfi_jumptables", &M); + + // Try to emit the jump table at the end of the text segment. + // Jump table must come after __cfi_check in the cross-dso mode. + // FIXME: this magic section name seems to do the trick. + DummyJumpTableFn->setSection(ObjectFormat == Triple::MachO + ? "__TEXT,__text,regular,pure_instructions" + : ".text.cfi"); + BasicBlock *BB = + BasicBlock::Create(M.getContext(), "entry", DummyJumpTableFn); + IRBuilder<> IRB(BB); + IRB.CreateUnreachable(); + appendToUsed(M, DummyJumpTableFn); +} + /// Given a disjoint set of type identifiers and functions, build a jump table /// for the functions, build the bit sets and lower the llvm.type.test calls. void LowerTypeTestsModule::buildBitSetsFromFunctionsNative( @@ -927,6 +924,11 @@ assert(!Functions.empty()); + // Create DummyJumpTableFn if necessary. + if (DummyJumpTableFn == nullptr) { + createJumpTableFn(); + } + // Build a simple layout based on the regular layout of jump tables. DenseMap GlobalLayout; unsigned EntrySize = getJumpTableEntrySize(); @@ -943,8 +945,10 @@ JumpTable->setVisibility(GlobalValue::HiddenVisibility); lowerTypeTestCalls(TypeIds, JumpTable, GlobalLayout); - std::string AsmStr; - raw_string_ostream AsmOS(AsmStr); + std::string AsmStr, ConstraintStr; + raw_string_ostream AsmOS(AsmStr), ConstraintOS(ConstraintStr); + SmallVector AsmArgs; + AsmArgs.reserve(Functions.size() * 2); // Build aliases pointing to offsets into the jump table, and replace // references to the original functions with references to the aliases. @@ -965,7 +969,10 @@ F->getType()); if (F->isWeakForLinker()) { - AsmOS << ".weak " << F->getName() << "\n"; + unsigned ArgIndex = AsmArgs.size(); + AsmOS << ".weak ${" << ArgIndex << ":c}\n"; + ConstraintOS << (ArgIndex > 0 ? ",s" : "s"); + AsmArgs.push_back(F); replaceWeakDeclarationWithJumpTablePtr(F, CombinedGlobalElemPtr); } else { F->replaceAllUsesWith(CombinedGlobalElemPtr); @@ -973,8 +980,6 @@ } else { assert(F->getType()->getAddressSpace() == 0); - createJumpTableAlias(AsmOS, F, JumpTable, I); - Function *DeclAlias = Function::Create(cast(F->getValueType()), GlobalValue::ExternalLinkage, "", &M); @@ -985,6 +990,9 @@ ? GlobalValue::HiddenVisibility : F->getVisibility()); DeclAlias->takeName(F); + createJumpTableAlias(AsmOS, ConstraintOS, AsmArgs, DeclAlias, JumpTable, + I); + // Unnamed functions can not be added to llvm.used. F->setName(DeclAlias->getName() + ".cfi"); F->replaceAllUsesWith(DeclAlias); @@ -993,28 +1001,28 @@ F->setLinkage(GlobalValue::InternalLinkage); } - // Try to emit the jump table at the end of the text segment. - // Jump table must come after __cfi_check in the cross-dso mode. - // FIXME: this magic section name seems to do the trick. - AsmOS << ".section " << (ObjectFormat == Triple::MachO - ? "__TEXT,__text,regular,pure_instructions" - : ".text.cfi, \"ax\", %progbits") - << "\n"; // Align the whole table by entry size. AsmOS << ".balign " << EntrySize << "\n"; if (Arch == Triple::thumb) AsmOS << ".thumb_func\n"; AsmOS << JumpTable->getName() << ":\n"; - for (unsigned I = 0; I != Functions.size(); ++I) - createJumpTableEntry(AsmOS, cast(Functions[I]->getGlobal()), I); - - M.appendModuleInlineAsm(AsmOS.str()); - SmallVector Used; - Used.reserve(Functions.size()); - for (auto *F : Functions) - Used.push_back(F->getGlobal()); - appendToUsed(M, Used); + for (unsigned I = 0; I != Functions.size(); ++I) + createJumpTableEntry(AsmOS, ConstraintOS, AsmArgs, + cast(Functions[I]->getGlobal())); + + // Add the new table as a inline asm call in DummyJumpTableFn. + IRBuilder<> IRB(DummyJumpTableFn->getEntryBlock().getTerminator()); + SmallVector ArgTypes; + ArgTypes.reserve(AsmArgs.size()); + for (const auto &Arg : AsmArgs) + ArgTypes.push_back(Arg->getType()); + InlineAsm *JumpTableAsm = + InlineAsm::get(FunctionType::get(IRB.getVoidTy(), ArgTypes, false), + AsmOS.str(), ConstraintOS.str(), + /*hasSideEffects=*/true); + + IRB.CreateCall(JumpTableAsm, AsmArgs); } /// Assign a dummy layout using an incrementing counter, tag each function Index: test/Transforms/LowerTypeTests/function-disjoint.ll =================================================================== --- test/Transforms/LowerTypeTests/function-disjoint.ll +++ test/Transforms/LowerTypeTests/function-disjoint.ll @@ -5,23 +5,6 @@ target datalayout = "e-p:64:64" -; X64: module asm "f = .cfi.jumptable + 0" - -; X64: module asm ".cfi.jumptable:" -; X64-NEXT: module asm "jmp f.cfi@plt" -; X64-NEXT: module asm "int3" -; X64-NEXT: module asm "int3" -; X64-NEXT: module asm "int3" - -; X64: module asm "g = .cfi.jumptable.1 + 0" - -; X64: module asm ".cfi.jumptable.1:" -; X64-NEXT: module asm "jmp g.cfi@plt" -; X64-NEXT: module asm "int3" -; X64-NEXT: module asm "int3" -; X64-NEXT: module asm "int3" - - ; X64: @.cfi.jumptable = external hidden constant [1 x [8 x i8]] ; X64: @.cfi.jumptable.1 = external hidden constant [1 x [8 x i8]] @@ -56,8 +39,21 @@ ret i1 %z } +; X64: define private void @__cfi_jumptables() +; X64: call void asm sideeffect " +; X64-SAME: ${0:c} = .cfi.jumptable + 0 +; X64-SAME: .cfi.jumptable: +; X64-SAME: jmp ${1:c}@plt +; X64-SAME: "s,s"(void ()* @f, void ()* @f.cfi) + +; X64: call void asm sideeffect " +; X64-SAME: ${0:c} = .cfi.jumptable.1 + 0 +; X64-SAME: .cfi.jumptable.1: +; X64-SAME: jmp ${1:c}@plt +; X64-SAME: "s,s"(void ()* @g, void ()* @g.cfi) + ; X64: declare void @f() ; X64: declare void @g() ; WASM32: ![[I0]] = !{i64 1} ; WASM32: ![[I1]] = !{i64 2} \ No newline at end of file Index: test/Transforms/LowerTypeTests/function-ext.ll =================================================================== --- test/Transforms/LowerTypeTests/function-ext.ll +++ test/Transforms/LowerTypeTests/function-ext.ll @@ -4,10 +4,6 @@ ; Tests that we correctly handle external references, including the case where ; all functions in a bitset are external references. -; X64: module asm ".cfi.jumptable:" -; X64-NEXT: module asm "jmp foo@plt" -; X64-NOT: module asm "jmp {{.*}}@plt" - ; X64: @.cfi.jumptable = external hidden constant [1 x [8 x i8]] ; WASM32: private constant [0 x i8] zeroinitializer @@ -27,3 +23,8 @@ !0 = !{i64 0, !"void"} ; WASM-NOT: !{i64 0} ; WASM-NOT: !{i64 1} + +; X64: define private void @__cfi_jumptables() +; X64: jmp ${0:c}@plt +; X64-NOT: jmp {{.*}}@plt +; X64-SAME: "s"(void ()* @foo) Index: test/Transforms/LowerTypeTests/function-weak.ll =================================================================== --- test/Transforms/LowerTypeTests/function-weak.ll +++ test/Transforms/LowerTypeTests/function-weak.ll @@ -6,8 +6,6 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" -; CHECK: module asm ".weak f" - ; CHECK: @x = global void ()* null, align 8 @x = global void ()* @f, align 8 @@ -53,6 +51,10 @@ ret i1 %x } +; CHECK: define private void @__cfi_jumptables() +; CHECK: .weak ${0:c} +; CHECK-SAME: "s,s"(void ()* @f, + ; CHECK: define internal void @__cfi_global_var_init() section ".text.startup" { ; CHECK-NEXT: entry: ; CHECK-NEXT: store { void ()*, void ()*, i32 } { void ()* select (i1 icmp ne (void ()* @f, void ()* null), void ()* bitcast ({{.*}}@.cfi.jumptable to void ()*), void ()* null), void ()* select (i1 icmp ne (void ()* @f, void ()* null), void ()* bitcast ({{.*}}@.cfi.jumptable to void ()*), void ()* null), i32 42 }, { void ()*, void ()*, i32 }* @s, align 8 @@ -63,5 +65,4 @@ ; CHECK-NEXT: ret void ; CHECK-NEXT: } - !0 = !{i32 0, !"typeid1"} Index: test/Transforms/LowerTypeTests/function.ll =================================================================== --- test/Transforms/LowerTypeTests/function.ll +++ test/Transforms/LowerTypeTests/function.ll @@ -9,52 +9,7 @@ target datalayout = "e-p:64:64" -; X86: module asm ".globl f" -; X86-NEXT: module asm ".type f, function" -; X86-NEXT: module asm "f = .cfi.jumptable + 0" -; X86-NEXT: module asm ".size f, 8" -; X86-NEXT: module asm ".type g, function" -; X86-NEXT: module asm "g = .cfi.jumptable + 8" -; X86-NEXT: module asm ".size g, 8" -; X86-NEXT: module asm ".section .text.cfi, \22ax\22, %progbits" -; X86-NEXT: module asm ".balign 8" -; X86-NEXT: module asm ".cfi.jumptable:" -; X86-NEXT: module asm "jmp f.cfi@plt" -; X86-NEXT: module asm "int3" -; X86-NEXT: module asm "int3" -; X86-NEXT: module asm "int3" -; X86-NEXT: module asm "jmp g.cfi@plt" -; X86-NEXT: module asm "int3" -; X86-NEXT: module asm "int3" -; X86-NEXT: module asm "int3" - -; ARM: module asm ".globl f" -; ARM-NEXT: module asm ".type f, function" -; ARM-NEXT: module asm "f = .cfi.jumptable + 0" -; ARM-NEXT: module asm ".size f, 4" -; ARM-NEXT: module asm ".type g, function" -; ARM-NEXT: module asm "g = .cfi.jumptable + 4" -; ARM-NEXT: module asm ".size g, 4" -; ARM-NEXT: module asm ".section .text.cfi, \22ax\22, %progbits" -; ARM-NEXT: module asm ".balign 4" -; ARM-NEXT: module asm ".cfi.jumptable:" -; ARM-NEXT: module asm "b f.cfi" -; ARM-NEXT: module asm "b g.cfi" - -; THUMB: module asm ".globl f" -; THUMB-NEXT: module asm ".type f, function" -; THUMB-NEXT: module asm ".thumb_set f, .cfi.jumptable + 0" -; THUMB-NEXT: module asm ".size f, 4" -; THUMB-NEXT: module asm ".type g, function" -; THUMB-NEXT: module asm ".thumb_set g, .cfi.jumptable + 4" -; THUMB-NEXT: module asm ".size g, 4" -; THUMB-NEXT: module asm ".section .text.cfi, \22ax\22, %progbits" -; THUMB-NEXT: module asm ".balign 4" -; THUMB-NEXT: module asm ".thumb_func" -; THUMB-NEXT: module asm ".cfi.jumptable:" -; THUMB-NEXT: module asm "b.w f.cfi" -; THUMB-NEXT: module asm "b.w g.cfi" - +; NATIVE: @llvm.used = appending global [1 x i8*] [i8* bitcast (void ()* @__cfi_jumptables to i8*)], section "llvm.metadata" ; X86: @.cfi.jumptable = external hidden constant [2 x [8 x i8]] ; ARM: @.cfi.jumptable = external hidden constant [2 x [4 x i8]] @@ -63,8 +18,6 @@ ; WASM32: private constant [0 x i8] zeroinitializer @0 = private unnamed_addr constant [2 x void (...)*] [void (...)* bitcast (void ()* @f to void (...)*), void (...)* bitcast (void ()* @g to void (...)*)], align 16 -; NATIVE: @llvm.used = appending global [2 x i8*] [i8* bitcast (void ()* @f.cfi to i8*), i8* bitcast (void ()* @g.cfi to i8*)], section "llvm.metadata" - ; NATIVE: define internal void @f.cfi() ; WASM32: define void @f() !type !{{[0-9]+}} !wasm.index ![[I0:[0-9]+]] define void @f() !type !0 { @@ -90,6 +43,53 @@ ret i1 %x } +; NATIVE: define private void @__cfi_jumptables() + +; X86: .globl ${0:c} +; X86-SAME: .type ${0:c}, function +; X86-SAME: ${0:c} = .cfi.jumptable + 0 +; X86-SAME: .size ${0:c}, 8 +; X86-SAME: .type ${1:c}, function +; X86-SAME: ${1:c} = .cfi.jumptable + 8 +; X86-SAME: .size ${1:c}, 8 +; X86-SAME: .balign 8 +; X86-SAME: .cfi.jumptable: +; X86-SAME: jmp ${2:c}@plt +; X86-SAME: int3 +; X86-SAME: int3 +; X86-SAME: int3 +; X86-SAME: jmp ${3:c}@plt +; X86-SAME: int3 +; X86-SAME: int3 +; X86-SAME: int3 + +; ARM: .globl ${0:c} +; ARM-SAME: .type ${0:c}, function +; ARM-SAME: ${0:c} = .cfi.jumptable + 0 +; ARM-SAME: .size ${0:c}, 4 +; ARM-SAME: .type ${1:c}, function +; ARM-SAME: ${1:c} = .cfi.jumptable + 4 +; ARM-SAME: .size ${1:c}, 4 +; ARM-SAME: .balign 4 +; ARM-SAME: .cfi.jumptable: +; ARM-SAME: b ${2:c} +; ARM-SAME: b ${3:c} + +; THUMB: .globl ${0:c} +; THUMB-SAME: .type ${0:c}, function +; THUMB-SAME: .thumb_set ${0:c}, .cfi.jumptable + 0 +; THUMB-SAME: .size ${0:c}, 4 +; THUMB-SAME: .type ${1:c}, function +; THUMB-SAME: .thumb_set ${1:c}, .cfi.jumptable + 4 +; THUMB-SAME: .size ${1:c}, 4 +; THUMB-SAME: .balign 4 +; THUMB-SAME: .thumb_func +; THUMB-SAME: .cfi.jumptable: +; THUMB-SAME: b.w ${2:c} +; THUMB-SAME: b.w ${3:c} + +; NATIVE-SAME: "s,s,s,s"(void ()* @f, void ()* @g, void ()* @f.cfi, void ()* @g.cfi) + ; NATIVE: declare void @f() ; NATIVE: declare hidden void @g() Index: test/Transforms/LowerTypeTests/section.ll =================================================================== --- test/Transforms/LowerTypeTests/section.ll +++ test/Transforms/LowerTypeTests/section.ll @@ -5,10 +5,6 @@ target triple = "x86_64-unknown-linux-gnu" -; CHECK: module asm ".section .text.cfi, -; CHECK: module asm ".cfi.jumptable:" -; CHECK-NEXT: module asm "jmp f.cfi@plt" - ; CHECK: define internal void @f.cfi() section "xxx" define void @f() section "xxx" !type !0 { @@ -22,6 +18,8 @@ ret i1 %0 } +; CHECK: define private void @__cfi_jumptables() section ".text.cfi" { + declare i1 @llvm.type.test(i8*, metadata) nounwind readnone !0 = !{i64 0, !"_ZTSFvE"}