diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -4225,6 +4225,20 @@ This is currently only supported for ELF binary formats. +.. _no_cfi: + +No CFI +------ + +``no_cfi @func`` + +With `Control-Flow Integrity (CFI) +`_, a '``no_cfi``' +constant represents a function reference that does not get replaced with a +reference to the CFI jump table in the ``LowerTypeTests`` pass. These constants +may be useful in low-level programs, such as operating system kernels, which +need to refer to the actual function body. + .. _constantexprs: Constant Expressions diff --git a/llvm/include/llvm/AsmParser/LLParser.h b/llvm/include/llvm/AsmParser/LLParser.h --- a/llvm/include/llvm/AsmParser/LLParser.h +++ b/llvm/include/llvm/AsmParser/LLParser.h @@ -62,12 +62,14 @@ APFloat APFloatVal{0.0}; Constant *ConstantVal; std::unique_ptr ConstantStructElts; + bool NoCFI = false; ValID() = default; ValID(const ValID &RHS) : Kind(RHS.Kind), Loc(RHS.Loc), UIntVal(RHS.UIntVal), FTy(RHS.FTy), StrVal(RHS.StrVal), StrVal2(RHS.StrVal2), APSIntVal(RHS.APSIntVal), - APFloatVal(RHS.APFloatVal), ConstantVal(RHS.ConstantVal) { + APFloatVal(RHS.APFloatVal), ConstantVal(RHS.ConstantVal), + NoCFI(RHS.NoCFI) { assert(!RHS.ConstantStructElts); } diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h --- a/llvm/include/llvm/AsmParser/LLToken.h +++ b/llvm/include/llvm/AsmParser/LLToken.h @@ -370,6 +370,7 @@ kw_insertvalue, kw_blockaddress, kw_dso_local_equivalent, + kw_no_cfi, kw_freeze, diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -381,9 +381,10 @@ CST_CODE_CE_UNOP = 25, // CE_UNOP: [opcode, opval] CST_CODE_POISON = 26, // POISON CST_CODE_DSO_LOCAL_EQUIVALENT = 27, // DSO_LOCAL_EQUIVALENT [gvty, gv] - CST_CODE_INLINEASM = 28, // INLINEASM: [sideeffect|alignstack| - // asmdialect|unwind, - // asmstr,conststr] + CST_CODE_INLINEASM = 28, // INLINEASM: [sideeffect|alignstack| + // asmdialect|unwind, + // asmstr,conststr] + CST_CODE_NO_CFI_VALUE = 29, // NO_CFI [ fty, f ] }; /// CastOpcodes - These are values used in the bitcode files to encode which diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h --- a/llvm/include/llvm/IR/Constants.h +++ b/llvm/include/llvm/IR/Constants.h @@ -926,6 +926,41 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(DSOLocalEquivalent, Value) +/// Wrapper for a value that won't be replaced with a CFI jump table +/// pointer in LowerTypeTestsModule. +class NoCFIValue final : public Constant { + friend class Constant; + + NoCFIValue(GlobalValue *GV); + + void *operator new(size_t S) { return User::operator new(S, 1); } + + void destroyConstantImpl(); + Value *handleOperandChangeImpl(Value *From, Value *To); + +public: + /// Return a NoCFIValue for the specified function. + static NoCFIValue *get(GlobalValue *GV); + + /// Transparently provide more efficient getOperand methods. + DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + + GlobalValue *getGlobalValue() const { + return cast(Op<0>().get()); + } + + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const Value *V) { + return V->getValueID() == NoCFIValueVal; + } +}; + +template <> +struct OperandTraits : public FixedNumOperandTraits { +}; + +DEFINE_TRANSPARENT_OPERAND_ACCESSORS(NoCFIValue, Value) + //===----------------------------------------------------------------------===// /// A constant value that is initialized with an expression using /// other constant values. diff --git a/llvm/include/llvm/IR/Value.def b/llvm/include/llvm/IR/Value.def --- a/llvm/include/llvm/IR/Value.def +++ b/llvm/include/llvm/IR/Value.def @@ -80,6 +80,7 @@ HANDLE_CONSTANT(BlockAddress) HANDLE_CONSTANT(ConstantExpr) HANDLE_CONSTANT_EXCLUDE_LLVM_C_API(DSOLocalEquivalent) +HANDLE_CONSTANT_EXCLUDE_LLVM_C_API(NoCFIValue) // ConstantAggregate. HANDLE_CONSTANT(ConstantArray) diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp --- a/llvm/lib/AsmParser/LLLexer.cpp +++ b/llvm/lib/AsmParser/LLLexer.cpp @@ -733,6 +733,7 @@ KEYWORD(x); KEYWORD(blockaddress); KEYWORD(dso_local_equivalent); + KEYWORD(no_cfi); // Metadata types. KEYWORD(distinct); diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -3286,6 +3286,20 @@ return false; } + case lltok::kw_no_cfi: { + // ValID ::= 'no_cfi' @foo + Lex.Lex(); + + if (parseValID(ID, PFS)) + return true; + + if (ID.Kind != ValID::t_GlobalID && ID.Kind != ValID::t_GlobalName) + return error(ID.Loc, "expected global value name in no_cfi"); + + ID.NoCFI = true; + return false; + } + case lltok::kw_trunc: case lltok::kw_zext: case lltok::kw_sext: @@ -5266,9 +5280,13 @@ } case ValID::t_GlobalName: V = getGlobalVal(ID.StrVal, Ty, ID.Loc); + if (V && ID.NoCFI) + V = NoCFIValue::get(cast(V)); return V == nullptr; case ValID::t_GlobalID: V = getGlobalVal(ID.UIntVal, Ty, ID.Loc); + if (V && ID.NoCFI) + V = NoCFIValue::get(cast(V)); return V == nullptr; case ValID::t_APSInt: if (!Ty->isIntegerTy()) diff --git a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp --- a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp @@ -219,6 +219,7 @@ STRINGIFY_CODE(CST_CODE, CE_SHUFVEC_EX) STRINGIFY_CODE(CST_CODE, CE_UNOP) STRINGIFY_CODE(CST_CODE, DSO_LOCAL_EQUIVALENT) + STRINGIFY_CODE(CST_CODE, NO_CFI_VALUE) case bitc::CST_CODE_BLOCKADDRESS: return "CST_CODE_BLOCKADDRESS"; STRINGIFY_CODE(CST_CODE, DATA) diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -2942,6 +2942,19 @@ V = DSOLocalEquivalent::get(GV); break; } + case bitc::CST_CODE_NO_CFI_VALUE: { + if (Record.size() < 2) + return error("Invalid record"); + Type *GVTy = getTypeByID(Record[0]); + if (!GVTy) + return error("Invalid record"); + GlobalValue *GV = dyn_cast_or_null( + ValueList.getConstantFwdRef(Record[1], GVTy)); + if (!GV) + return error("Invalid record"); + V = NoCFIValue::get(GV); + break; + } } ValueList.assignValue(V, NextCstNo); diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -2659,6 +2659,10 @@ Code = bitc::CST_CODE_DSO_LOCAL_EQUIVALENT; Record.push_back(VE.getTypeID(Equiv->getGlobalValue()->getType())); Record.push_back(VE.getValueID(Equiv->getGlobalValue())); + } else if (const auto *NC = dyn_cast(C)) { + Code = bitc::CST_CODE_NO_CFI_VALUE; + Record.push_back(VE.getTypeID(NC->getGlobalValue()->getType())); + Record.push_back(VE.getValueID(NC->getGlobalValue())); } else { #ifndef NDEBUG C->dump(); diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -2504,6 +2504,9 @@ if (const auto *Equiv = dyn_cast(CV)) return getObjFileLowering().lowerDSOLocalEquivalent(Equiv, TM); + if (const NoCFIValue *NC = dyn_cast(CV)) + return MCSymbolRefExpr::create(getSymbol(NC->getGlobalValue()), Ctx); + const ConstantExpr *CE = dyn_cast(CV); if (!CE) { llvm_unreachable("Unknown constant value to lower!"); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -1626,6 +1626,9 @@ if (const auto *Equiv = dyn_cast(C)) return getValue(Equiv->getGlobalValue()); + if (const auto *NC = dyn_cast(C)) + return getValue(NC->getGlobalValue()); + VectorType *VecTy = cast(V->getType()); // Now that we know the number and type of the elements, get that number of diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -1474,6 +1474,12 @@ return; } + if (const auto *NC = dyn_cast(CV)) { + Out << "no_cfi "; + WriteAsOperandInternal(Out, NC->getGlobalValue(), WriterCtx); + return; + } + if (const ConstantArray *CA = dyn_cast(CV)) { Type *ETy = CA->getType()->getElementType(); Out << '['; diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp --- a/llvm/lib/IR/Constants.cpp +++ b/llvm/lib/IR/Constants.cpp @@ -535,6 +535,9 @@ case Constant::DSOLocalEquivalentVal: delete static_cast(C); break; + case Constant::NoCFIValueVal: + delete static_cast(C); + break; case Constant::UndefValueVal: delete static_cast(C); break; @@ -1962,6 +1965,49 @@ return nullptr; } +NoCFIValue *NoCFIValue::get(GlobalValue *GV) { + NoCFIValue *&NC = GV->getContext().pImpl->NoCFIValues[GV]; + if (!NC) + NC = new NoCFIValue(GV); + + assert(NC->getGlobalValue() == GV && + "NoCFIValue does not match the expected global value"); + return NC; +} + +NoCFIValue::NoCFIValue(GlobalValue *GV) + : Constant(GV->getType(), Value::NoCFIValueVal, &Op<0>(), 1) { + setOperand(0, GV); +} + +/// Remove the constant from the constant table. +void NoCFIValue::destroyConstantImpl() { + const GlobalValue *GV = getGlobalValue(); + GV->getContext().pImpl->NoCFIValues.erase(GV); +} + +Value *NoCFIValue::handleOperandChangeImpl(Value *From, Value *To) { + assert(From == getGlobalValue() && "Changing value does not match operand."); + + auto *ToObj = To->stripPointerCasts(); + assert(isa(ToObj) && + "Can only replace the operands with a global value"); + + auto *GV = cast(ToObj); + NoCFIValue *&NewNC = getContext().pImpl->NoCFIValues[GV]; + if (NewNC) + return llvm::ConstantExpr::getBitCast(NewNC, getType()); + + getContext().pImpl->NoCFIValues.erase(getGlobalValue()); + NewNC = this; + setOperand(0, GV); + + if (GV->getType() != getType()) + mutateType(GV->getType()); + + return nullptr; +} + //---- ConstantExpr::get() implementations. // diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h --- a/llvm/lib/IR/LLVMContextImpl.h +++ b/llvm/lib/IR/LLVMContextImpl.h @@ -1433,6 +1433,8 @@ DenseMap DSOLocalEquivalents; + DenseMap NoCFIValues; + ConstantUniqueMap ExprConstants; ConstantUniqueMap InlineAsms; diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp --- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp +++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp @@ -1774,8 +1774,9 @@ bool IsJumpTableCanonical) { SmallSetVector Constants; for (Use &U : llvm::make_early_inc_range(Old->uses())) { - // Skip block addresses - if (isa(U.getUser())) + // Skip block addresses and no_cfi values, which refer to the function + // body instead of the jump table. + if (isa(U.getUser())) continue; // Skip direct calls to externally defined or non-dso_local functions diff --git a/llvm/lib/Transforms/Utils/ValueMapper.cpp b/llvm/lib/Transforms/Utils/ValueMapper.cpp --- a/llvm/lib/Transforms/Utils/ValueMapper.cpp +++ b/llvm/lib/Transforms/Utils/ValueMapper.cpp @@ -450,6 +450,12 @@ DSOLocalEquivalent::get(Func), NewTy); } + if (const auto *NC = dyn_cast(C)) { + auto *Val = mapValue(NC->getGlobalValue()); + GlobalValue *GV = cast(Val); + return getVM()[NC] = NoCFIValue::get(GV); + } + auto mapValueOrNull = [this](Value *V) { auto Mapped = mapValue(V); assert((Mapped || (Flags & RF_NullMapMissingGlobalValues)) && diff --git a/llvm/test/Bitcode/nocfivalue.ll b/llvm/test/Bitcode/nocfivalue.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Bitcode/nocfivalue.ll @@ -0,0 +1,43 @@ +; RUN: llvm-as < %s | llvm-dis | FileCheck %s +; RUN: verify-uselistorder %s + +; CHECK: @a = global [4 x void ()*] [void ()* no_cfi @f1, void ()* @f1, void ()* @f2, void ()* no_cfi @f2] +@a = global [4 x void ()*] [void ()* no_cfi @f1, void ()* @f1, void ()* @f2, void ()* no_cfi @f2] +; CHECK: @b = constant void ()* no_cfi @f3 +@b = constant void ()* no_cfi @f3 +; CHECK: @c = constant void ()* @f3 +@c = constant void ()* @f3 + +; CHECK: declare void @f1() +declare void @f1() + +; CHECK: declare void @f2() +declare void @f2() + +; CHECK: define void @f3() +define void @f3() { + ; CHECK: call void no_cfi @f4() + call void no_cfi @f4() + ; CHECK: call void @f4() + call void @f4() + ; CHECK: call void no_cfi @f5() + call void no_cfi @f5() + ; CHECK: call void @f5() + call void @f5() + ret void +} + +; CHECK: declare void @f4() +declare void @f4() + +; CHECK: declare void @f5() +declare void @f5() + +define void @g() { + %n = alloca void ()*, align 8 + ; CHECK: store void ()* no_cfi @f5, void ()** %n, align 8 + store void ()* no_cfi @f5, void ()** %n, align 8 + %1 = load void ()*, void ()** %n + call void %1() + ret void +} diff --git a/llvm/test/CodeGen/X86/nocfivalue.ll b/llvm/test/CodeGen/X86/nocfivalue.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/nocfivalue.ll @@ -0,0 +1,37 @@ +; RUN: opt -S -lowertypetests %s | llc -asm-verbose=false | FileCheck %s + +target datalayout = "e-p:64:64" +target triple = "x86_64-unknown-linux-gnu" + +; CHECK: a: +; CHECK-NEXT: .quad f1 +; CHECK-NEXT: .quad .L.cfi.jumptable +; CHECK-NEXT: .quad .L.cfi.jumptable +; CHECK-NEXT: .quad f2 +; CHECK-NEXT: .quad f3 +; CHECK-NEXT: .quad f3.cfi +@a = global [6 x void ()*] [void ()* no_cfi @f1, void ()* @f1, void ()* @f2, void ()* no_cfi @f2, void ()* @f3, void ()* no_cfi @f3] + +declare !type !0 void @f1() + +define internal void @f2() !type !0 { + ret void +} + +define void @f3() #0 !type !0 { + ret void +} + +declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone + +define i1 @foo(i8* %p) { + %x = call i1 @llvm.type.test(i8* %p, metadata !"typeid1") + ret i1 %x +} + +!llvm.module.flags = !{!1} + +attributes #0 = { "cfi-canonical-jump-table" } + +!0 = !{i32 0, !"typeid1"} +!1 = !{i32 4, !"CFI Canonical Jump Tables", i32 0} diff --git a/llvm/test/Transforms/LowerTypeTests/nocfivalue.ll b/llvm/test/Transforms/LowerTypeTests/nocfivalue.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/LowerTypeTests/nocfivalue.ll @@ -0,0 +1,36 @@ +; RUN: opt -S -lowertypetests < %s | FileCheck %s + +target datalayout = "e-p:64:64" +target triple = "x86_64-unknown-linux-gnu" + +; CHECK: @a = global [6 x void ()*] [void ()* no_cfi @f1, void ()* @.cfi.jumptable, void ()* bitcast ([8 x i8]* getelementptr inbounds ([3 x [8 x i8]], [3 x [8 x i8]]* bitcast (void ()* @.cfi.jumptable to [3 x [8 x i8]]*), i64 0, i64 1) to void ()*), void ()* no_cfi @f2, void ()* @f3, void ()* no_cfi @f3.cfi] +@a = global [6 x void ()*] [void ()* no_cfi @f1, void ()* @f1, void ()* @f2, void ()* no_cfi @f2, void ()* @f3, void ()* no_cfi @f3] + +; CHECK: define void @f1() +define void @f1() !type !0 { + ret void +} + +; CHECK: define internal void @f2() +define internal void @f2() !type !0 { + ret void +} + +; CHECK: define hidden void @f3.cfi() +define void @f3() #0 !type !0 { + ret void +} + +declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone + +define i1 @foo(i8* %p) { + %x = call i1 @llvm.type.test(i8* %p, metadata !"typeid1") + ret i1 %x +} + +!llvm.module.flags = !{!1} + +attributes #0 = { "cfi-canonical-jump-table" } + +!0 = !{i32 0, !"typeid1"} +!1 = !{i32 4, !"CFI Canonical Jump Tables", i32 0} diff --git a/llvm/utils/emacs/llvm-mode.el b/llvm/utils/emacs/llvm-mode.el --- a/llvm/utils/emacs/llvm-mode.el +++ b/llvm/utils/emacs/llvm-mode.el @@ -67,7 +67,7 @@ ;; Runtime preemption specifiers "dso_preemptable" "dso_local" "dso_local_equivalent" - "gc" "atomic" "volatile" "personality" "prologue" "section") 'symbols) . font-lock-keyword-face) + "gc" "atomic" "no_cfi" "volatile" "personality" "prologue" "section") 'symbols) . font-lock-keyword-face) ;; Arithmetic and Logical Operators `(,(regexp-opt '("add" "sub" "mul" "sdiv" "udiv" "urem" "srem" "and" "or" "xor" "setne" "seteq" "setlt" "setgt" "setle" "setge") 'symbols) . font-lock-keyword-face) diff --git a/llvm/utils/vim/syntax/llvm.vim b/llvm/utils/vim/syntax/llvm.vim --- a/llvm/utils/vim/syntax/llvm.vim +++ b/llvm/utils/vim/syntax/llvm.vim @@ -124,6 +124,7 @@ \ nocallback \ nocapture \ nocf_check + \ no_cfi \ noduplicate \ nofree \ noimplicitfloat