diff --git a/clang/docs/ClangCommandLineReference.rst b/clang/docs/ClangCommandLineReference.rst --- a/clang/docs/ClangCommandLineReference.rst +++ b/clang/docs/ClangCommandLineReference.rst @@ -979,6 +979,10 @@ Generalize pointers in CFI indirect call type signature checks +.. option:: -fsanitize-kcfi-offset= + +Set the number of bytes between the KCFI type identifier and the start of the function. + .. option:: -fsanitize-coverage-allowlist=, -fsanitize-coverage-whitelist= Restrict sanitizer coverage instrumentation exclusively to modules and functions that match the provided special case list, except the blocked ones diff --git a/clang/docs/ControlFlowIntegrity.rst b/clang/docs/ControlFlowIntegrity.rst --- a/clang/docs/ControlFlowIntegrity.rst +++ b/clang/docs/ControlFlowIntegrity.rst @@ -306,6 +306,19 @@ library boundaries are no different from calls within a single program or shared library. +.. _kcfi: + +``-fsanitize=kcfi`` +------------------- + +KCFI, enabled by ``-fsanitize=kcfi``, is an alternative indirect call +control-flow integrity scheme designed for low-level system software, such +as operating system kernels. Unlike ``-fsanitize=cfi-icall``, it doesn't +require ``-flto``, won't result in function pointers being replaced with jump +table references, and never breaks cross-DSO function address equality. These +properties make KCFI easier to adopt in low-level software. KCFI is limited to +indirect call checking only, and isn't compatible with executable-only memory. + Member Function Pointer Call Checking ===================================== diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst --- a/clang/docs/UsersManual.rst +++ b/clang/docs/UsersManual.rst @@ -1692,6 +1692,8 @@ flow analysis. - ``-fsanitize=cfi``: :doc:`control flow integrity ` checks. Requires ``-flto``. + - ``-fsanitize=kcfi``: kernel indirect call forward-edge control flow + integrity. - ``-fsanitize=safe-stack``: :doc:`safe stack ` protection against stack-based memory corruption errors. diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def --- a/clang/include/clang/Basic/Features.def +++ b/clang/include/clang/Basic/Features.def @@ -227,6 +227,7 @@ FEATURE(is_trivially_copyable, LangOpts.CPlusPlus) FEATURE(is_union, LangOpts.CPlusPlus) FEATURE(modules, LangOpts.Modules) +FEATURE(kcfi, LangOpts.Sanitize.has(SanitizerKind::KCFI)) FEATURE(safe_stack, LangOpts.Sanitize.has(SanitizerKind::SafeStack)) FEATURE(shadow_call_stack, LangOpts.Sanitize.has(SanitizerKind::ShadowCallStack)) diff --git a/clang/include/clang/Basic/Sanitizers.def b/clang/include/clang/Basic/Sanitizers.def --- a/clang/include/clang/Basic/Sanitizers.def +++ b/clang/include/clang/Basic/Sanitizers.def @@ -126,6 +126,9 @@ CFIDerivedCast | CFIICall | CFIMFCall | CFIUnrelatedCast | CFINVCall | CFIVCall) +// Kernel Control Flow Integrity +SANITIZER("kcfi", KCFI) + // Safe Stack SANITIZER("safe-stack", SafeStack) diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -3227,6 +3227,11 @@ #undef SANITIZER_CHECK }; +void CodeGenFunction::EmitKCFICheck(llvm::Value *Ptr, llvm::ConstantInt *Hash) { + Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::kcfi_check), + {Ptr, Hash}); +} + static void emitCheckHandlerCall(CodeGenFunction &CGF, llvm::FunctionType *FnType, ArrayRef FnArgs, @@ -5355,11 +5360,11 @@ } const auto *FnType = cast(PointeeType); + bool IsIndirectCall = !TargetDecl || !isa(TargetDecl); // If we are checking indirect calls and this call is indirect, check that the // function pointer is a member of the bit set for the function type. - if (SanOpts.has(SanitizerKind::CFIICall) && - (!TargetDecl || !isa(TargetDecl))) { + if (SanOpts.has(SanitizerKind::CFIICall) && IsIndirectCall) { SanitizerScope SanScope(this); EmitSanitizerStatReport(llvm::SanStat_CFI_ICall); @@ -5392,6 +5397,10 @@ } } + if (SanOpts.has(SanitizerKind::KCFI) && IsIndirectCall) + EmitKCFICheck(Callee.getFunctionPointer(), + CGM.CreateKCFITypeId(QualType(FnType, 0))); + CallArgList Args; if (Chain) Args.add(RValue::get(Builder.CreateBitCast(Chain, CGM.VoidPtrTy)), diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -4604,6 +4604,8 @@ /// passing to a runtime sanitizer handler. llvm::Constant *EmitCheckSourceLocation(SourceLocation Loc); + void EmitKCFICheck(llvm::Value *Ptr, llvm::ConstantInt *Hash); + /// Create a basic block that will either trap or call a handler function in /// the UBSan runtime with the provided arguments, and create a conditional /// branch to it. diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h --- a/clang/lib/CodeGen/CodeGenModule.h +++ b/clang/lib/CodeGen/CodeGenModule.h @@ -1393,6 +1393,9 @@ /// Generate a cross-DSO type identifier for MD. llvm::ConstantInt *CreateCrossDsoCfiTypeId(llvm::Metadata *MD); + /// Generate a KCFI type identifier for T. + llvm::ConstantInt *CreateKCFITypeId(QualType T); + /// Create a metadata identifier for the given type. This may either be an /// MDString (for external identifiers) or a distinct unnamed MDNode (for /// internal identifiers). @@ -1411,6 +1414,12 @@ void CreateFunctionTypeMetadataForIcall(const FunctionDecl *FD, llvm::Function *F); + /// Set type hash as prefix data to the given function + void SetKCFITypePrefix(const FunctionDecl *FD, llvm::Function *F); + + /// Emit KCFI type identifier constants and remove unused identifiers + void FinalizeKCFITypePrefixes(); + /// Whether this function's return type has no side effects, and thus may /// be trivially discarded if it is unused. bool MayDropFunctionReturn(const ASTContext &Context, QualType ReturnType); @@ -1652,7 +1661,8 @@ llvm::AttrBuilder &FuncAttrs); llvm::Metadata *CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map, - StringRef Suffix); + StringRef Suffix, + bool OnlyExternal = true); }; } // end namespace CodeGen diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -47,6 +47,7 @@ #include "clang/CodeGen/BackendUtil.h" #include "clang/CodeGen/ConstantInitBuilder.h" #include "clang/Frontend/FrontendDiagnostic.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/ADT/Triple.h" #include "llvm/Analysis/TargetLibraryInfo.h" @@ -65,6 +66,7 @@ #include "llvm/Support/MD5.h" #include "llvm/Support/TimeProfiler.h" #include "llvm/Support/X86TargetParser.h" +#include "llvm/Support/xxhash.h" using namespace clang; using namespace CodeGen; @@ -554,6 +556,8 @@ CodeGenFunction(*this).EmitCfiCheckFail(); CodeGenFunction(*this).EmitCfiCheckStub(); } + if (LangOpts.Sanitize.has(SanitizerKind::KCFI)) + FinalizeKCFITypePrefixes(); emitAtAvailableLinkGuard(); if (Context.getTargetInfo().getTriple().isWasm() && !Context.getTargetInfo().getTriple().isOSEmscripten()) { @@ -1641,6 +1645,15 @@ return llvm::ConstantInt::get(Int64Ty, llvm::MD5Hash(MDS->getString())); } +llvm::ConstantInt *CodeGenModule::CreateKCFITypeId(QualType T) { + if (auto *MDS = dyn_cast(CreateMetadataIdentifierImpl( + T, MetadataIdMap, "", /*OnlyExternal=*/false))) + return llvm::ConstantInt::get( + Int32Ty, static_cast(llvm::xxHash64(MDS->getString()))); + + return nullptr; +} + void CodeGenModule::SetLLVMFunctionAttributes(GlobalDecl GD, const CGFunctionInfo &Info, llvm::Function *F, bool IsThunk) { @@ -2234,6 +2247,60 @@ F->addTypeMetadata(0, llvm::ConstantAsMetadata::get(CrossDsoTypeId)); } +void CodeGenModule::SetKCFITypePrefix(const FunctionDecl *FD, + llvm::Function *F) { + + if (isa(FD) && !cast(FD)->isStatic()) + return; + + F->setPrefixData(CreateKCFITypeId(FD->getType())); + F->addFnAttr("kcfi"); +} + +static bool allowKCFIIdentifier(StringRef Name) { + // KCFI type identifier constants are only necessary for external assembly + // functions, which means it's safe to skip unusual names. Subset of + // MCAsmInfo::isAcceptableChar() and MCAsmInfoXCOFF::isAcceptableChar(). + for (const char &C : Name) { + if (llvm::isAlnum(C) || C == '_' || C == '.') + continue; + return false; + } + return true; +} + +void CodeGenModule::FinalizeKCFITypePrefixes() { + llvm::Module &M = getModule(); + for (auto &F : M.functions()) { + bool AddressTaken = F.hasAddressTaken(); + + // Remove KCFI prefix data and attribute from non-address-taken local + // functions. + if (!AddressTaken && F.hasLocalLinkage()) { + F.setPrefixData(nullptr); + F.removeFnAttr("kcfi"); + } + + if (!AddressTaken || !F.isDeclaration() || !F.hasPrefixData()) + continue; + + // Generate a weak constant with the expected KCFI type identifier for all + // address-taken function declarations. + auto *Id = dyn_cast(F.getPrefixData()); + if (!Id) + continue; + + StringRef Name = F.getName(); + if (!allowKCFIIdentifier(Name)) + continue; + + std::string Asm = (".weak __kcfi_typeid_" + Name + "\n.set __kcfi_typeid_" + + Name + ", " + Twine(Id->getSExtValue()) + "\n") + .str(); + M.appendModuleInlineAsm(Asm); + } +} + void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F, bool IsIncompleteFunction, bool IsThunk) { @@ -2316,6 +2383,9 @@ !CodeGenOpts.SanitizeCfiCanonicalJumpTables) CreateFunctionTypeMetadataForIcall(FD, F); + if (LangOpts.Sanitize.has(SanitizerKind::KCFI)) + SetKCFITypePrefix(FD, F); + if (getLangOpts().OpenMP && FD->hasAttr()) getOpenMPRuntime().emitDeclareSimdFunction(FD, F); @@ -6600,7 +6670,8 @@ llvm::Metadata * CodeGenModule::CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map, - StringRef Suffix) { + StringRef Suffix, + bool OnlyExternal /*=true*/) { if (auto *FnType = T->getAs()) T = getContext().getFunctionType( FnType->getReturnType(), FnType->getParamTypes(), @@ -6610,7 +6681,7 @@ if (InternalId) return InternalId; - if (isExternallyVisible(T->getLinkage())) { + if (isExternallyVisible(T->getLinkage()) || !OnlyExternal) { std::string OutName; llvm::raw_string_ostream Out(OutName); getCXXABI().getMangleContext().mangleTypeName(T, Out); diff --git a/clang/lib/Driver/SanitizerArgs.cpp b/clang/lib/Driver/SanitizerArgs.cpp --- a/clang/lib/Driver/SanitizerArgs.cpp +++ b/clang/lib/Driver/SanitizerArgs.cpp @@ -37,7 +37,8 @@ static const SanitizerMask NotAllowedWithMinimalRuntime = SanitizerKind::Function | SanitizerKind::Vptr; static const SanitizerMask RequiresPIE = - SanitizerKind::DataFlow | SanitizerKind::HWAddress | SanitizerKind::Scudo; + SanitizerKind::DataFlow | SanitizerKind::HWAddress | SanitizerKind::Scudo | + SanitizerKind::KCFI; static const SanitizerMask NeedsUnwindTables = SanitizerKind::Address | SanitizerKind::HWAddress | SanitizerKind::Thread | SanitizerKind::Memory | SanitizerKind::DataFlow; @@ -58,8 +59,9 @@ SanitizerKind::FloatDivideByZero | SanitizerKind::ObjCCast; static const SanitizerMask Unrecoverable = SanitizerKind::Unreachable | SanitizerKind::Return; -static const SanitizerMask AlwaysRecoverable = - SanitizerKind::KernelAddress | SanitizerKind::KernelHWAddress; +static const SanitizerMask AlwaysRecoverable = SanitizerKind::KernelAddress | + SanitizerKind::KernelHWAddress | + SanitizerKind::KCFI; static const SanitizerMask NeedsLTO = SanitizerKind::CFI; static const SanitizerMask TrappingSupported = (SanitizerKind::Undefined & ~SanitizerKind::Vptr) | SanitizerKind::Integer | @@ -692,6 +694,25 @@ options::OPT_fno_sanitize_cfi_canonical_jump_tables, true); } + if (AllAddedKinds & SanitizerKind::KCFI && DiagnoseErrors) { + if (AllAddedKinds & SanitizerKind::CFI) + D.Diag(diag::err_drv_argument_not_allowed_with) + << "-fsanitize=kcfi" + << lastArgumentForMask(D, Args, SanitizerKind::CFI); + + if (Arg *A = Args.getLastArg(options::OPT_fpatchable_function_entry_EQ)) { + StringRef S = A->getValue(); + unsigned N, M; + // With -fpatchable-function-entry=N,M, where M > 0, + // llvm::AsmPrinter::emitFunctionHeader injects nops before before the + // KCFI type identifier, which is currently unsupported. + if (!S.consumeInteger(10, N) && S.consume_front(",") && + !S.consumeInteger(10, M) && M > 0) + D.Diag(diag::err_drv_argument_not_allowed_with) + << "-fsanitize=kcfi" << A->getAsString(Args); + } + } + Stats = Args.hasFlag(options::OPT_fsanitize_stats, options::OPT_fno_sanitize_stats, false); diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp --- a/clang/lib/Driver/ToolChain.cpp +++ b/clang/lib/Driver/ToolChain.cpp @@ -1036,6 +1036,9 @@ getTriple().getArch() == llvm::Triple::arm || getTriple().isWasm() || getTriple().isAArch64()) Res |= SanitizerKind::CFIICall; + if (getTriple().getArch() == llvm::Triple::x86_64 || + getTriple().isAArch64(64)) + Res |= SanitizerKind::KCFI; if (getTriple().getArch() == llvm::Triple::x86_64 || getTriple().isAArch64(64) || getTriple().isRISCV()) Res |= SanitizerKind::ShadowCallStack; diff --git a/clang/test/CodeGen/kcfi.c b/clang/test/CodeGen/kcfi.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/kcfi.c @@ -0,0 +1,57 @@ +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fsanitize=kcfi -o - %s | FileCheck --check-prefixes=CHECK %s +#if !__has_feature(kcfi) +#error Missing kcfi? +#endif + +// CHECK: module asm ".weak __kcfi_typeid_f4" +// CHECK: module asm ".set __kcfi_typeid_f4, [[#]]" + +typedef int (*fn_t)(void); + +// CHECK: define dso_local i32 @f1() #[[#ATTR:]] prefix i32 [[#%d,HASH:]] +int f1(void) { return 0; } + +// CHECK: define dso_local i32 @f2() #[[#ATTR]] prefix i32 [[#%d,HASH2:]] +unsigned int f2(void) { return 2; } + +// CHECK-LABEL: define dso_local i32 @__call(ptr{{.*}} %f) +int __call(fn_t f) __attribute__((__no_sanitize__("kcfi"))) { + // CHECK-NOT: call void @llvm.kcfi_check + // CHECK: %call = call i32 %[[#]]() + return f(); +} + +// CHECK-LABEL: define dso_local i32 @call(ptr{{.*}} %f) +int call(fn_t f) { + // CHECK: call void @llvm.kcfi.check(ptr %[[#]], i32 [[#HASH]]) + // CHECK: %call = call i32 %[[#]]() + return f(); +} + +// CHECK-DAG: define internal i32 @f3() #[[#ATTR]] prefix i32 [[#HASH]] +static int f3(void) { return 1; } + +// CHECK-DAG: declare i32 @f4() #[[#DECLATTR:]] prefix i32 [[#HASH]] +extern int f4(void); + +// CHECK-DAG: declare void @llvm.kcfi.check(ptr, i32 immarg) + +// CHECK: define internal i32 @f5() #[[#LOCALATTR:]] +// CHECK-NOT: prefix i32 +// CHECK-SAME: { +static int f5(void) { return 2; } + +int test(void) { + return call(f1) + + __call((fn_t)f2) + + call(f3) + + call(f4) + + f5(); +} + +// CHECK: attributes #[[#ATTR]] = {{{.*}}"kcfi" +// CHECK: attributes #[[#DECLATTR]] = {{{.*}}"kcfi" + +// CHECK: attributes #[[#LOCALATTR]] = { +// CHECK-NOT: {{.*}}"kcfi" +// CHECK-SAME: } diff --git a/clang/test/Driver/fsanitize.c b/clang/test/Driver/fsanitize.c --- a/clang/test/Driver/fsanitize.c +++ b/clang/test/Driver/fsanitize.c @@ -647,6 +647,27 @@ // RUN: %clang -target x86_64-linux-gnu -fsanitize=cfi -fsanitize-stats -flto -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CFI-STATS // CHECK-CFI-STATS: -fsanitize-stats +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi -fsanitize=cfi -flto -fvisibility=hidden %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI-NOCFI +// CHECK-KCFI-NOCFI: error: invalid argument '-fsanitize=kcfi' not allowed with '-fsanitize=cfi' + +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi -fpatchable-function-entry=1 %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI-PATCHABLE-NOM +// CHECK-KCFI-PATCHABLE-NOM: "-fsanitize=kcfi" + +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi -fpatchable-function-entry=1,0 %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI-PATCHABLE-M0 +// CHECK-KCFI-PATCHABLE-M0: "-fsanitize=kcfi" + +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi -fpatchable-function-entry=1,1 %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI-PATCHABLE-M1 +// CHECK-KCFI-PATCHABLE-M1: error: invalid argument '-fsanitize=kcfi' not allowed with '-fpatchable-function-entry=1,1' + +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi -fsanitize-trap=kcfi %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI-NOTRAP +// CHECK-KCFI-NOTRAP: error: unsupported argument 'kcfi' to option '-fsanitize-trap=' + +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI +// CHECK-KCFI: "-fsanitize=kcfi" + +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi -fno-sanitize-recover=kcfi %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI-RECOVER +// CHECK-KCFI-RECOVER: error: unsupported argument 'kcfi' to option '-fno-sanitize-recover=' + // RUN: %clang_cl -fsanitize=address -c -MDd -### -- %s 2>&1 | FileCheck %s -check-prefix=CHECK-ASAN-DEBUGRTL // RUN: %clang_cl -fsanitize=address -c -MTd -### -- %s 2>&1 | FileCheck %s -check-prefix=CHECK-ASAN-DEBUGRTL // RUN: %clang_cl -fsanitize=address -c -LDd -### -- %s 2>&1 | FileCheck %s -check-prefix=CHECK-ASAN-DEBUGRTL diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h --- a/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -400,6 +400,9 @@ void emitBBAddrMapSection(const MachineFunction &MF); + void emitKCFITrapEntry(const MachineFunction &MF, const MCSymbol *Symbol); + virtual void emitKCFITypeId(const MachineFunction &MF); + void emitPseudoProbe(const MachineInstr &MI); void emitRemarksSection(remarks::RemarkStreamer &RS); diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1766,6 +1766,9 @@ def int_load_relative: DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_anyint_ty], [IntrReadMem, IntrArgMemOnly]>; +def int_kcfi_check : + Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty], [ImmArg>]>; + def int_asan_check_memaccess : Intrinsic<[],[llvm_ptr_ty, llvm_i32_ty], [ImmArg>]>; diff --git a/llvm/include/llvm/MC/MCObjectFileInfo.h b/llvm/include/llvm/MC/MCObjectFileInfo.h --- a/llvm/include/llvm/MC/MCObjectFileInfo.h +++ b/llvm/include/llvm/MC/MCObjectFileInfo.h @@ -16,6 +16,7 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/Triple.h" #include "llvm/BinaryFormat/Swift.h" +#include "llvm/MC/MCSection.h" #include "llvm/Support/VersionTuple.h" #include @@ -356,6 +357,8 @@ MCSection *getBBAddrMapSection(const MCSection &TextSec) const; + MCSection *getKCFITrapSection(const MCSection &TextSec) const; + MCSection *getPseudoProbeSection(const MCSection *TextSec) const; MCSection *getPseudoProbeDescSection(StringRef FuncName) const; diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -924,21 +924,26 @@ // Emit the prefix data. if (F.hasPrefixData()) { - if (MAI->hasSubsectionsViaSymbols()) { - // Preserving prefix data on platforms which use subsections-via-symbols - // is a bit tricky. Here we introduce a symbol for the prefix data - // and use the .alt_entry attribute to mark the function's real entry point - // as an alternative entry point to the prefix-data symbol. - MCSymbol *PrefixSym = OutContext.createLinkerPrivateTempSymbol(); - OutStreamer->emitLabel(PrefixSym); + bool SubsectionsViaSymbols = MAI->hasSubsectionsViaSymbols(); + + if (F.hasFnAttribute("kcfi")) + emitKCFITypeId(*MF); + else { + if (SubsectionsViaSymbols) { + // Preserving prefix data on platforms which use subsections-via-symbols + // is a bit tricky. Here we introduce a symbol for the prefix data + // and use the .alt_entry attribute to mark the function's real entry + // point as an alternative entry point to the prefix-data symbol. + MCSymbol *PrefixSym = OutContext.createLinkerPrivateTempSymbol(); + OutStreamer->emitLabel(PrefixSym); + } emitGlobalConstant(F.getParent()->getDataLayout(), F.getPrefixData()); + } - // Emit an .alt_entry directive for the actual function symbol. + // Emit an .alt_entry directive for the actual function symbol. + if (SubsectionsViaSymbols) OutStreamer->emitSymbolAttribute(CurrentFnSym, MCSA_AltEntry); - } else { - emitGlobalConstant(F.getParent()->getDataLayout(), F.getPrefixData()); - } } // Emit M NOPs for -fpatchable-function-entry=N,M where M>0. We arbitrarily @@ -1326,6 +1331,24 @@ OutStreamer->PopSection(); } +void AsmPrinter::emitKCFITrapEntry(const MachineFunction &MF, + const MCSymbol *Symbol) { + MCSection *Section = + getObjFileLowering().getKCFITrapSection(*MF.getSection()); + + if (Section) { + OutStreamer->PushSection(); + OutStreamer->SwitchSection(Section); + OutStreamer->emitSymbolValue(Symbol, getPointerSize()); + OutStreamer->PopSection(); + } +} + +void AsmPrinter::emitKCFITypeId(const MachineFunction &MF) { + const Function &F = MF.getFunction(); + emitGlobalConstant(F.getParent()->getDataLayout(), F.getPrefixData()); +} + void AsmPrinter::emitPseudoProbe(const MachineInstr &MI) { if (PP) { auto GUID = MI.getOperand(0).getImm(); diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp --- a/llvm/lib/MC/MCObjectFileInfo.cpp +++ b/llvm/lib/MC/MCObjectFileInfo.cpp @@ -1116,6 +1116,25 @@ cast(TextSec.getBeginSymbol())); } +MCSection * +MCObjectFileInfo::getKCFITrapSection(const MCSection &TextSec) const { + if (Ctx->getObjectFileType() != MCContext::IsELF) + return nullptr; + + const MCSectionELF &ElfSec = static_cast(TextSec); + unsigned Flags = ELF::SHF_LINK_ORDER | ELF::SHF_ALLOC | ELF::SHF_WRITE; + StringRef GroupName; + if (const MCSymbol *Group = ElfSec.getGroup()) { + GroupName = Group->getName(); + Flags |= ELF::SHF_GROUP; + } + + return Ctx->getELFSection(".kcfi_traps", ELF::SHT_PROGBITS, Flags, 0, + GroupName, + /*IsComdat=*/true, ElfSec.getUniqueID(), + cast(TextSec.getBeginSymbol())); +} + MCSection * MCObjectFileInfo::getPseudoProbeSection(const MCSection *TextSec) const { if (Ctx->getObjectFileType() == MCContext::IsELF) { diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -111,6 +111,7 @@ typedef std::tuple HwasanMemaccessTuple; std::map HwasanMemaccessSymbols; + void LowerKCFI_CHECK(const MachineInstr &MI); void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI); void emitHwasanMemaccessSymbols(Module &M); @@ -317,6 +318,80 @@ recordSled(CurSled, MI, Kind, 2); } +void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) { + Register Addr = MI.getOperand(0).getReg(); + + if (Addr.id() == AArch64::XZR) { + // Checking XZR makes no sense. Instead of emitting a load, zero X16 + // and use it for the ESR AddrIndex below. + Addr = Register(AArch64::X16); + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs) + .addReg(Addr) + .addReg(AArch64::XZR) + .addReg(AArch64::XZR) + .addImm(0)); + } else { + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDURWi) + .addReg(AArch64::W16) + .addReg(Addr) + .addImm(-4)); + } + + int64_t Type = MI.getOperand(1).getImm(); + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi) + .addReg(AArch64::W17) + .addReg(AArch64::W17) + .addImm(Type & 0xFFFF) + .addImm(0)); + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi) + .addReg(AArch64::W17) + .addReg(AArch64::W17) + .addImm((Type >> 16) & 0xFFFF) + .addImm(16)); + + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSWrs) + .addReg(AArch64::WZR) + .addReg(AArch64::W16) + .addReg(AArch64::W17) + .addImm(0)); + + MCSymbol *Pass = OutContext.createTempSymbol(); + EmitToStreamer(*OutStreamer, + MCInstBuilder(AArch64::Bcc) + .addImm(AArch64CC::EQ) + .addExpr(MCSymbolRefExpr::create(Pass, OutContext))); + + assert(Addr.isPhysical() && + "Unable to encode the target register for the KCFI trap"); + + // The base ESR is 0x8000 and the register information is encoded + // in bits 0-9 as follows: + // - 0-4: n, where the register Xn contains the target address + // - 5-9: m, where the register Wm contains the type hash + // Where n, m are in [0, 30]. + unsigned TypeIndex = AArch64::W17 - AArch64::W0; + unsigned AddrIndex; + + switch (Addr.id()) { + default: + AddrIndex = Addr.id() - AArch64::X0; + break; + case AArch64::FP: + AddrIndex = 29; + break; + case AArch64::LR: + AddrIndex = 30; + break; + } + + assert(AddrIndex < 31 && TypeIndex < 31); + + unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31); + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BRK).addImm(ESR)); + + OutStreamer->emitLabel(Pass); +} + void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) { Register Reg = MI.getOperand(0).getReg(); bool IsShort = @@ -1433,6 +1508,10 @@ LowerPATCHABLE_TAIL_CALL(*MI); return; + case AArch64::KCFI_CHECK: + LowerKCFI_CHECK(*MI); + return; + case AArch64::HWASAN_CHECK_MEMACCESS: case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES: LowerHWASAN_CHECK_MEMACCESS(*MI); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -1368,6 +1368,13 @@ def MOVbaseTLS : Pseudo<(outs GPR64:$dst), (ins), [(set GPR64:$dst, AArch64threadpointer)]>, Sched<[WriteSys]>; +let Defs = [ X16, X17, NZCV ] in { +def KCFI_CHECK : Pseudo< + (outs), (ins GPR64noip:$ptr, i32imm:$type), + [(int_kcfi_check GPR64noip:$ptr, (i32 timm:$type))]>, + Sched<[]>; +} + let Uses = [ X9 ], Defs = [ X16, X17, LR, NZCV ] in { def HWASAN_CHECK_MEMACCESS : Pseudo< (outs), (ins GPR64noip:$ptr, i32imm:$accessinfo), diff --git a/llvm/lib/Target/X86/X86AsmPrinter.h b/llvm/lib/Target/X86/X86AsmPrinter.h --- a/llvm/lib/Target/X86/X86AsmPrinter.h +++ b/llvm/lib/Target/X86/X86AsmPrinter.h @@ -98,6 +98,9 @@ void LowerFENTRY_CALL(const MachineInstr &MI, X86MCInstLower &MCIL); + // KCFI specific lowering for X86. + void LowerKCFI_CHECK(const MachineInstr &MI); + // Address sanitizer specific lowering for X86. void LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI); @@ -151,6 +154,7 @@ bool runOnMachineFunction(MachineFunction &MF) override; void emitFunctionBodyStart() override; void emitFunctionBodyEnd() override; + void emitKCFITypeId(const MachineFunction &MF) override; bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override { return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags; diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp --- a/llvm/lib/Target/X86/X86AsmPrinter.cpp +++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp @@ -33,6 +33,7 @@ #include "llvm/MC/MCCodeEmitter.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" +#include "llvm/MC/MCInstBuilder.h" #include "llvm/MC/MCSectionCOFF.h" #include "llvm/MC/MCSectionELF.h" #include "llvm/MC/MCSectionMachO.h" @@ -108,6 +109,37 @@ } } +void X86AsmPrinter::emitKCFITypeId(const MachineFunction &MF) { + // Emit a function symbol for the type identifier data. + MCSymbol *FnSym = OutContext.getOrCreateSymbol("__cfi_" + MF.getName()); + if (MAI->hasDotTypeDotSizeDirective()) + OutStreamer->emitSymbolAttribute(FnSym, MCSA_ELF_TypeFunction); + OutStreamer->emitLabel(FnSym); + + EmitAndCountInstruction(MCInstBuilder(X86::INT3)); + EmitAndCountInstruction(MCInstBuilder(X86::INT3)); + + // Embed the type hash in a mov instruction. + auto *Hash = cast(MF.getFunction().getPrefixData()); + + EmitAndCountInstruction(MCInstBuilder(X86::MOV32ri) + .addReg(X86::EAX) + .addImm(Hash->getZExtValue())); + + EmitAndCountInstruction(MCInstBuilder(X86::INT3)); + EmitAndCountInstruction(MCInstBuilder(X86::INT3)); + + if (MAI->hasDotTypeDotSizeDirective()) { + MCSymbol *EndSym = OutContext.createTempSymbol("__cfi_func_end"); + OutStreamer->emitLabel(EndSym); + + const MCExpr *SizeExp = MCBinaryExpr::createSub( + MCSymbolRefExpr::create(EndSym, OutContext), + MCSymbolRefExpr::create(FnSym, OutContext), OutContext); + OutStreamer->emitELFSize(FnSym, SizeExp); + } +} + /// PrintSymbolOperand - Print a raw symbol reference operand. This handles /// jump tables, constant pools, global address and external symbols, all of /// which print to a label with various suffixes for relocation types etc. diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td --- a/llvm/lib/Target/X86/X86InstrCompiler.td +++ b/llvm/lib/Target/X86/X86InstrCompiler.td @@ -256,6 +256,17 @@ "#SEH_Epilogue", []>; } +//===----------------------------------------------------------------------===// +// Pseudo instructions used by KCFI. +//===----------------------------------------------------------------------===// +let + Defs = [EFLAGS] in { +def KCFI_CHECK : PseudoI< + (outs), (ins GR64:$ptr, i32imm:$type), + [(int_kcfi_check GR64:$ptr, (i32 timm:$type))]>, + Sched<[]>; +} + //===----------------------------------------------------------------------===// // Pseudo instructions used by address sanitizer. //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp --- a/llvm/lib/Target/X86/X86MCInstLower.cpp +++ b/llvm/lib/Target/X86/X86MCInstLower.cpp @@ -1336,6 +1336,31 @@ .addExpr(Op)); } +void X86AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) { + const MachineFunction &MF = *MI.getMF(); + + EmitAndCountInstruction(MCInstBuilder(X86::CMP32mi) + .addReg(MI.getOperand(0).getReg()) + .addImm(1) + .addReg(X86::NoRegister) + .addImm(-6) + .addReg(X86::NoRegister) + .addImm(MI.getOperand(1).getImm())); + + MCSymbol *Pass = OutContext.createTempSymbol(); + EmitAndCountInstruction( + MCInstBuilder(X86::JCC_1) + .addExpr(MCSymbolRefExpr::create(Pass, OutContext)) + .addImm(X86::COND_E)); + + MCSymbol *Trap = OutContext.createTempSymbol(); + OutStreamer->emitLabel(Trap); + EmitAndCountInstruction(MCInstBuilder(X86::TRAP)); + emitKCFITrapEntry(MF, Trap); + + OutStreamer->emitLabel(Pass); +} + void X86AsmPrinter::LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI) { // FIXME: Make this work on non-ELF. if (!TM.getTargetTriple().isOSBinFormatELF()) { @@ -2618,6 +2643,9 @@ EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget))); return; + case X86::KCFI_CHECK: + return LowerKCFI_CHECK(*MI); + case X86::ASAN_CHECK_MEMACCESS: return LowerASAN_CHECK_MEMACCESS(*MI); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -2673,6 +2673,21 @@ } break; } + case Intrinsic::kcfi_check: { + // If the first argument to llvm.kcfi.check() is known function, and the + // expected hash in the second argument matches the hash in the function + // prefix data, the check will always pass and can be removed. + auto *Target = dyn_cast(CI.getArgOperand(0)->stripPointerCasts()); + + if (Target && Target->hasFnAttribute("kcfi") && Target->hasPrefixData()) { + auto *Hash = cast(Target->getPrefixData()); + auto *Expected = cast(CI.getArgOperand(1)); + + if (Hash->getZExtValue() == Expected->getZExtValue()) + return eraseInstFromFunction(CI); + } + break; + } default: { // Handle target specific intrinsics Optional V = targetInstCombineIntrinsic(*II); diff --git a/llvm/test/CodeGen/AArch64/kcfi.ll b/llvm/test/CodeGen/AArch64/kcfi.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/kcfi.ll @@ -0,0 +1,30 @@ +; RUN: llc -mtriple=aarch64-- < %s | FileCheck %s +; RUN: llc -mtriple=aarch64-- -stop-before=finalize-isel < %s | FileCheck %s --check-prefix=ISEL + +; CHECK: .word 12345678 +define void @f1(ptr noundef %x) #0 prefix i32 12345678 { + +; CHECK-LABEL: f1: +; CHECK: // %bb.0: +; CHECK: ldur w16, [x0, #-4] +; CHECK-NEXT: movk w17, #24910 +; CHECK-NEXT: movk w17, #188, lsl #16 +; CHECK-NEXT: cmp w16, w17 +; CHECK-NEXT: b.eq .Ltmp0 +; CHECK-NEXT: brk #0x8220 +; CHECK-NEXT: .Ltmp0: +; CHECK-NEXT: blr x0 + +; ISEL: name: f1 +; ISEL: body: +; ISEL: KCFI_CHECK %[[#CALL:]], 12345678, implicit-def dead $x16, implicit-def dead $x17, implicit-def dead $nzcv +; ISEL: BLR %[[#CALL]] + + call void @llvm.kcfi.check(ptr %x, i32 12345678) + call void %x() + ret void +} + +declare void @llvm.kcfi.check(ptr, i32 immarg) + +attributes #0 = { "kcfi" } diff --git a/llvm/test/CodeGen/X86/kcfi.ll b/llvm/test/CodeGen/X86/kcfi.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/kcfi.ll @@ -0,0 +1,39 @@ +; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -stop-before=finalize-isel < %s | FileCheck %s --check-prefix=ISEL + +; CHECK: .type __cfi_f1,@function +; CHECK-LABEL: __cfi_f1: +; CHECK-NEXT: int3 +; CHECK-NEXT: int3 +; CHECK-NEXT: movl $12345678, %eax +; CHECK-NEXT: int3 +; CHECK-NEXT: int3 +; CHECK-LABEL: .L__cfi_func_end0: +; CHECK-NEXT: .size __cfi_f1, .L__cfi_func_end0-__cfi_f1 +define void @f1(ptr noundef %x) #0 prefix i32 12345678 { + +; CHECK-LABEL: f1: +; CHECK: # %bb.0: +; CHECK: cmpl $12345678, -6(%rdi) # imm = 0xBC614E +; CHECK-NEXT: je .Ltmp0 +; CHECK-NEXT: .Ltmp1: +; CHECK-NEXT: ud2 +; CHECK-NEXT: .section .kcfi_traps,"awo",@progbits,.text +; CHECK-NEXT: .quad .Ltmp1 +; CHECK-NEXT: .text +; CHECK-NEXT: .Ltmp0: +; CHECK-NEXT: callq *%rdi + +; ISEL: name: f1 +; ISEL: body: +; ISEL: KCFI_CHECK %[[#CALL:]], 12345678, implicit-def dead $eflags +; ISEL: CALL64r %[[#CALL]], csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp + + call void @llvm.kcfi.check(ptr %x, i32 12345678) + call void %x() + ret void +} + +declare void @llvm.kcfi.check(ptr, i32 immarg) + +attributes #0 = { "kcfi" } diff --git a/llvm/test/Transforms/InstCombine/kcfi_check.ll b/llvm/test/Transforms/InstCombine/kcfi_check.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/kcfi_check.ll @@ -0,0 +1,35 @@ +; RUN: opt < %s -passes=instcombine -S | FileCheck %s + +define void @f1() #0 prefix i32 10 { + ret void +} + +declare void @f2() #0 prefix i32 11 + +define internal void @f3() { + ret void +} + +define void @g(ptr noundef %x) { + ; CHECK: call void @llvm.kcfi.check(ptr %x, i32 10) + call void @llvm.kcfi.check(ptr %x, i32 10) + + ; CHECK-NOT: call void @llvm.kcfi.check(ptr nonnull @f1, i32 10) + ; CHECK: call void @llvm.kcfi.check(ptr nonnull @f1, i32 11) + call void @llvm.kcfi.check(ptr nonnull @f1, i32 10) + call void @llvm.kcfi.check(ptr nonnull @f1, i32 11) + + ; CHECK: call void @llvm.kcfi.check(ptr nonnull @f2, i32 10) + ; CHECK-NOT: call void @llvm.kcfi.check(ptr nonnull @f2, i32 11) + call void @llvm.kcfi.check(ptr nonnull @f2, i32 10) + call void @llvm.kcfi.check(ptr nonnull @f2, i32 11) + + ; CHECK: call void @llvm.kcfi.check(ptr nonnull @f3, i32 10) + call void @llvm.kcfi.check(ptr nonnull @f3, i32 10) + ret void +} + +; CHECK: declare void @llvm.kcfi.check(ptr, i32 immarg) +declare void @llvm.kcfi.check(ptr, i32 immarg) + +attributes #0 = { "kcfi" }