diff --git a/clang/docs/ClangCommandLineReference.rst b/clang/docs/ClangCommandLineReference.rst --- a/clang/docs/ClangCommandLineReference.rst +++ b/clang/docs/ClangCommandLineReference.rst @@ -971,6 +971,10 @@ Generalize pointers in CFI indirect call type signature checks +.. option:: -fsanitize-kcfi-offset= + +Set the number of bytes between the KCFI type identifier and the start of the function. + .. option:: -fsanitize-coverage-allowlist=, -fsanitize-coverage-whitelist= Restrict sanitizer coverage instrumentation exclusively to modules and functions that match the provided special case list, except the blocked ones diff --git a/clang/docs/ControlFlowIntegrity.rst b/clang/docs/ControlFlowIntegrity.rst --- a/clang/docs/ControlFlowIntegrity.rst +++ b/clang/docs/ControlFlowIntegrity.rst @@ -306,6 +306,19 @@ library boundaries are no different from calls within a single program or shared library. +.. _kcfi: + +``-fsanitize=kcfi`` +------------------- + +KCFI, enabled by ``-fsanitize=kcfi``, is an alternative indirect call +control-flow integrity scheme designed for low-level system software, such +as operating system kernels. Unlike ``-fsanitize=cfi-icall``, it doesn't +require ``-flto``, won't result in function pointers being replaced with jump +table references, and never breaks cross-DSO function address equality. These +properties make KCFI easier to adopt in low-level software. KCFI is limited to +indirect call checking only, and isn't compatible with executable-only memory. + Member Function Pointer Call Checking ===================================== diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst --- a/clang/docs/UsersManual.rst +++ b/clang/docs/UsersManual.rst @@ -1693,6 +1693,8 @@ flow analysis. - ``-fsanitize=cfi``: :doc:`control flow integrity ` checks. Requires ``-flto``. + - ``-fsanitize=kcfi``: kernel indirect call forward-edge control flow + integrity. - ``-fsanitize=safe-stack``: :doc:`safe stack ` protection against stack-based memory corruption errors. @@ -3965,6 +3967,8 @@ Enable control flow integrity (CFI) checks for cross-DSO calls. -fsanitize-cfi-icall-generalize-pointers Generalize pointers in CFI indirect call type signature checks + -fsanitize-kcfi-offset= + Offset between the KCFI type identifier and the start of a function. -fsanitize-coverage= Specify the type of coverage instrumentation for Sanitizers -fsanitize-hwaddress-abi= diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def --- a/clang/include/clang/Basic/CodeGenOptions.def +++ b/clang/include/clang/Basic/CodeGenOptions.def @@ -249,6 +249,8 @@ ///< CFI icall function signatures CODEGENOPT(SanitizeCfiCanonicalJumpTables, 1, 0) ///< Make jump table symbols canonical ///< instead of creating a local jump table. +VALUE_CODEGENOPT(SanitizeKCFIOffset, 32, 0) ///< Define the KCFI type identifier offset from + ///< the start of the function. CODEGENOPT(SanitizeCoverageType, 2, 0) ///< Type of sanitizer coverage ///< instrumentation. CODEGENOPT(SanitizeCoverageIndirectCalls, 1, 0) ///< Enable sanitizer coverage diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def --- a/clang/include/clang/Basic/Features.def +++ b/clang/include/clang/Basic/Features.def @@ -224,6 +224,7 @@ FEATURE(is_trivially_copyable, LangOpts.CPlusPlus) FEATURE(is_union, LangOpts.CPlusPlus) FEATURE(modules, LangOpts.Modules) +FEATURE(kcfi, LangOpts.Sanitize.has(SanitizerKind::KCFI)) FEATURE(safe_stack, LangOpts.Sanitize.has(SanitizerKind::SafeStack)) FEATURE(shadow_call_stack, LangOpts.Sanitize.has(SanitizerKind::ShadowCallStack)) diff --git a/clang/include/clang/Basic/Sanitizers.def b/clang/include/clang/Basic/Sanitizers.def --- a/clang/include/clang/Basic/Sanitizers.def +++ b/clang/include/clang/Basic/Sanitizers.def @@ -124,6 +124,9 @@ CFIDerivedCast | CFIICall | CFIMFCall | CFIUnrelatedCast | CFINVCall | CFIVCall) +// Kernel Control Flow Integrity +SANITIZER("kcfi", KCFI) + // Safe Stack SANITIZER("safe-stack", SafeStack) diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -1769,6 +1769,11 @@ PosFlag, NegFlag, BothFlags<[], " the jump table addresses canonical in the symbol table">>, Group; +def fsanitize_kcfi_offset + : Joined<["-"], "fsanitize-kcfi-offset=">, MetaVarName<"">, + Group, + HelpText<"Assume there are bytes between the KCFI type identifer and start of the function.">, + MarshallingInfoInt>; defm sanitize_stats : BoolOption<"f", "sanitize-stats", CodeGenOpts<"SanitizeStats">, DefaultFalse, PosFlag, NegFlag, diff --git a/clang/include/clang/Driver/SanitizerArgs.h b/clang/include/clang/Driver/SanitizerArgs.h --- a/clang/include/clang/Driver/SanitizerArgs.h +++ b/clang/include/clang/Driver/SanitizerArgs.h @@ -48,6 +48,7 @@ bool AsanOutlineInstrumentation = false; llvm::AsanDtorKind AsanDtorKind = llvm::AsanDtorKind::Invalid; std::string HwasanAbi; + unsigned int KCFIOffset = 0; bool LinkRuntimes = true; bool LinkCXXRuntimes = false; bool NeedPIE = false; diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -3201,6 +3201,11 @@ #undef SANITIZER_CHECK }; +void CodeGenFunction::EmitKCFICheck(llvm::Value *Ptr, llvm::ConstantInt *Hash) { + Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::kcfi_check), + {Builder.CreateBitCast(Ptr, Int8PtrTy), Hash}); +} + static void emitCheckHandlerCall(CodeGenFunction &CGF, llvm::FunctionType *FnType, ArrayRef FnArgs, @@ -5273,11 +5278,11 @@ } const auto *FnType = cast(PointeeType); + bool IsIndirectCall = !TargetDecl || !isa(TargetDecl); // If we are checking indirect calls and this call is indirect, check that the // function pointer is a member of the bit set for the function type. - if (SanOpts.has(SanitizerKind::CFIICall) && - (!TargetDecl || !isa(TargetDecl))) { + if (SanOpts.has(SanitizerKind::CFIICall) && IsIndirectCall) { SanitizerScope SanScope(this); EmitSanitizerStatReport(llvm::SanStat_CFI_ICall); @@ -5310,6 +5315,10 @@ } } + if (SanOpts.has(SanitizerKind::KCFI) && IsIndirectCall) + EmitKCFICheck(Callee.getFunctionPointer(), + CGM.CreateKCFITypeId(QualType(FnType, 0))); + CallArgList Args; if (Chain) Args.add(RValue::get(Builder.CreateBitCast(Chain, CGM.VoidPtrTy)), diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -4601,6 +4601,8 @@ /// passing to a runtime sanitizer handler. llvm::Constant *EmitCheckSourceLocation(SourceLocation Loc); + void EmitKCFICheck(llvm::Value *Ptr, llvm::ConstantInt *Hash); + /// Create a basic block that will either trap or call a handler function in /// the UBSan runtime with the provided arguments, and create a conditional /// branch to it. diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h --- a/clang/lib/CodeGen/CodeGenModule.h +++ b/clang/lib/CodeGen/CodeGenModule.h @@ -1382,6 +1382,9 @@ /// Generate a cross-DSO type identifier for MD. llvm::ConstantInt *CreateCrossDsoCfiTypeId(llvm::Metadata *MD); + /// Generate a KCFI type identifier for T. + llvm::ConstantInt *CreateKCFITypeId(QualType T); + /// Create a metadata identifier for the given type. This may either be an /// MDString (for external identifiers) or a distinct unnamed MDNode (for /// internal identifiers). @@ -1400,6 +1403,12 @@ void CreateFunctionTypeMetadataForIcall(const FunctionDecl *FD, llvm::Function *F); + /// Set type hash as prefix data to the given function + void SetKCFITypePrefix(const FunctionDecl *FD, llvm::Function *F); + + /// Emit KCFI type identifier constants + void EmitKCFIConstants(); + /// Whether this function's return type has no side effects, and thus may /// be trivially discarded if it is unused. bool MayDropFunctionReturn(const ASTContext &Context, QualType ReturnType); @@ -1621,7 +1630,8 @@ llvm::AttrBuilder &FuncAttrs); llvm::Metadata *CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map, - StringRef Suffix); + StringRef Suffix, + bool OnlyExternal = true); }; } // end namespace CodeGen diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -46,6 +46,7 @@ #include "clang/CodeGen/BackendUtil.h" #include "clang/CodeGen/ConstantInitBuilder.h" #include "clang/Frontend/FrontendDiagnostic.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/ADT/Triple.h" #include "llvm/Analysis/TargetLibraryInfo.h" @@ -64,6 +65,7 @@ #include "llvm/Support/MD5.h" #include "llvm/Support/TimeProfiler.h" #include "llvm/Support/X86TargetParser.h" +#include "llvm/Support/xxhash.h" using namespace clang; using namespace CodeGen; @@ -547,6 +549,8 @@ CodeGenFunction(*this).EmitCfiCheckFail(); CodeGenFunction(*this).EmitCfiCheckStub(); } + if (LangOpts.Sanitize.has(SanitizerKind::KCFI)) + EmitKCFIConstants(); emitAtAvailableLinkGuard(); if (Context.getTargetInfo().getTriple().isWasm() && !Context.getTargetInfo().getTriple().isOSEmscripten()) { @@ -1594,6 +1598,15 @@ return llvm::ConstantInt::get(Int64Ty, llvm::MD5Hash(MDS->getString())); } +llvm::ConstantInt *CodeGenModule::CreateKCFITypeId(QualType T) { + if (auto *MDS = dyn_cast(CreateMetadataIdentifierImpl( + T, MetadataIdMap, "", /*OnlyExternal=*/false))) + return llvm::ConstantInt::get( + Int32Ty, static_cast(llvm::xxHash64(MDS->getString()))); + + return nullptr; +} + void CodeGenModule::SetLLVMFunctionAttributes(GlobalDecl GD, const CGFunctionInfo &Info, llvm::Function *F, bool IsThunk) { @@ -2187,6 +2200,53 @@ F->addTypeMetadata(0, llvm::ConstantAsMetadata::get(CrossDsoTypeId)); } +void CodeGenModule::SetKCFITypePrefix(const FunctionDecl *FD, + llvm::Function *F) { + + if (isa(FD) && !cast(FD)->isStatic()) + return; + + F->setPrefixData(CreateKCFITypeId(FD->getType())); + F->addFnAttr("kcfi"); + + if (CodeGenOpts.SanitizeKCFIOffset) + F->addFnAttr("kcfi-offset", Twine(CodeGenOpts.SanitizeKCFIOffset).str()); +} + +static bool allowKCFIIdentifier(const std::string &Name) { + // KCFI type identifier constants are only necessary for external assembly + // functions, which means it's safe to skip unusual names. Subset of + // MCAsmInfo::isAcceptableChar() and MCAsmInfoXCOFF::isAcceptableChar(). + for (const char &C : Name) { + if (llvm::isAlnum(C) || C == '_' || C == '.') + continue; + return false; + } + return true; +} + +void CodeGenModule::EmitKCFIConstants() { + llvm::Module &M = getModule(); + // Generate a weak constant with the expected KCFI type identifier for all + // address-taken function declarations. + for (auto &F : M.functions()) { + if (!F.isDeclaration() || !F.hasAddressTaken() || !F.hasPrefixData()) + continue; + + auto *Id = dyn_cast(F.getPrefixData()); + if (!Id) + continue; + + std::string Name = "__kcfi_typeid_" + F.getName().str(); + if (!allowKCFIIdentifier(Name)) + continue; + + std::string Asm = ".weak " + Name + "\n" + ".set " + Name + ", " + + std::to_string(Id->getZExtValue()) + "\n"; + M.appendModuleInlineAsm(Asm); + } +} + void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F, bool IsIncompleteFunction, bool IsThunk) { @@ -2269,6 +2329,9 @@ !CodeGenOpts.SanitizeCfiCanonicalJumpTables) CreateFunctionTypeMetadataForIcall(FD, F); + if (LangOpts.Sanitize.has(SanitizerKind::KCFI)) + SetKCFITypePrefix(FD, F); + if (getLangOpts().OpenMP && FD->hasAttr()) getOpenMPRuntime().emitDeclareSimdFunction(FD, F); @@ -6498,7 +6561,8 @@ llvm::Metadata * CodeGenModule::CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map, - StringRef Suffix) { + StringRef Suffix, + bool OnlyExternal /*=true*/) { if (auto *FnType = T->getAs()) T = getContext().getFunctionType( FnType->getReturnType(), FnType->getParamTypes(), @@ -6508,7 +6572,7 @@ if (InternalId) return InternalId; - if (isExternallyVisible(T->getLinkage())) { + if (isExternallyVisible(T->getLinkage()) || !OnlyExternal) { std::string OutName; llvm::raw_string_ostream Out(OutName); getCXXABI().getMangleContext().mangleTypeName(T, Out); diff --git a/clang/lib/Driver/SanitizerArgs.cpp b/clang/lib/Driver/SanitizerArgs.cpp --- a/clang/lib/Driver/SanitizerArgs.cpp +++ b/clang/lib/Driver/SanitizerArgs.cpp @@ -37,7 +37,8 @@ static const SanitizerMask NotAllowedWithMinimalRuntime = SanitizerKind::Function | SanitizerKind::Vptr; static const SanitizerMask RequiresPIE = - SanitizerKind::DataFlow | SanitizerKind::HWAddress | SanitizerKind::Scudo; + SanitizerKind::DataFlow | SanitizerKind::HWAddress | SanitizerKind::Scudo | + SanitizerKind::KCFI; static const SanitizerMask NeedsUnwindTables = SanitizerKind::Address | SanitizerKind::HWAddress | SanitizerKind::Thread | SanitizerKind::Memory | SanitizerKind::DataFlow; @@ -58,8 +59,9 @@ SanitizerKind::FloatDivideByZero | SanitizerKind::ObjCCast; static const SanitizerMask Unrecoverable = SanitizerKind::Unreachable | SanitizerKind::Return; -static const SanitizerMask AlwaysRecoverable = - SanitizerKind::KernelAddress | SanitizerKind::KernelHWAddress; +static const SanitizerMask AlwaysRecoverable = SanitizerKind::KernelAddress | + SanitizerKind::KernelHWAddress | + SanitizerKind::KCFI; static const SanitizerMask NeedsLTO = SanitizerKind::CFI; static const SanitizerMask TrappingSupported = (SanitizerKind::Undefined & ~SanitizerKind::Vptr) | SanitizerKind::Integer | @@ -145,17 +147,17 @@ struct Ignorelist { const char *File; SanitizerMask Mask; - } Ignorelists[] = {{"asan_ignorelist.txt", SanitizerKind::Address}, - {"hwasan_ignorelist.txt", SanitizerKind::HWAddress}, - {"memtag_ignorelist.txt", SanitizerKind::MemTag}, - {"msan_ignorelist.txt", SanitizerKind::Memory}, - {"tsan_ignorelist.txt", SanitizerKind::Thread}, - {"dfsan_abilist.txt", SanitizerKind::DataFlow}, - {"cfi_ignorelist.txt", SanitizerKind::CFI}, - {"ubsan_ignorelist.txt", - SanitizerKind::Undefined | SanitizerKind::Integer | - SanitizerKind::Nullability | - SanitizerKind::FloatDivideByZero}}; + } Ignorelists[] = { + {"asan_ignorelist.txt", SanitizerKind::Address}, + {"hwasan_ignorelist.txt", SanitizerKind::HWAddress}, + {"memtag_ignorelist.txt", SanitizerKind::MemTag}, + {"msan_ignorelist.txt", SanitizerKind::Memory}, + {"tsan_ignorelist.txt", SanitizerKind::Thread}, + {"dfsan_abilist.txt", SanitizerKind::DataFlow}, + {"cfi_ignorelist.txt", SanitizerKind::CFI | SanitizerKind::KCFI}, + {"ubsan_ignorelist.txt", + SanitizerKind::Undefined | SanitizerKind::Integer | + SanitizerKind::Nullability | SanitizerKind::FloatDivideByZero}}; for (auto BL : Ignorelists) { if (!(Kinds & BL.Mask)) @@ -165,7 +167,7 @@ llvm::sys::path::append(Path, "share", BL.File); if (D.getVFS().exists(Path)) IgnorelistFiles.push_back(std::string(Path.str())); - else if (BL.Mask == SanitizerKind::CFI && DiagnoseErrors) + else if (BL.Mask & SanitizerKind::CFI && DiagnoseErrors) // If cfi_ignorelist.txt cannot be found in the resource dir, driver // should fail. D.Diag(clang::diag::err_drv_no_such_file) << Path; @@ -676,11 +678,22 @@ << "-fsanitize-cfi-cross-dso" << "-fsanitize-cfi-icall-generalize-pointers"; + if (AllAddedKinds & SanitizerKind::KCFI && DiagnoseErrors) + D.Diag(diag::err_drv_argument_not_allowed_with) << "-fsanitize=cfi*" + << "-fsanitize=kcfi"; + CfiCanonicalJumpTables = Args.hasFlag(options::OPT_fsanitize_cfi_canonical_jump_tables, options::OPT_fno_sanitize_cfi_canonical_jump_tables, true); } + if (AllAddedKinds & SanitizerKind::KCFI) + if (Arg *A = Args.getLastArg(options::OPT_fsanitize_kcfi_offset)) { + StringRef S = A->getValue(); + if (S.getAsInteger(0, KCFIOffset) && DiagnoseErrors) + D.Diag(clang::diag::err_drv_invalid_value) << A->getAsString(Args) << S; + } + Stats = Args.hasFlag(options::OPT_fsanitize_stats, options::OPT_fno_sanitize_stats, false); @@ -1135,6 +1148,10 @@ if (CfiCanonicalJumpTables) CmdArgs.push_back("-fsanitize-cfi-canonical-jump-tables"); + if (KCFIOffset) + CmdArgs.push_back( + Args.MakeArgString("-fsanitize-kcfi-offset=" + Twine(KCFIOffset))); + if (Stats) CmdArgs.push_back("-fsanitize-stats"); diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp --- a/clang/lib/Driver/ToolChain.cpp +++ b/clang/lib/Driver/ToolChain.cpp @@ -1035,6 +1035,9 @@ getTriple().getArch() == llvm::Triple::arm || getTriple().isWasm() || getTriple().isAArch64()) Res |= SanitizerKind::CFIICall; + if (getTriple().getArch() == llvm::Triple::x86_64 || + getTriple().isAArch64(64)) + Res |= SanitizerKind::KCFI; if (getTriple().getArch() == llvm::Triple::x86_64 || getTriple().isAArch64(64) || getTriple().isRISCV()) Res |= SanitizerKind::ShadowCallStack; diff --git a/clang/test/CodeGen/kcfi.c b/clang/test/CodeGen/kcfi.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/kcfi.c @@ -0,0 +1,50 @@ +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fsanitize=kcfi -o - %s | FileCheck --check-prefixes=CHECK,OFFSET0 %s +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fsanitize=kcfi -fsanitize-kcfi-offset=1 -fpatchable-function-entry=1 -fpatchable-function-entry-offset=1 -o - %s | FileCheck --check-prefixes=CHECK,OFFSET1 %s +#if !__has_feature(kcfi) +#error Missing kcfi? +#endif + +// CHECK: module asm ".weak __kcfi_typeid_f4" +// CHECK: module asm ".set __kcfi_typeid_f4, [[#]]" + +typedef int (*fn_t)(void); + +// CHECK: define dso_local i32 @f1() #[[#ATTR:]] prefix i32 [[#%d,HASH:]] +int f1(void) { return 0; } + +// CHECK: define dso_local i32 @f2() #[[#ATTR]] prefix i32 [[#%d,HASH2:]] +unsigned int f2(void) { return 2; } + +// CHECK-LABEL: define dso_local i32 @__call(i32 ()*{{.*}} %f) +int __call(fn_t f) __attribute__((__no_sanitize__("kcfi"))) { + // CHECK-NOT: call void @llvm.kcfi_check + // CHECK: %call = call i32 %[[#]]() + return f(); +} + +// CHECK-LABEL: define dso_local i32 @call(i32 ()*{{.*}} %f) +int call(fn_t f) { + // CHECK: %[[#CAST:]] = bitcast i32 ()* %{{.}} to i8* + // CHECK: call void @llvm.kcfi.check(i8* %[[#CAST]], i32 [[#HASH]]) + // CHECK: %call = call i32 %[[#]]() + return f(); +} + +// CHECK-DAG: define internal i32 @f3() #[[#ATTR]] prefix i32 [[#HASH]] +static int f3(void) { return 1; } + +// CHECK-DAG: declare i32 @f4() #[[#DECATTR:]] prefix i32 [[#HASH]] +extern int f4(void); + +// CHECK-DAG: declare void @llvm.kcfi.check(i8*, i32 immarg) + +int test(void) { + return call(f1) + + __call((fn_t)f2) + + call(f3) + + call(f4); +} + +// OFFSET0: attributes #[[#ATTR]] = {{{.*}}"kcfi" +// OFFSET1: attributes #[[#ATTR]] = {{{.*}}"kcfi" {{.*}}"kcfi-offset"="1" +// CHECK: attributes #[[#DECATTR]] = {{{.*}}"kcfi" diff --git a/clang/test/Driver/fsanitize.c b/clang/test/Driver/fsanitize.c --- a/clang/test/Driver/fsanitize.c +++ b/clang/test/Driver/fsanitize.c @@ -646,6 +646,18 @@ // RUN: %clang -target x86_64-linux-gnu -fsanitize=cfi -fsanitize-stats -flto -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CFI-STATS // CHECK-CFI-STATS: -fsanitize-stats +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi -fsanitize=cfi -flto -fvisibility=hidden %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI-NOCFI +// CHECK-KCFI-NOCFI: error: invalid argument '-fsanitize=cfi*' not allowed with '-fsanitize=kcfi' + +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi -fsanitize-trap=kcfi %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI-NOTRAP +// CHECK-KCFI-NOTRAP: error: unsupported argument 'kcfi' to option '-fsanitize-trap' + +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI +// CHECK-KCFI: "-fsanitize=kcfi" + +// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi -fno-sanitize-recover=kcfi %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI-RECOVER +// CHECK-KCFI-RECOVER: error: unsupported argument 'kcfi' to option 'fno-sanitize-recover=' + // RUN: %clang_cl -fsanitize=address -c -MDd -### -- %s 2>&1 | FileCheck %s -check-prefix=CHECK-ASAN-DEBUGRTL // RUN: %clang_cl -fsanitize=address -c -MTd -### -- %s 2>&1 | FileCheck %s -check-prefix=CHECK-ASAN-DEBUGRTL // RUN: %clang_cl -fsanitize=address -c -LDd -### -- %s 2>&1 | FileCheck %s -check-prefix=CHECK-ASAN-DEBUGRTL diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h --- a/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -366,6 +366,12 @@ void emitBBAddrMapSection(const MachineFunction &MF); + int64_t getKCFIOffset(const MachineFunction &MF); + void emitKCFIEntry(MCSection *Section, const MCSymbol *Symbol); + void emitKCFITypeEntry(const MachineFunction &MF, const MCSymbol *Symbol); + void emitKCFITrapEntry(const MachineFunction &MF, const MCSymbol *Symbol); + virtual void emitKCFITypeId(const MachineFunction &MF); + void emitPseudoProbe(const MachineInstr &MI); void emitRemarksSection(remarks::RemarkStreamer &RS); diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1705,6 +1705,9 @@ def int_load_relative: DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_anyint_ty], [IntrReadMem, IntrArgMemOnly]>; +def int_kcfi_check : + Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty], [ImmArg>]>; + def int_asan_check_memaccess : Intrinsic<[],[llvm_ptr_ty, llvm_i32_ty], [ImmArg>]>; diff --git a/llvm/include/llvm/MC/MCObjectFileInfo.h b/llvm/include/llvm/MC/MCObjectFileInfo.h --- a/llvm/include/llvm/MC/MCObjectFileInfo.h +++ b/llvm/include/llvm/MC/MCObjectFileInfo.h @@ -355,6 +355,9 @@ MCSection *getBBAddrMapSection(const MCSection &TextSec) const; + MCSection *getKCFISection(const MCSection &TextSec, + const std::string &Name) const; + MCSection *getPseudoProbeSection(const MCSection *TextSec) const; MCSection *getPseudoProbeDescSection(StringRef FuncName) const; diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -755,21 +755,24 @@ // Emit the prefix data. if (F.hasPrefixData()) { - if (MAI->hasSubsectionsViaSymbols()) { - // Preserving prefix data on platforms which use subsections-via-symbols - // is a bit tricky. Here we introduce a symbol for the prefix data - // and use the .alt_entry attribute to mark the function's real entry point - // as an alternative entry point to the prefix-data symbol. - MCSymbol *PrefixSym = OutContext.createLinkerPrivateTempSymbol(); - OutStreamer->emitLabel(PrefixSym); + if (F.hasFnAttribute("kcfi")) + emitKCFITypeId(*MF); + else { + if (MAI->hasSubsectionsViaSymbols()) { + // Preserving prefix data on platforms which use subsections-via-symbols + // is a bit tricky. Here we introduce a symbol for the prefix data + // and use the .alt_entry attribute to mark the function's real entry + // point as an alternative entry point to the prefix-data symbol. + MCSymbol *PrefixSym = OutContext.createLinkerPrivateTempSymbol(); + OutStreamer->emitLabel(PrefixSym); + } emitGlobalConstant(F.getParent()->getDataLayout(), F.getPrefixData()); + } - // Emit an .alt_entry directive for the actual function symbol. + // Emit an .alt_entry directive for the actual function symbol. + if (MAI->hasSubsectionsViaSymbols()) OutStreamer->emitSymbolAttribute(CurrentFnSym, MCSA_AltEntry); - } else { - emitGlobalConstant(F.getParent()->getDataLayout(), F.getPrefixData()); - } } // Emit M NOPs for -fpatchable-function-entry=N,M where M>0. We arbitrarily @@ -1157,6 +1160,52 @@ OutStreamer->PopSection(); } +int64_t AsmPrinter::getKCFIOffset(const MachineFunction &MF) { + int64_t Offset = 0; + const Function &F = MF.getFunction(); + + (void)F.getFnAttribute("kcfi-offset") + .getValueAsString() + .getAsInteger(10, Offset); + + return Offset; +} + +void AsmPrinter::emitKCFIEntry(MCSection *Section, const MCSymbol *Symbol) { + if (Section) { + OutStreamer->PushSection(); + OutStreamer->SwitchSection(Section); + OutStreamer->emitSymbolValue(Symbol, getPointerSize()); + OutStreamer->PopSection(); + } +} + +void AsmPrinter::emitKCFITypeEntry(const MachineFunction &MF, + const MCSymbol *Symbol) { + emitKCFIEntry( + getObjFileLowering().getKCFISection(*MF.getSection(), ".kcfi_types"), + Symbol); +} + +void AsmPrinter::emitKCFITrapEntry(const MachineFunction &MF, + const MCSymbol *Symbol) { + emitKCFIEntry( + getObjFileLowering().getKCFISection(*MF.getSection(), ".kcfi_traps"), + Symbol); +} + +void AsmPrinter::emitKCFITypeId(const MachineFunction &MF) { + // Emit a .kcfi_types entry for the prefix data. + MCSymbol *PrefixSym = OutContext.createLinkerPrivateTempSymbol(); + OutStreamer->emitLabel(PrefixSym); + emitKCFITypeEntry(MF, PrefixSym); + + // Emit the actual prefix data normally. Note that arch-specific + // implementations may emit additional data. + const Function &F = MF.getFunction(); + emitGlobalConstant(F.getParent()->getDataLayout(), F.getPrefixData()); +} + void AsmPrinter::emitPseudoProbe(const MachineInstr &MI) { if (PP) { auto GUID = MI.getOperand(0).getImm(); diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp --- a/llvm/lib/MC/MCObjectFileInfo.cpp +++ b/llvm/lib/MC/MCObjectFileInfo.cpp @@ -1100,6 +1100,24 @@ cast(TextSec.getBeginSymbol())); } +MCSection *MCObjectFileInfo::getKCFISection(const MCSection &TextSec, + const std::string &Name) const { + if (Ctx->getObjectFileType() != MCContext::IsELF) + return nullptr; + + const MCSectionELF &ElfSec = static_cast(TextSec); + unsigned Flags = ELF::SHF_LINK_ORDER; + StringRef GroupName; + if (const MCSymbol *Group = ElfSec.getGroup()) { + GroupName = Group->getName(); + Flags |= ELF::SHF_GROUP; + } + + return Ctx->getELFSection(Name, ELF::SHT_PROGBITS, Flags | ELF::SHF_ALLOC, 0, + GroupName, true, ElfSec.getUniqueID(), + cast(TextSec.getBeginSymbol())); +} + MCSection * MCObjectFileInfo::getPseudoProbeSection(const MCSection *TextSec) const { if (Ctx->getObjectFileType() == MCContext::IsELF) { diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -111,6 +111,7 @@ typedef std::tuple HwasanMemaccessTuple; std::map HwasanMemaccessSymbols; + void LowerKCFI_CHECK(const MachineInstr &MI); void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI); void emitHwasanMemaccessSymbols(Module &M); @@ -317,6 +318,47 @@ recordSled(CurSled, MI, Kind, 2); } +void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) { + const MachineFunction &MF = *MI.getMF(); + int64_t Offset = -4 - getKCFIOffset(MF); + + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDURWi) + .addReg(AArch64::W16) + .addReg(MI.getOperand(0).getReg()) + .addImm(Offset)); + + int64_t Type = MI.getOperand(1).getImm(); + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi) + .addReg(AArch64::W17) + .addReg(AArch64::W17) + .addImm(Type & 0xFFFF) + .addImm(0)); + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi) + .addReg(AArch64::W17) + .addReg(AArch64::W17) + .addImm((Type >> 16) & 0xFFFF) + .addImm(16)); + + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSWrs) + .addReg(AArch64::WZR) + .addReg(AArch64::W16) + .addReg(AArch64::W17) + .addImm(0)); + + MCSymbol *Pass = OutContext.createTempSymbol(); + EmitToStreamer(*OutStreamer, + MCInstBuilder(AArch64::Bcc) + .addImm(AArch64CC::EQ) + .addExpr(MCSymbolRefExpr::create(Pass, OutContext))); + + MCSymbol *Trap = OutContext.createTempSymbol(); + OutStreamer->emitLabel(Trap); + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BRK).addImm(0x801)); + emitKCFITrapEntry(MF, Trap); + + OutStreamer->emitLabel(Pass); +} + void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) { Register Reg = MI.getOperand(0).getReg(); bool IsShort = @@ -1433,6 +1475,10 @@ LowerPATCHABLE_TAIL_CALL(*MI); return; + case AArch64::KCFI_CHECK: + LowerKCFI_CHECK(*MI); + return; + case AArch64::HWASAN_CHECK_MEMACCESS: case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES: LowerHWASAN_CHECK_MEMACCESS(*MI); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -1368,6 +1368,13 @@ def MOVbaseTLS : Pseudo<(outs GPR64:$dst), (ins), [(set GPR64:$dst, AArch64threadpointer)]>, Sched<[WriteSys]>; +let Defs = [ X16, X17, NZCV ] in { +def KCFI_CHECK : Pseudo< + (outs), (ins GPR64noip:$ptr, i32imm:$type), + [(int_kcfi_check GPR64noip:$ptr, (i32 timm:$type))]>, + Sched<[]>; +} + let Uses = [ X9 ], Defs = [ X16, X17, LR, NZCV ] in { def HWASAN_CHECK_MEMACCESS : Pseudo< (outs), (ins GPR64noip:$ptr, i32imm:$accessinfo), diff --git a/llvm/lib/Target/X86/X86AsmPrinter.h b/llvm/lib/Target/X86/X86AsmPrinter.h --- a/llvm/lib/Target/X86/X86AsmPrinter.h +++ b/llvm/lib/Target/X86/X86AsmPrinter.h @@ -98,6 +98,9 @@ void LowerFENTRY_CALL(const MachineInstr &MI, X86MCInstLower &MCIL); + // KCFI specific lowering for X86. + void LowerKCFI_CHECK(const MachineInstr &MI); + // Address sanitizer specific lowering for X86. void LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI); @@ -151,6 +154,7 @@ bool runOnMachineFunction(MachineFunction &MF) override; void emitFunctionBodyStart() override; void emitFunctionBodyEnd() override; + void emitKCFITypeId(const MachineFunction &MF) override; bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override { return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags; diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp --- a/llvm/lib/Target/X86/X86AsmPrinter.cpp +++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp @@ -107,6 +107,12 @@ } } +void X86AsmPrinter::emitKCFITypeId(const MachineFunction &MF) { + AsmPrinter::emitKCFITypeId(MF); + // Emit two zero bytes to avoid gadgets in llvm.kcfi.check(). + OutStreamer->emitZeros(2); +} + /// PrintSymbolOperand - Print a raw symbol reference operand. This handles /// jump tables, constant pools, global address and external symbols, all of /// which print to a label with various suffixes for relocation types etc. diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td --- a/llvm/lib/Target/X86/X86InstrCompiler.td +++ b/llvm/lib/Target/X86/X86InstrCompiler.td @@ -256,6 +256,17 @@ "#SEH_Epilogue", []>; } +//===----------------------------------------------------------------------===// +// Pseudo instructions used by KCFI. +//===----------------------------------------------------------------------===// +let + Defs = [EFLAGS] in { +def KCFI_CHECK : PseudoI< + (outs), (ins GR64:$ptr, i32imm:$type), + [(int_kcfi_check GR64:$ptr, (i32 timm:$type))]>, + Sched<[]>; +} + //===----------------------------------------------------------------------===// // Pseudo instructions used by address sanitizer. //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp --- a/llvm/lib/Target/X86/X86MCInstLower.cpp +++ b/llvm/lib/Target/X86/X86MCInstLower.cpp @@ -1330,6 +1330,32 @@ .addExpr(Op)); } +void X86AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) { + const MachineFunction &MF = *MI.getMF(); + int64_t Offset = -6 - getKCFIOffset(MF); + + EmitAndCountInstruction(MCInstBuilder(X86::CMP32mi) + .addReg(MI.getOperand(0).getReg()) + .addImm(1) + .addReg(X86::NoRegister) + .addImm(Offset) + .addReg(X86::NoRegister) + .addImm(MI.getOperand(1).getImm())); + + MCSymbol *Pass = OutContext.createTempSymbol(); + EmitAndCountInstruction( + MCInstBuilder(X86::JCC_1) + .addExpr(MCSymbolRefExpr::create(Pass, OutContext)) + .addImm(X86::COND_E)); + + MCSymbol *Trap = OutContext.createTempSymbol(); + OutStreamer->emitLabel(Trap); + EmitAndCountInstruction(MCInstBuilder(X86::TRAP)); + emitKCFITrapEntry(MF, Trap); + + OutStreamer->emitLabel(Pass); +} + void X86AsmPrinter::LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI) { // FIXME: Make this work on non-ELF. if (!TM.getTargetTriple().isOSBinFormatELF()) { @@ -2612,6 +2638,9 @@ EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget))); return; + case X86::KCFI_CHECK: + return LowerKCFI_CHECK(*MI); + case X86::ASAN_CHECK_MEMACCESS: return LowerASAN_CHECK_MEMACCESS(*MI); diff --git a/llvm/test/CodeGen/AArch64/kcfi-check.ll b/llvm/test/CodeGen/AArch64/kcfi-check.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/kcfi-check.ll @@ -0,0 +1,31 @@ +; RUN: llc < %s | FileCheck %s + +target triple = "aarch64--" + +; CHECK: .Ltmp0 +; CHECK-NEXT: .section .kcfi_types,"ao",@progbits,.text +; CHECK-NEXT: .xword .Ltmp0 +; CHECK-NEXT: .text +; CHECK-NEXT: .word 12345678 +define void @f1(i8* %x) #0 prefix i32 12345678 { +; CHECK-LABEL: f1: +; CHECK: // %bb.0: +; CHECK-NEXT: ldur w16, [x0, #-4] +; CHECK-NEXT: movk w17, #24910 +; CHECK-NEXT: movk w17, #188, lsl #16 +; CHECK-NEXT: cmp w16, w17 +; CHECK-NEXT: b.eq .Ltmp1 +; CHECK-NEXT: .Ltmp2: +; CHECK-NEXT: brk #0x801 +; CHECK-NEXT: .section .kcfi_traps,"ao",@progbits,.text +; CHECK-NEXT: .xword .Ltmp2 +; CHECK-NEXT: .text +; CHECK-NEXT: .Ltmp1: +; CHECK-NEXT: ret + call void @llvm.kcfi.check(i8* %x, i32 12345678) + ret void +} + +declare void @llvm.kcfi.check(i8*, i32 immarg) + +attributes #0 = { "kcfi" } diff --git a/llvm/test/CodeGen/X86/kcfi-check.ll b/llvm/test/CodeGen/X86/kcfi-check.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/kcfi-check.ll @@ -0,0 +1,29 @@ +; RUN: llc < %s | FileCheck %s + +target triple = "x86_64-unknown-linux-gnu" + +; CHECK: .Ltmp0 +; CHECK-NEXT: .section .kcfi_types,"ao",@progbits,.text +; CHECK-NEXT: .quad .Ltmp0 +; CHECK-NEXT: .text +; CHECK-NEXT: .long 12345678 +; CHECK-NEXT: .zero 2 +define void @f1(i8* %x) #0 prefix i32 12345678 { +; CHECK-LABEL: f1: +; CHECK: # %bb.0: +; CHECK-NEXT: cmpl $12345678, -6(%rdi) # imm = 0xBC614E +; CHECK-NEXT: je .Ltmp1 +; CHECK-NEXT: .Ltmp2: +; CHECK-NEXT: ud2 +; CHECK-NEXT: .section .kcfi_traps,"ao",@progbits,.text +; CHECK-NEXT: .quad .Ltmp2 +; CHECK-NEXT: .text +; CHECK-NEXT: .Ltmp1: +; CHECK-NEXT: retq + call void @llvm.kcfi.check(i8* %x, i32 12345678) + ret void +} + +declare void @llvm.kcfi.check(i8*, i32 immarg) + +attributes #0 = { "kcfi" }