diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -72,6 +72,7 @@ #include "llvm/Transforms/Instrumentation/GCOVProfiler.h" #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" #include "llvm/Transforms/Instrumentation/InstrProfiling.h" +#include "llvm/Transforms/Instrumentation/KCFI.h" #include "llvm/Transforms/Instrumentation/MemProfiler.h" #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" #include "llvm/Transforms/Instrumentation/SanitizerBinaryMetadata.h" @@ -644,6 +645,31 @@ } } +static void addKCFIPass(const Triple &TargetTriple, const LangOptions &LangOpts, + PassBuilder &PB) { + // If the back-end supports KCFI operand bundle lowering, skip KCFIPass. + if (TargetTriple.getArch() == llvm::Triple::x86_64 || + TargetTriple.isAArch64(64)) + return; + + // Ensure we lower KCFI operand bundles with -O0. + PB.registerOptimizerLastEPCallback( + [&](ModulePassManager &MPM, OptimizationLevel Level) { + if (Level == OptimizationLevel::O0 && + LangOpts.Sanitize.has(SanitizerKind::KCFI)) + MPM.addPass(createModuleToFunctionPassAdaptor(KCFIPass())); + }); + + // When optimizations are requested, run KCIFPass after InstCombine to + // avoid unnecessary checks. + PB.registerPeepholeEPCallback( + [&](FunctionPassManager &FPM, OptimizationLevel Level) { + if (Level != OptimizationLevel::O0 && + LangOpts.Sanitize.has(SanitizerKind::KCFI)) + FPM.addPass(KCFIPass()); + }); +} + static void addSanitizers(const Triple &TargetTriple, const CodeGenOptions &CodeGenOpts, const LangOptions &LangOpts, PassBuilder &PB) { @@ -946,8 +972,10 @@ // Don't add sanitizers if we are here from ThinLTO PostLink. That already // done on PreLink stage. - if (!IsThinLTOPostLink) + if (!IsThinLTOPostLink) { addSanitizers(TargetTriple, CodeGenOpts, LangOpts, PB); + addKCFIPass(TargetTriple, LangOpts, PB); + } if (Optional Options = getGCOVOptions(CodeGenOpts, LangOpts)) PB.registerPipelineStartEPCallback( diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp --- a/clang/lib/Driver/ToolChain.cpp +++ b/clang/lib/Driver/ToolChain.cpp @@ -1088,7 +1088,7 @@ ~SanitizerKind::Function) | (SanitizerKind::CFI & ~SanitizerKind::CFIICall) | SanitizerKind::CFICastStrict | SanitizerKind::FloatDivideByZero | - SanitizerKind::UnsignedIntegerOverflow | + SanitizerKind::KCFI | SanitizerKind::UnsignedIntegerOverflow | SanitizerKind::UnsignedShiftBase | SanitizerKind::ImplicitConversion | SanitizerKind::Nullability | SanitizerKind::LocalBounds; if (getTriple().getArch() == llvm::Triple::x86 || @@ -1096,9 +1096,6 @@ getTriple().getArch() == llvm::Triple::arm || getTriple().isWasm() || getTriple().isAArch64() || getTriple().isRISCV()) Res |= SanitizerKind::CFIICall; - if (getTriple().getArch() == llvm::Triple::x86_64 || - getTriple().isAArch64(64)) - Res |= SanitizerKind::KCFI; if (getTriple().getArch() == llvm::Triple::x86_64 || getTriple().isAArch64(64) || getTriple().isRISCV()) Res |= SanitizerKind::ShadowCallStack; diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -258,6 +258,7 @@ void initializeLowerSwitchLegacyPassPass(PassRegistry &); void initializeLowerMatrixIntrinsicsLegacyPassPass(PassRegistry &); void initializeLowerMatrixIntrinsicsMinimalLegacyPassPass(PassRegistry &); +void initializeKCFIPass(PassRegistry &); void initializeMIRAddFSDiscriminatorsPass(PassRegistry &); void initializeMIRCanonicalizerPass(PassRegistry &); void initializeMIRNamerPass(PassRegistry &); diff --git a/llvm/include/llvm/Transforms/Instrumentation/KCFI.h b/llvm/include/llvm/Transforms/Instrumentation/KCFI.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Transforms/Instrumentation/KCFI.h @@ -0,0 +1,26 @@ +//===-- KCFI.h - Generic KCFI operand bundle lowering -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass emits generic KCFI indirect call checks for targets that don't +// support lowering KCFI operand bundles in the back-end. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_KCFI_H +#define LLVM_TRANSFORMS_INSTRUMENTATION_KCFI_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { +class KCFIPass : public PassInfoMixin { +public: + static bool isRequired() { return true; } + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; +} // namespace llvm +#endif // LLVM_TRANSFORMS_INSTRUMENTATION_KCFI_H diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -75,11 +75,11 @@ #include "llvm/Analysis/TypeBasedAliasAnalysis.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/Dominators.h" -#include "llvm/IRPrinter/IRPrintingPasses.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/PrintPasses.h" #include "llvm/IR/SafepointIRVerifier.h" #include "llvm/IR/Verifier.h" +#include "llvm/IRPrinter/IRPrintingPasses.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" @@ -137,6 +137,7 @@ #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" #include "llvm/Transforms/Instrumentation/InstrOrderFile.h" #include "llvm/Transforms/Instrumentation/InstrProfiling.h" +#include "llvm/Transforms/Instrumentation/KCFI.h" #include "llvm/Transforms/Instrumentation/MemProfiler.h" #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -322,6 +322,7 @@ FUNCTION_PASS("newgvn", NewGVNPass()) FUNCTION_PASS("jump-threading", JumpThreadingPass()) FUNCTION_PASS("partially-inline-libcalls", PartiallyInlineLibCallsPass()) +FUNCTION_PASS("kcfi", KCFIPass()) FUNCTION_PASS("lcssa", LCSSAPass()) FUNCTION_PASS("loop-data-prefetch", LoopDataPrefetchPass()) FUNCTION_PASS("loop-load-elim", LoopLoadEliminationPass()) diff --git a/llvm/lib/Transforms/Instrumentation/CMakeLists.txt b/llvm/lib/Transforms/Instrumentation/CMakeLists.txt --- a/llvm/lib/Transforms/Instrumentation/CMakeLists.txt +++ b/llvm/lib/Transforms/Instrumentation/CMakeLists.txt @@ -11,6 +11,7 @@ Instrumentation.cpp InstrOrderFile.cpp InstrProfiling.cpp + KCFI.cpp PGOInstrumentation.cpp PGOMemOPSizeOpt.cpp PoisonChecking.cpp diff --git a/llvm/lib/Transforms/Instrumentation/KCFI.cpp b/llvm/lib/Transforms/Instrumentation/KCFI.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/Transforms/Instrumentation/KCFI.cpp @@ -0,0 +1,111 @@ +//===-- KCFI.cpp - Generic KCFI operand bundle lowering ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass emits generic KCFI indirect call checks for targets that don't +// support lowering KCFI operand bundles in the back-end. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Instrumentation/KCFI.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalObject.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/MDBuilder.h" +#include "llvm/IR/Module.h" +#include "llvm/InitializePasses.h" +#include "llvm/Pass.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/Instrumentation.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" + +using namespace llvm; + +#define DEBUG_TYPE "kcfi" + +STATISTIC(NumKCFIChecks, "Number of kcfi operands transformed into checks"); + +namespace { +class DiagnosticInfoKCFI : public DiagnosticInfo { + const Twine &Msg; + +public: + DiagnosticInfoKCFI(const Twine &DiagMsg, + DiagnosticSeverity Severity = DS_Error) + : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {} + void print(DiagnosticPrinter &DP) const override { DP << Msg; } +}; +} // namespace + +PreservedAnalyses KCFIPass::run(Function &F, FunctionAnalysisManager &AM) { + Module &M = *F.getParent(); + if (!M.getModuleFlag("kcfi")) + return PreservedAnalyses::all(); + + // Find call instructions with KCFI operand bundles. + SmallVector KCFICalls; + for (Instruction &I : instructions(F)) { + if (auto *CI = dyn_cast(&I)) + if (CI->getOperandBundle(LLVMContext::OB_kcfi)) + KCFICalls.push_back(CI); + } + + if (KCFICalls.empty()) + return PreservedAnalyses::all(); + + LLVMContext &Ctx = M.getContext(); + // patchable-function-prefix emits nops between the KCFI type identifier + // and the function start. As we don't know the size of the emitted nops, + // don't allow this attribute with generic lowering. + if (F.hasFnAttribute("patchable-function-prefix")) + Ctx.diagnose( + DiagnosticInfoKCFI("-fpatchable-function-entry=N,M, where M>0 is not " + "compatible with -fsanitize=kcfi on this target")); + + IntegerType *Int32Ty = Type::getInt32Ty(Ctx); + MDNode *VeryUnlikelyWeights = + MDBuilder(Ctx).createBranchWeights(1, (1U << 20) - 1); + + for (CallInst *CI : KCFICalls) { + // Get the expected hash value. + const uint32_t ExpectedHash = + cast(CI->getOperandBundle(LLVMContext::OB_kcfi)->Inputs[0]) + ->getZExtValue(); + + // Drop the KCFI operand bundle. + CallBase *Call = + CallBase::removeOperandBundle(CI, LLVMContext::OB_kcfi, CI); + assert(Call != CI); + Call->copyMetadata(*CI); + CI->replaceAllUsesWith(Call); + CI->eraseFromParent(); + + if (!Call->isIndirectCall()) + continue; + + // Emit a check and trap if the target hash doesn't match. + IRBuilder<> Builder(Call); + Value *HashPtr = Builder.CreateConstInBoundsGEP1_32( + Int32Ty, Call->getCalledOperand(), -1); + Value *Test = Builder.CreateICmpNE(Builder.CreateLoad(Int32Ty, HashPtr), + ConstantInt::get(Int32Ty, ExpectedHash)); + Instruction *ThenTerm = + SplitBlockAndInsertIfThen(Test, Call, false, VeryUnlikelyWeights); + Builder.SetInsertPoint(ThenTerm); + Builder.CreateCall(Intrinsic::getDeclaration(&M, Intrinsic::trap)); + ++NumKCFIChecks; + } + + return PreservedAnalyses::none(); +} diff --git a/llvm/test/Transforms/KCFI/kcfi-patchable-function-prefix.ll b/llvm/test/Transforms/KCFI/kcfi-patchable-function-prefix.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/KCFI/kcfi-patchable-function-prefix.ll @@ -0,0 +1,12 @@ +; RUN: not opt -S -passes=kcfi %s 2>&1 | FileCheck %s + +; CHECK: error: -fpatchable-function-entry=N,M, where M>0 is not compatible with -fsanitize=kcfi on this target +define void @f1(ptr noundef %x) #0 { + call void %x() [ "kcfi"(i32 12345678) ] + ret void +} + +attributes #0 = { "patchable-function-prefix"="1" } + +!llvm.module.flags = !{!0} +!0 = !{i32 4, !"kcfi", i32 1} diff --git a/llvm/test/Transforms/KCFI/kcfi.ll b/llvm/test/Transforms/KCFI/kcfi.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/KCFI/kcfi.ll @@ -0,0 +1,21 @@ +; RUN: opt -S -passes=kcfi %s | FileCheck %s + +; CHECK-LABEL: define void @f1( +define void @f1(ptr noundef %x) { + ; CHECK: %[[#GEPI:]] = getelementptr inbounds i32, ptr %x, i32 -1 + ; CHECK-NEXT: %[[#LOAD:]] = load i32, ptr %[[#GEPI]], align 4 + ; CHECK-NEXT: %[[#ICMP:]] = icmp ne i32 %[[#LOAD]], 12345678 + ; CHECK-NEXT: br i1 %[[#ICMP]], label %[[#TRAP:]], label %[[#CALL:]], !prof ![[#WEIGHTS:]] + ; CHECK: [[#TRAP]]: + ; CHECK-NEXT: call void @llvm.trap() + ; CHECK-NEXT: br label %[[#CALL]] + ; CHECK: [[#CALL]]: + ; CHECK-NEXT: call void %x() + ; CHECK-NOT: [ "kcfi"(i32 12345678) ] + call void %x() [ "kcfi"(i32 12345678) ] + ret void +} + +!llvm.module.flags = !{!0} +!0 = !{i32 4, !"kcfi", i32 1} +; CHECK: ![[#WEIGHTS]] = !{!"branch_weights", i32 1, i32 1048575} diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Instrumentation/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Instrumentation/BUILD.gn --- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Instrumentation/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Instrumentation/BUILD.gn @@ -20,6 +20,7 @@ "InstrOrderFile.cpp", "InstrProfiling.cpp", "Instrumentation.cpp", + "KCFI.cpp", "MemProfiler.cpp", "MemorySanitizer.cpp", "PGOInstrumentation.cpp",