diff --git a/compiler-rt/lib/dfsan/dfsan.cpp b/compiler-rt/lib/dfsan/dfsan.cpp --- a/compiler-rt/lib/dfsan/dfsan.cpp +++ b/compiler-rt/lib/dfsan/dfsan.cpp @@ -18,15 +18,16 @@ // prefixed __dfsan_. //===----------------------------------------------------------------------===// +#include "dfsan/dfsan.h" + #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_file.h" -#include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_flag_parser.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_libc.h" -#include "dfsan/dfsan.h" - using namespace __dfsan; typedef atomic_uint16_t atomic_dfsan_label; @@ -37,6 +38,10 @@ static atomic_dfsan_label __dfsan_last_label; static dfsan_label_info __dfsan_label_info[kNumLabels]; +// True if dfsan_use_fast16labels() was called. +static bool use_fast16labels = false; +static bool flags_initialized = false; + Flags __dfsan::flags_data; SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL dfsan_label __dfsan_retval_tls; @@ -164,6 +169,13 @@ Die(); } +// Configures the DFSan runtime to use fast16labels mode. +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_use_fast16labels() { + use_fast16labels = true; + if (flags_initialized) + flags().fast16labels = true; +} + // Resolves the union of two unequal labels. Nonequality is a precondition for // this function (the instrumentation pass inlines the equality test). extern "C" SANITIZER_INTERFACE_ATTRIBUTE @@ -403,6 +415,13 @@ InitializeCommonFlags(); if (Verbosity()) ReportUnrecognizedFlags(); if (common_flags()->help) parser.PrintFlagDescriptions(); + + // If dfsan_use_fast16labels() was called before InitializeFlags(), we need + // to enable fast16labels mode here. + if (use_fast16labels) + flags().fast16labels = true; + + flags_initialized = true; } static void InitializePlatformEarly() { diff --git a/compiler-rt/lib/dfsan/done_abilist.txt b/compiler-rt/lib/dfsan/done_abilist.txt --- a/compiler-rt/lib/dfsan/done_abilist.txt +++ b/compiler-rt/lib/dfsan/done_abilist.txt @@ -28,6 +28,8 @@ fun:dfsan_set_write_callback=custom fun:dfsan_flush=uninstrumented fun:dfsan_flush=discard +fun:dfsan_use_fast16labels=uninstrumented +fun:dfsan_use_fast16labels=discard ############################################################################### # glibc diff --git a/compiler-rt/test/dfsan/fast16labels.c b/compiler-rt/test/dfsan/fast16labels.c --- a/compiler-rt/test/dfsan/fast16labels.c +++ b/compiler-rt/test/dfsan/fast16labels.c @@ -1,15 +1,15 @@ -// RUN: %clang_dfsan %s -o %t -// RUN: DFSAN_OPTIONS=fast16labels=1 %run %t -// RUN: DFSAN_OPTIONS=fast16labels=1 not %run %t dfsan_create_label 2>&1 \ +// RUN: %clang_dfsan %s -mllvm -fast-16-labels -o %t +// RUN: %run %t +// RUN: not %run %t dfsan_create_label 2>&1 \ // RUN: | FileCheck %s --check-prefix=CREATE-LABEL -// RUN: DFSAN_OPTIONS=fast16labels=1 not %run %t dfsan_get_label_info 2>&1 \ +// RUN: not %run %t dfsan_get_label_info 2>&1 \ // RUN: | FileCheck %s --check-prefix=GET-LABEL-INFO -// RUN: DFSAN_OPTIONS=fast16labels=1 not %run %t dfsan_has_label_with_desc \ -// RUN: 2>&1 | FileCheck %s --check-prefix=HAS-LABEL-WITH-DESC -// RUN: DFSAN_OPTIONS=fast16labels=1:dump_labels_at_exit=/dev/stdout not %run \ -// RUN: %t 2>&1 | FileCheck %s --check-prefix=DUMP-LABELS +// RUN: not %run %t dfsan_has_label_with_desc 2>&1 \ +// RUN: | FileCheck %s --check-prefix=HAS-LABEL-WITH-DESC +// RUN: DFSAN_OPTIONS=dump_labels_at_exit=/dev/stdout not %run %t 2>&1 \ +// RUN: | FileCheck %s --check-prefix=DUMP-LABELS // -// Tests DFSAN_OPTIONS=fast16labels=1 +// Tests fast16labels mode. // #include diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp --- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp @@ -91,6 +91,7 @@ #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/Local.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" #include #include #include @@ -176,6 +177,14 @@ cl::desc("Insert calls to __dfsan_*_callback functions on data events."), cl::Hidden, cl::init(false)); +// Use a distinct bit for each base label, enabling faster unions with less +// instrumentation. Limits the max number of base labels to 16. +static cl::opt ClFast16Labels( + "fast-16-labels", + cl::desc("Use more efficient instrumentation, limiting the number of " + "labels to 16."), + cl::Hidden, cl::init(false)); + static StringRef GetGlobalTypeString(const GlobalValue &G) { // Types of GlobalVariables are always pointer types. Type *GType = G.getValueType(); @@ -359,6 +368,7 @@ FunctionType *DFSanVarargWrapperFnTy; FunctionType *DFSanLoadStoreCmpCallbackFnTy; FunctionType *DFSanMemTransferCallbackFnTy; + FunctionType *DFSanUseFast16LabelsFnTy; FunctionCallee DFSanUnionFn; FunctionCallee DFSanCheckedUnionFn; FunctionCallee DFSanUnionLoadFn; @@ -370,6 +380,7 @@ FunctionCallee DFSanStoreCallbackFn; FunctionCallee DFSanMemTransferCallbackFn; FunctionCallee DFSanCmpCallbackFn; + FunctionCallee DFSanUseFast16LabelsFn; MDNode *ColdCallWeights; DFSanABIList ABIList; DenseMap UnwrappedFnMap; @@ -612,6 +623,8 @@ DFSanMemTransferCallbackFnTy = FunctionType::get(Type::getVoidTy(*Ctx), DFSanMemTransferCallbackArgs, /*isVarArg=*/false); + DFSanUseFast16LabelsFnTy = + FunctionType::get(Type::getVoidTy(*Ctx), {}, /*isVarArg=*/false); if (GetArgTLSPtr) { Type *ArgTLSTy = ArrayType::get(ShadowTy, 64); @@ -793,6 +806,9 @@ Mod->getOrInsertFunction("__dfsan_nonzero_label", DFSanNonzeroLabelFnTy); DFSanVarargWrapperFn = Mod->getOrInsertFunction("__dfsan_vararg_wrapper", DFSanVarargWrapperFnTy); + + DFSanUseFast16LabelsFn = Mod->getOrInsertFunction("dfsan_use_fast16labels", + DFSanUseFast16LabelsFnTy); } // Initializes event callback functions and declare them in the module @@ -1053,6 +1069,24 @@ } } + if (ClFast16Labels) { + // Call dfsan_use_fast16labels() during preinit to enable runtime support + // for fast16labels mode. + auto *UseFast16LabelsInit = + new GlobalVariable(M, PointerType::getUnqual(DFSanUseFast16LabelsFnTy), + /*constant=*/true, GlobalVariable::PrivateLinkage, + cast(DFSanUseFast16LabelsFn.getCallee()), + "__dfsan_fast16labels_preinit"); + UseFast16LabelsInit->setSection(".preinit_array"); + + // Dedup with comdat + if (Triple(M.getTargetTriple()).supportsCOMDAT()) + UseFast16LabelsInit->setComdat( + M.getOrInsertComdat("dfsan_fast16labels.module_preinit")); + + appendToUsed(M, UseFast16LabelsInit); + } + return Changed || !FnsToInstrument.empty() || M.global_size() != InitialGlobalSize || M.size() != InitialModuleSize; } @@ -1179,7 +1213,10 @@ return CCS.Shadow; IRBuilder<> IRB(Pos); - if (AvoidNewBlocks) { + if (ClFast16Labels) { + CCS.Block = Pos->getParent(); + CCS.Shadow = IRB.CreateOr(V1, V2); + } else if (AvoidNewBlocks) { CallInst *Call = IRB.CreateCall(DFS.DFSanCheckedUnionFn, {V1, V2}); Call->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt); Call->addParamAttr(0, Attribute::ZExt); @@ -1289,6 +1326,30 @@ IRB.CreateAlignedLoad(DFS.ShadowTy, ShadowAddr1, ShadowAlign), Pos); } } + + if (ClFast16Labels && Size % (64 / DFS.ShadowWidthBits) == 0) { + // First OR all the WideShadows, then OR individual shadows within the + // combined WideShadow. This is fewer instructions than ORing shadows + // individually. + IRBuilder<> IRB(Pos); + Value *WideAddr = + IRB.CreateBitCast(ShadowAddr, Type::getInt64PtrTy(*DFS.Ctx)); + Value *CombinedWideShadow = + IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign); + for (uint64_t Ofs = 64 / DFS.ShadowWidthBits; Ofs != Size; + Ofs += 64 / DFS.ShadowWidthBits) { + WideAddr = IRB.CreateGEP(Type::getInt64Ty(*DFS.Ctx), WideAddr, + ConstantInt::get(DFS.IntptrTy, 1)); + Value *NextWideShadow = + IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign); + CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, NextWideShadow); + } + for (unsigned Width = 32; Width >= DFS.ShadowWidthBits; Width >>= 1) { + Value *ShrShadow = IRB.CreateLShr(CombinedWideShadow, Width); + CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, ShrShadow); + } + return IRB.CreateTrunc(CombinedWideShadow, DFS.ShadowTy); + } if (!AvoidNewBlocks && Size % (64 / DFS.ShadowWidthBits) == 0) { // Fast path for the common case where each byte has identical shadow: load // shadow 64 bits at a time, fall out to a __dfsan_union_load call if any diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll b/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll @@ -0,0 +1,100 @@ +; Test that fast-16-labels mode uses inline ORs rather than calling +; __dfsan_union or __dfsan_union_load. +; RUN: opt < %s -dfsan -fast-16-labels -S | FileCheck %s --implicit-check-not="call{{.*}}__dfsan_union" +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define i8 @add(i8 %a, i8 %b) { + ; CHECK-LABEL: define i8 @"dfs$add" + ; CHECK-DAG: %[[ALABEL:.*]] = load{{.*}}__dfsan_arg_tls, i64 0, i64 0 + ; CHECK-DAG: %[[BLABEL:.*]] = load{{.*}}__dfsan_arg_tls, i64 0, i64 1 + ; CHECK: %[[ADDLABEL:.*]] = or i16 %[[ALABEL]], %[[BLABEL]] + ; CHECK: add i8 + ; CHECK: store i16 %[[ADDLABEL]], i16* @__dfsan_retval_tls + ; CHECK: ret i8 + %c = add i8 %a, %b + ret i8 %c +} + +define i8 @load8(i8* %p) { + ; CHECK-LABEL: define i8 @"dfs$load8" + ; CHECK: load i16, i16* + ; CHECK: ptrtoint i8* {{.*}} to i64 + ; CHECK: and i64 + ; CHECK: mul i64 + ; CHECK: inttoptr i64 + ; CHECK: load i16, i16* + ; CHECK: or i16 + ; CHECK: load i8, i8* + ; CHECK: store i16 {{.*}} @__dfsan_retval_tls + ; CHECK: ret i8 + + %a = load i8, i8* %p + ret i8 %a +} + +define i16 @load16(i16* %p) { + ; CHECK-LABEL: define i16 @"dfs$load16" + ; CHECK: ptrtoint i16* + ; CHECK: and i64 + ; CHECK: mul i64 + ; CHECK: inttoptr i64 {{.*}} i16* + ; CHECK: getelementptr i16 + ; CHECK: load i16, i16* + ; CHECK: load i16, i16* + ; CHECK: or i16 + ; CHECK: or i16 + ; CHECK: load i16, i16* + ; CHECK: store {{.*}} @__dfsan_retval_tls + ; CHECK: ret i16 + + %a = load i16, i16* %p + ret i16 %a +} + +define i32 @load32(i32* %p) { + ; CHECK-LABEL: define i32 @"dfs$load32" + ; CHECK: ptrtoint i32* + ; CHECK: and i64 + ; CHECK: mul i64 + ; CHECK: inttoptr i64 {{.*}} i16* + ; CHECK: bitcast i16* {{.*}} i64* + ; CHECK: load i64, i64* + ; CHECK: lshr i64 {{.*}}, 32 + ; CHECK: or i64 + ; CHECK: lshr i64 {{.*}}, 16 + ; CHECK: or i64 + ; CHECK: trunc i64 {{.*}} i16 + ; CHECK: or i16 + ; CHECK: load i32, i32* + ; CHECK: store i16 {{.*}} @__dfsan_retval_tls + ; CHECK: ret i32 + + %a = load i32, i32* %p + ret i32 %a +} + +define i64 @load64(i64* %p) { + ; CHECK-LABEL: define i64 @"dfs$load64" + ; CHECK: ptrtoint i64* + ; CHECK: and i64 + ; CHECK: mul i64 + ; CHECK: inttoptr i64 {{.*}} i16* + ; CHECK: bitcast i16* {{.*}} i64* + ; CHECK: load i64, i64* + ; CHECK: getelementptr i64, i64* {{.*}}, i64 1 + ; CHECK: load i64, i64* + ; CHECK: or i64 + ; CHECK: lshr i64 {{.*}}, 32 + ; CHECK: or i64 + ; CHECK: lshr i64 {{.*}}, 16 + ; CHECK: or i64 + ; CHECK: trunc i64 {{.*}} i16 + ; CHECK: or i16 + ; CHECK: load i64, i64* + ; CHECK: store i16 {{.*}} @__dfsan_retval_tls + ; CHECK: ret i64 + + %a = load i64, i64* %p + ret i64 %a +}