[esan|wset] Optionally assume intra-cache-line accesses

[bruening]

Summary

Adds an option -esan-assume-intra-cache-line which causes esan to assume
that a single memory access touches just one cache line, even if it is not
aligned, for better performance at a potential accuracy cost. Experiments
show that the performance difference can be 2x or more, and accuracy loss
is typically negligible, so we turn this on by default. This currently
applies just to the working set tool.

[aizatsky]

lib/Transforms/Instrumentation/EfficiencySanitizer.cpp

712 ↗

(On Diff #59590)

Do you want a statistics for how many were assumed?

[bruening]

Add stat on the number of assumptions.

llvm/trunk/lib/Transforms/Instrumentation/EfficiencySanitizer.cpp

 
 static cl::opt<bool> ClInstrumentLoadsAndStores(
     "esan-instrument-loads-and-stores", cl::init(true),
     cl::desc("Instrument loads and stores"), cl::Hidden);
 static cl::opt<bool> ClInstrumentMemIntrinsics(
     "esan-instrument-memintrinsics", cl::init(true),
     cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden);
 
+// Experiments show that the performance difference can be 2x or more,
+// and accuracy loss is typically negligible, so we turn this on by default.
+static cl::opt<bool> ClAssumeIntraCacheLine(
+    "esan-assume-intra-cache-line", cl::init(true),
+    cl::desc("Assume each memory access touches just one cache line, for "
+             "better performance but with a potential loss of accuracy."),
+    cl::Hidden);
+
 STATISTIC(NumInstrumentedLoads, "Number of instrumented loads");
 STATISTIC(NumInstrumentedStores, "Number of instrumented stores");
 STATISTIC(NumFastpaths, "Number of instrumented fastpaths");
 STATISTIC(NumAccessesWithIrregularSize,
           "Number of accesses with a size outside our targeted callout sizes");
 STATISTIC(NumIgnoredStructs, "Number of ignored structs");
 STATISTIC(NumIgnoredGEPs, "Number of ignored GEP instructions");
 STATISTIC(NumInstrumentedGEPs, "Number of instrumented GEP instructions");
+STATISTIC(NumAssumedIntraCacheLine,
+          "Number of accesses assumed to be intra-cache-line");
 
 static const uint64_t EsanCtorAndDtorPriority = 0;
 static const char *const EsanModuleCtorName = "esan.module_ctor";
 static const char *const EsanModuleDtorName = "esan.module_dtor";
 static const char *const EsanInitName = "__esan_init";
 static const char *const EsanExitName = "__esan_exit";
 
 // We need to specify the tool to the runtime earlier than
bool EfficiencySanitizer::instrumentFastpathWorkingSet(
   const uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy);
   // Bail to the slowpath if the access might touch multiple cache lines.
   // An access aligned to its size is guaranteed to be intra-cache-line.
   // getMemoryAccessFuncIndex has already ruled out a size larger than 16
   // and thus larger than a cache line for platforms this tool targets
   // (and our shadow memory setup assumes 64-byte cache lines).
   assert(TypeSize <= 64);
   if (!(TypeSize == 8 ||
-        (Alignment % (TypeSize / 8)) == 0))
+        (Alignment % (TypeSize / 8)) == 0)) {
+    if (ClAssumeIntraCacheLine)
+      ++NumAssumedIntraCacheLine;
+    else
       return false;
+  }
 
   // We inline instrumentation to set the corresponding shadow bits for
   // each cache line touched by the application.  Here we handle a single
   // load or store where we've already ruled out the possibility that it
   // might touch more than one cache line and thus we simply update the
   // shadow memory for a single cache line.
   // Our shadow memory model is fine with races when manipulating shadow values.
   // We generate the following code:

llvm/trunk/test/Instrumentation/EfficiencySanitizer/working_set_basic.ll

 ; CHECK-NEXT:   %10 = inttoptr i64 %3 to i8*
 ; CHECK-NEXT:   store i8 %9, i8* %10
 ; CHECK-NEXT:   br label %11
 ; CHECK:        %tmp1 = load i64, i64* %a, align 8
 ; CHECK-NEXT:   ret i64 %tmp1
 }
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Not guaranteed to be intra-cache-line
+; Not guaranteed to be intra-cache-line, but our defaults are to
+; assume they are:
 
 define i16 @unaligned2(i16* %a) {
 entry:
   %tmp1 = load i16, i16* %a, align 1
   ret i16 %tmp1
-; CHECK:          %0 = bitcast i16* %a to i8*
-; CHECK-NEXT:   call void @__esan_unaligned_load2(i8* %0)
-; CHECK-NEXT:   %tmp1 = load i16, i16* %a, align 1
+; CHECK:        %0 = ptrtoint i16* %a to i64
+; CHECK-NEXT:   %1 = and i64 %0, 17592186044415
+; CHECK-NEXT:   %2 = add i64 %1, 1337006139375616
+; CHECK-NEXT:   %3 = lshr i64 %2, 6
+; CHECK-NEXT:   %4 = inttoptr i64 %3 to i8*
+; CHECK-NEXT:   %5 = load i8, i8* %4
+; CHECK-NEXT:   %6 = and i8 %5, -127
+; CHECK-NEXT:   %7 = icmp ne i8 %6, -127
+; CHECK-NEXT:   br i1 %7, label %8, label %11
+; CHECK:        %9 = or i8 %5, -127
+; CHECK-NEXT:   %10 = inttoptr i64 %3 to i8*
+; CHECK-NEXT:   store i8 %9, i8* %10
+; CHECK-NEXT:   br label %11
+; CHECK:        %tmp1 = load i16, i16* %a, align 1
 ; CHECK-NEXT:   ret i16 %tmp1
 }
 
 define i32 @unaligned4(i32* %a) {
 entry:
   %tmp1 = load i32, i32* %a, align 2
   ret i32 %tmp1
-; CHECK:          %0 = bitcast i32* %a to i8*
-; CHECK-NEXT:   call void @__esan_unaligned_load4(i8* %0)
-; CHECK-NEXT:   %tmp1 = load i32, i32* %a, align 2
+; CHECK:        %0 = ptrtoint i32* %a to i64
+; CHECK-NEXT:   %1 = and i64 %0, 17592186044415
+; CHECK-NEXT:   %2 = add i64 %1, 1337006139375616
+; CHECK-NEXT:   %3 = lshr i64 %2, 6
+; CHECK-NEXT:   %4 = inttoptr i64 %3 to i8*
+; CHECK-NEXT:   %5 = load i8, i8* %4
+; CHECK-NEXT:   %6 = and i8 %5, -127
+; CHECK-NEXT:   %7 = icmp ne i8 %6, -127
+; CHECK-NEXT:   br i1 %7, label %8, label %11
+; CHECK:        %9 = or i8 %5, -127
+; CHECK-NEXT:   %10 = inttoptr i64 %3 to i8*
+; CHECK-NEXT:   store i8 %9, i8* %10
+; CHECK-NEXT:   br label %11
+; CHECK:        %tmp1 = load i32, i32* %a, align 2
 ; CHECK-NEXT:   ret i32 %tmp1
 }
 
 define i64 @unaligned8(i64* %a) {
 entry:
   %tmp1 = load i64, i64* %a, align 4
   ret i64 %tmp1
-; CHECK:          %0 = bitcast i64* %a to i8*
-; CHECK-NEXT:   call void @__esan_unaligned_load8(i8* %0)
-; CHECK-NEXT:   %tmp1 = load i64, i64* %a, align 4
+; CHECK:        %0 = ptrtoint i64* %a to i64
+; CHECK-NEXT:   %1 = and i64 %0, 17592186044415
+; CHECK-NEXT:   %2 = add i64 %1, 1337006139375616
+; CHECK-NEXT:   %3 = lshr i64 %2, 6
+; CHECK-NEXT:   %4 = inttoptr i64 %3 to i8*
+; CHECK-NEXT:   %5 = load i8, i8* %4
+; CHECK-NEXT:   %6 = and i8 %5, -127
+; CHECK-NEXT:   %7 = icmp ne i8 %6, -127
+; CHECK-NEXT:   br i1 %7, label %8, label %11
+; CHECK:        %9 = or i8 %5, -127
+; CHECK-NEXT:   %10 = inttoptr i64 %3 to i8*
+; CHECK-NEXT:   store i8 %9, i8* %10
+; CHECK-NEXT:   br label %11
+; CHECK:        %tmp1 = load i64, i64* %a, align 4
 ; CHECK-NEXT:   ret i64 %tmp1
 }
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ; Ensure that esan converts intrinsics to calls:
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1)
 declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1)

llvm/trunk/test/Instrumentation/EfficiencySanitizer/working_set_strict.ll

+; Test EfficiencySanitizer working set instrumentation without aggressive
+; optimization flags.
+;
+; RUN: opt < %s -esan -esan-working-set -esan-assume-intra-cache-line=0 -S | FileCheck %s
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Intra-cache-line
+
+define i8 @aligned1(i8* %a) {
+entry:
+  %tmp1 = load i8, i8* %a, align 1
+  ret i8 %tmp1
+; CHECK: @llvm.global_ctors = {{.*}}@esan.module_ctor
+; CHECK:        %0 = ptrtoint i8* %a to i64
+; CHECK-NEXT:   %1 = and i64 %0, 17592186044415
+; CHECK-NEXT:   %2 = add i64 %1, 1337006139375616
+; CHECK-NEXT:   %3 = lshr i64 %2, 6
+; CHECK-NEXT:   %4 = inttoptr i64 %3 to i8*
+; CHECK-NEXT:   %5 = load i8, i8* %4
+; CHECK-NEXT:   %6 = and i8 %5, -127
+; CHECK-NEXT:   %7 = icmp ne i8 %6, -127
+; CHECK-NEXT:   br i1 %7, label %8, label %11
+; CHECK:        %9 = or i8 %5, -127
+; CHECK-NEXT:   %10 = inttoptr i64 %3 to i8*
+; CHECK-NEXT:   store i8 %9, i8* %10
+; CHECK-NEXT:   br label %11
+; CHECK:        %tmp1 = load i8, i8* %a, align 1
+; CHECK-NEXT:   ret i8 %tmp1
+}
+
+define i16 @aligned2(i16* %a) {
+entry:
+  %tmp1 = load i16, i16* %a, align 2
+  ret i16 %tmp1
+; CHECK:        %0 = ptrtoint i16* %a to i64
+; CHECK-NEXT:   %1 = and i64 %0, 17592186044415
+; CHECK-NEXT:   %2 = add i64 %1, 1337006139375616
+; CHECK-NEXT:   %3 = lshr i64 %2, 6
+; CHECK-NEXT:   %4 = inttoptr i64 %3 to i8*
+; CHECK-NEXT:   %5 = load i8, i8* %4
+; CHECK-NEXT:   %6 = and i8 %5, -127
+; CHECK-NEXT:   %7 = icmp ne i8 %6, -127
+; CHECK-NEXT:   br i1 %7, label %8, label %11
+; CHECK:        %9 = or i8 %5, -127
+; CHECK-NEXT:   %10 = inttoptr i64 %3 to i8*
+; CHECK-NEXT:   store i8 %9, i8* %10
+; CHECK-NEXT:   br label %11
+; CHECK:        %tmp1 = load i16, i16* %a, align 2
+; CHECK-NEXT:   ret i16 %tmp1
+}
+
+define i32 @aligned4(i32* %a) {
+entry:
+  %tmp1 = load i32, i32* %a, align 4
+  ret i32 %tmp1
+; CHECK:        %0 = ptrtoint i32* %a to i64
+; CHECK-NEXT:   %1 = and i64 %0, 17592186044415
+; CHECK-NEXT:   %2 = add i64 %1, 1337006139375616
+; CHECK-NEXT:   %3 = lshr i64 %2, 6
+; CHECK-NEXT:   %4 = inttoptr i64 %3 to i8*
+; CHECK-NEXT:   %5 = load i8, i8* %4
+; CHECK-NEXT:   %6 = and i8 %5, -127
+; CHECK-NEXT:   %7 = icmp ne i8 %6, -127
+; CHECK-NEXT:   br i1 %7, label %8, label %11
+; CHECK:        %9 = or i8 %5, -127
+; CHECK-NEXT:   %10 = inttoptr i64 %3 to i8*
+; CHECK-NEXT:   store i8 %9, i8* %10
+; CHECK-NEXT:   br label %11
+; CHECK:        %tmp1 = load i32, i32* %a, align 4
+; CHECK-NEXT:   ret i32 %tmp1
+}
+
+define i64 @aligned8(i64* %a) {
+entry:
+  %tmp1 = load i64, i64* %a, align 8
+  ret i64 %tmp1
+; CHECK:        %0 = ptrtoint i64* %a to i64
+; CHECK-NEXT:   %1 = and i64 %0, 17592186044415
+; CHECK-NEXT:   %2 = add i64 %1, 1337006139375616
+; CHECK-NEXT:   %3 = lshr i64 %2, 6
+; CHECK-NEXT:   %4 = inttoptr i64 %3 to i8*
+; CHECK-NEXT:   %5 = load i8, i8* %4
+; CHECK-NEXT:   %6 = and i8 %5, -127
+; CHECK-NEXT:   %7 = icmp ne i8 %6, -127
+; CHECK-NEXT:   br i1 %7, label %8, label %11
+; CHECK:        %9 = or i8 %5, -127
+; CHECK-NEXT:   %10 = inttoptr i64 %3 to i8*
+; CHECK-NEXT:   store i8 %9, i8* %10
+; CHECK-NEXT:   br label %11
+; CHECK:        %tmp1 = load i64, i64* %a, align 8
+; CHECK-NEXT:   ret i64 %tmp1
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Not guaranteed to be intra-cache-line
+
+define i16 @unaligned2(i16* %a) {
+entry:
+  %tmp1 = load i16, i16* %a, align 1
+  ret i16 %tmp1
+; CHECK:        %0 = bitcast i16* %a to i8*
+; CHECK-NEXT:   call void @__esan_unaligned_load2(i8* %0)
+; CHECK-NEXT:   %tmp1 = load i16, i16* %a, align 1
+; CHECK-NEXT:   ret i16 %tmp1
+}
+
+define i32 @unaligned4(i32* %a) {
+entry:
+  %tmp1 = load i32, i32* %a, align 2
+  ret i32 %tmp1
+; CHECK:        %0 = bitcast i32* %a to i8*
+; CHECK-NEXT:   call void @__esan_unaligned_load4(i8* %0)
+; CHECK-NEXT:   %tmp1 = load i32, i32* %a, align 2
+; CHECK-NEXT:   ret i32 %tmp1
+}
+
+define i64 @unaligned8(i64* %a) {
+entry:
+  %tmp1 = load i64, i64* %a, align 4
+  ret i64 %tmp1
+; CHECK:        %0 = bitcast i64* %a to i8*
+; CHECK-NEXT:   call void @__esan_unaligned_load8(i8* %0)
+; CHECK-NEXT:   %tmp1 = load i64, i64* %a, align 4
+; CHECK-NEXT:   ret i64 %tmp1
+}