Index: compiler-rt/trunk/lib/esan/CMakeLists.txt =================================================================== --- compiler-rt/trunk/lib/esan/CMakeLists.txt +++ compiler-rt/trunk/lib/esan/CMakeLists.txt @@ -11,7 +11,8 @@ esan.cpp esan_flags.cpp esan_interface.cpp - esan_interceptors.cpp) + esan_interceptors.cpp + working_set.cpp) foreach (arch ${ESAN_SUPPORTED_ARCH}) add_compiler_rt_runtime(clang_rt.esan Index: compiler-rt/trunk/lib/esan/esan.cpp =================================================================== --- compiler-rt/trunk/lib/esan/esan.cpp +++ compiler-rt/trunk/lib/esan/esan.cpp @@ -19,6 +19,7 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flag_parser.h" #include "sanitizer_common/sanitizer_flags.h" +#include "working_set.h" // See comment below. extern "C" { @@ -31,6 +32,15 @@ ToolType WhichTool; ShadowMapping Mapping; +// Different tools use different scales within the same shadow mapping scheme. +// The scale used here must match that used by the compiler instrumentation. +// This array is indexed by the ToolType enum. +static const uptr ShadowScale[] = { + 0, // ESAN_None. + 2, // ESAN_CacheFrag: 4B:1B, so 4 to 1 == >>2. + 6, // ESAN_WorkingSet: 64B:1B, so 64 to 1 == >>6. +}; + // We are combining multiple performance tuning tools under the umbrella of // one EfficiencySanitizer super-tool. Most of our tools have very similar // memory access instrumentation, shadow memory mapping, libc interception, @@ -57,6 +67,8 @@ if (WhichTool == ESAN_CacheFrag) { // TODO(bruening): add shadow mapping and update shadow bits here. // We'll move this to cache_frag.cpp once we have something. + } else if (WhichTool == ESAN_WorkingSet) { + processRangeAccessWorkingSet(PC, Addr, Size, IsWrite); } } @@ -113,10 +125,7 @@ static void initializeShadow() { DCHECK(verifyShadowScheme()); - if (WhichTool == ESAN_CacheFrag) - Mapping.initialize(2); // 4B:1B, so 4 to 1 == >>2. - else - UNREACHABLE("unknown tool shadow mapping"); + Mapping.initialize(ShadowScale[WhichTool]); VPrintf(1, "Shadow scale=%d offset=%p\n", Mapping.Scale, Mapping.Offset); @@ -157,7 +166,7 @@ ::__cxa_atexit((void (*)())finalizeLibrary); VPrintf(1, "in esan::%s\n", __FUNCTION__); - if (WhichTool != ESAN_CacheFrag) { + if (WhichTool <= ESAN_None || WhichTool >= ESAN_Max) { Printf("ERROR: unknown tool %d requested\n", WhichTool); Die(); } @@ -165,6 +174,12 @@ initializeShadow(); initializeInterceptors(); + if (WhichTool == ESAN_CacheFrag) { + // FIXME: add runtime code for this tool + } else if (WhichTool == ESAN_WorkingSet) { + initializeWorkingSet(); + } + EsanIsInitialized = true; } @@ -175,6 +190,8 @@ // strategy for how to generate a final report. // We'll move this to cache_frag.cpp once we have something. Report("%s is not finished: nothing yet to report\n", SanitizerToolName); + } else if (WhichTool == ESAN_WorkingSet) { + return finalizeWorkingSet(); } return 0; } Index: compiler-rt/trunk/lib/esan/esan_interface_internal.h =================================================================== --- compiler-rt/trunk/lib/esan/esan_interface_internal.h +++ compiler-rt/trunk/lib/esan/esan_interface_internal.h @@ -28,6 +28,8 @@ typedef enum Type : u32 { ESAN_None = 0, ESAN_CacheFrag, + ESAN_WorkingSet, + ESAN_Max, } ToolType; // This function should be called at the very beginning of the process, Index: compiler-rt/trunk/lib/esan/working_set.h =================================================================== --- compiler-rt/trunk/lib/esan/working_set.h +++ compiler-rt/trunk/lib/esan/working_set.h @@ -0,0 +1,30 @@ +//===-- working_set.h -------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of EfficiencySanitizer, a family of performance tuners. +// +// Header for working-set-specific code. +//===----------------------------------------------------------------------===// + +#ifndef WORKING_SET_H +#define WORKING_SET_H + +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_internal_defs.h" + +namespace __esan { + +void initializeWorkingSet(); +int finalizeWorkingSet(); +void processRangeAccessWorkingSet(uptr PC, uptr Addr, SIZE_T Size, + bool IsWrite); + +} // namespace __esan + +#endif // WORKING_SET_H Index: compiler-rt/trunk/lib/esan/working_set.cpp =================================================================== --- compiler-rt/trunk/lib/esan/working_set.cpp +++ compiler-rt/trunk/lib/esan/working_set.cpp @@ -0,0 +1,90 @@ +//===-- working_set.cpp ---------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of EfficiencySanitizer, a family of performance tuners. +// +// This file contains working-set-specific code. +//===----------------------------------------------------------------------===// + +#include "working_set.h" +#include "esan.h" +#include "esan_flags.h" +#include "esan_shadow.h" + +// We shadow every cache line of app memory with one shadow byte. +// - The highest bit of each shadow byte indicates whether the corresponding +// cache line has ever been accessed. +// - The lowest bit of each shadow byte indicates whether the corresponding +// cache line was accessed since the last sample. +// - The other bits can be used either for a single working set snapshot +// between two consecutive samples, or an aggregate working set snapshot +// over multiple sample periods (future work). +// We live with races in accessing each shadow byte. +typedef unsigned char byte; + +namespace __esan { + +// See the shadow byte layout description above. +static const u32 TotalWorkingSetBitIdx = 7; +static const u32 CurWorkingSetBitIdx = 0; +static const byte ShadowAccessedVal = + (1 << TotalWorkingSetBitIdx) | (1 << CurWorkingSetBitIdx); + +void processRangeAccessWorkingSet(uptr PC, uptr Addr, SIZE_T Size, + bool IsWrite) { + if (Size == 0) + return; + SIZE_T I = 0; + uptr LineSize = getFlags()->cache_line_size; + // As Addr+Size could overflow at the top of a 32-bit address space, + // we avoid the simpler formula that rounds the start and end. + SIZE_T NumLines = Size / LineSize + + // Add any extra at the start or end adding on an extra line: + (LineSize - 1 + Addr % LineSize + Size % LineSize) / LineSize; + byte *Shadow = (byte *)appToShadow(Addr); + // Write shadow bytes until we're word-aligned. + while (I < NumLines && (uptr)Shadow % 4 != 0) { + if ((*Shadow & ShadowAccessedVal) != ShadowAccessedVal) + *Shadow |= ShadowAccessedVal; + ++Shadow; + ++I; + } + // Write whole shadow words at a time. + // Using a word-stride loop improves the runtime of a microbenchmark of + // memset calls by 10%. + u32 WordValue = ShadowAccessedVal | ShadowAccessedVal << 8 | + ShadowAccessedVal << 16 | ShadowAccessedVal << 24; + while (I + 4 <= NumLines) { + if ((*(u32*)Shadow & WordValue) != WordValue) + *(u32*)Shadow |= WordValue; + Shadow += 4; + I += 4; + } + // Write any trailing shadow bytes. + while (I < NumLines) { + if ((*Shadow & ShadowAccessedVal) != ShadowAccessedVal) + *Shadow |= ShadowAccessedVal; + ++Shadow; + ++I; + } +} + +void initializeWorkingSet() { + // The shadow mapping assumes 64 so this cannot be changed. + CHECK(getFlags()->cache_line_size == 64); +} + +int finalizeWorkingSet() { + // FIXME NYI: we need to add memory scanning to report the total lines + // touched, and later add sampling to get intermediate values. + Report("%s is not finished: nothing yet to report\n", SanitizerToolName); + return 0; +} + +} // namespace __esan Index: compiler-rt/trunk/test/esan/TestCases/workingset-memset.cpp =================================================================== --- compiler-rt/trunk/test/esan/TestCases/workingset-memset.cpp +++ compiler-rt/trunk/test/esan/TestCases/workingset-memset.cpp @@ -0,0 +1,21 @@ +// RUN: %clang_esan_wset -O0 %s -o %t 2>&1 +// RUN: %run %t 2>&1 | FileCheck %s + +#include +#include +#include +#include +#include + +int main(int argc, char **argv) { + const int size = 128*1024*1024; + char *p = (char *)mmap(0, size, PROT_READ | PROT_WRITE, + MAP_ANON | MAP_PRIVATE, -1, 0); + // Test the slowpath at different cache line boundaries. + for (int i = 0; i < 630; i++) + memset((char *)p + 63*i, i, 63*i); + munmap(p, size); + return 0; + // FIXME: once the memory scan and size report is in place add it here. + // CHECK: {{.*}}EfficiencySanitizer is not finished: nothing yet to report +} Index: compiler-rt/trunk/test/esan/lit.cfg =================================================================== --- compiler-rt/trunk/test/esan/lit.cfg +++ compiler-rt/trunk/test/esan/lit.cfg @@ -13,12 +13,15 @@ base_cxxflags = config.cxx_mode_flags + base_cflags frag_cflags = (["-fsanitize=efficiency-cache-frag"] + base_cflags) +wset_cflags = (["-fsanitize=efficiency-working-set"] + base_cflags) def build_invocation(compile_flags): return " " + " ".join([config.clang] + compile_flags) + " " config.substitutions.append( ("%clang_esan_frag ", build_invocation(frag_cflags)) ) +config.substitutions.append( ("%clang_esan_wset ", + build_invocation(wset_cflags)) ) default_esan_opts = '' config.substitutions.append(('%env_esan_opts=',