Index: lib/profile/InstrProfData.inc =================================================================== --- lib/profile/InstrProfData.inc +++ lib/profile/InstrProfData.inc @@ -174,13 +174,15 @@ * name hash and the function address. */ VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0) +/* For memory intrinsic functions size profiling. */ +VALUE_PROF_KIND(IPVK_MemOPSize, 1) /* These two kinds must be the last to be * declared. This is to make sure the string * array created with the template can be * indexed with the kind value. */ VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget) -VALUE_PROF_KIND(IPVK_Last, IPVK_IndirectCallTarget) +VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize) #undef VALUE_PROF_KIND /* VALUE_PROF_KIND end */ @@ -605,6 +607,8 @@ #define VARIANT_MASK_IR_PROF (0x1ULL << 56) #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime +#define INSTR_PROF_MEMOP_SMALLVAL_VAR __llvm_profile_memop_smallval +#define INSTR_PROF_MEMOP_LARGEVAL_VAR __llvm_profile_memop_largeval /* The variable that holds the name of the profile data * specified via command line. */ @@ -649,6 +653,9 @@ #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target #define INSTR_PROF_VALUE_PROF_FUNC_STR \ INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) +#define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range +#define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC) /* InstrProfile per-function control data alignment. */ #define INSTR_PROF_DATA_ALIGNMENT 8 Index: lib/profile/InstrProfiling.h =================================================================== --- lib/profile/InstrProfiling.h +++ lib/profile/InstrProfiling.h @@ -199,6 +199,9 @@ */ extern uint64_t INSTR_PROF_RAW_VERSION_VAR; /* __llvm_profile_raw_version */ +extern unsigned INSTR_PROF_MEMOP_SMALLVAL_VAR; /* __llvm_profile_memop_smallval */ +extern unsigned INSTR_PROF_MEMOP_LARGEVAL_VAR; /* __llvm_profile_memop_largeval */ + /*! * This variable is a weak symbol defined in InstrProfiling.c. It allows * compiler instrumentation to provide overriding definition with value Index: lib/profile/InstrProfiling.c =================================================================== --- lib/profile/InstrProfiling.c +++ lib/profile/InstrProfiling.c @@ -19,6 +19,9 @@ COMPILER_RT_WEAK uint64_t INSTR_PROF_RAW_VERSION_VAR = INSTR_PROF_RAW_VERSION; +COMPILER_RT_WEAK unsigned INSTR_PROF_MEMOP_SMALLVAL_VAR = 0; +COMPILER_RT_WEAK unsigned INSTR_PROF_MEMOP_LARGEVAL_VAR = 0; + COMPILER_RT_WEAK char INSTR_PROF_PROFILE_NAME_VAR[1] = {0}; COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_magic(void) { Index: lib/profile/InstrProfilingValue.c =================================================================== --- lib/profile/InstrProfilingValue.c +++ lib/profile/InstrProfilingValue.c @@ -25,6 +25,11 @@ #define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 8 #define INSTR_PROF_VNODE_POOL_SIZE 1024 +static unsigned DefaultSmallTargetValue = 8; +static unsigned DefaultLargeTargetValue = 8192; +static unsigned SmallTargetValue; +static unsigned LargeTargetValue; + #ifndef _MSC_VER /* A shared static pool in addition to the vnodes statically * allocated by the compiler. */ @@ -45,6 +50,15 @@ } if (VPMaxNumValsPerSite > INSTR_PROF_MAX_NUM_VAL_PER_SITE) VPMaxNumValsPerSite = INSTR_PROF_MAX_NUM_VAL_PER_SITE; + if (INSTR_PROF_MEMOP_SMALLVAL_VAR != 0) + SmallTargetValue = INSTR_PROF_MEMOP_SMALLVAL_VAR; + else + SmallTargetValue = DefaultSmallTargetValue; + if (INSTR_PROF_MEMOP_LARGEVAL_VAR != 0) + LargeTargetValue = INSTR_PROF_MEMOP_LARGEVAL_VAR; + else + LargeTargetValue = DefaultLargeTargetValue; + } COMPILER_RT_VISIBILITY void lprofSetMaxValsPerSite(uint32_t MaxVals) { @@ -219,6 +233,60 @@ } } +/* + * Value less than or equal to SmallTargetValue will be have its own entry. + * value greater than SmallTargetValue but less than LargeTargetValue will + * have one entry and value greater than or equal to LargeTargetValue will + * have another entry. + */ +COMPILER_RT_VISIBILITY void +__llvm_profile_instrument_range(uint64_t TargetValue, void *Data, + uint32_t CounterIndex) { + __llvm_profile_data *PData = (__llvm_profile_data *)Data; + if (!PData) + return; + + if (!PData->Values) { + if (!allocateValueProfileCounters(PData)) + return; + } + + ValueProfNode **ValueCounters = (ValueProfNode **)PData->Values; + ValueProfNode *PrevVNode = NULL; + ValueProfNode *CurVNode = ValueCounters[CounterIndex]; + + if (TargetValue >= LargeTargetValue) + TargetValue = -LargeTargetValue; + else if (TargetValue >= SmallTargetValue) + TargetValue = -SmallTargetValue; + while (CurVNode) { + if (TargetValue == CurVNode->Value) { + CurVNode->Count++; + return; + } + PrevVNode = CurVNode; + CurVNode = CurVNode->Next; + } + + CurVNode = allocateOneNode(PData, CounterIndex, TargetValue); + if (!CurVNode) + return; + CurVNode->Value = TargetValue; + CurVNode->Count++; + + uint32_t Success = 0; + if (!ValueCounters[CounterIndex]) + Success = + COMPILER_RT_BOOL_CMPXCHG(&ValueCounters[CounterIndex], 0, CurVNode); + else if (PrevVNode && !PrevVNode->Next) + Success = COMPILER_RT_BOOL_CMPXCHG(&(PrevVNode->Next), 0, CurVNode); + + if (!Success && !hasStaticCounters) { + free(CurVNode); + return; + } +} + /* * A wrapper struct that represents value profile runtime data. * Like InstrProfRecord class which is used by profiling host tools,