Index: compiler-rt/include/CMakeLists.txt =================================================================== --- compiler-rt/include/CMakeLists.txt +++ compiler-rt/include/CMakeLists.txt @@ -30,7 +30,7 @@ if (COMPILER_RT_BUILD_PROFILE) set(PROFILE_HEADERS - profile/InstrProfData.inc + ../../llvm/include/llvm/ProfileData/InstrProfData.inc ) endif(COMPILER_RT_BUILD_PROFILE) Index: compiler-rt/include/profile/InstrProfData.inc =================================================================== --- compiler-rt/include/profile/InstrProfData.inc +++ /dev/null @@ -1,910 +0,0 @@ -/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ -|* -|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -|* See https://llvm.org/LICENSE.txt for license information. -|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -|* -\*===----------------------------------------------------------------------===*/ -/* - * This is the master file that defines all the data structure, signature, - * constant literals that are shared across profiling runtime library, - * compiler (instrumentation), and host tools (reader/writer). The entities - * defined in this file affect the profile runtime ABI, the raw profile format, - * or both. - * - * The file has two identical copies. The master copy lives in LLVM and - * the other one sits in compiler-rt/lib/profile directory. To make changes - * in this file, first modify the master copy and copy it over to compiler-rt. - * Testing of any change in this file can start only after the two copies are - * synced up. - * - * The first part of the file includes macros that defines types, names, and - * initializers for the member fields of the core data structures. The field - * declarations for one structure is enabled by defining the field activation - * macro associated with that structure. Only one field activation record - * can be defined at one time and the rest definitions will be filtered out by - * the preprocessor. - * - * Examples of how the template is used to instantiate structure definition: - * 1. To declare a structure: - * - * struct ProfData { - * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ - * Type Name; - * #include "llvm/ProfileData/InstrProfData.inc" - * }; - * - * 2. To construct LLVM type arrays for the struct type: - * - * Type *DataTypes[] = { - * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ - * LLVMType, - * #include "llvm/ProfileData/InstrProfData.inc" - * }; - * - * 4. To construct constant array for the initializers: - * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ - * Initializer, - * Constant *ConstantVals[] = { - * #include "llvm/ProfileData/InstrProfData.inc" - * }; - * - * - * The second part of the file includes definitions all other entities that - * are related to runtime ABI and format. When no field activation macro is - * defined, this file can be included to introduce the definitions. - * -\*===----------------------------------------------------------------------===*/ - -/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in - * the compiler runtime. */ -#ifndef INSTR_PROF_VISIBILITY -#define INSTR_PROF_VISIBILITY -#endif - -/* INSTR_PROF_DATA start. */ -/* Definition of member fields of the per-function control structure. */ -#ifndef INSTR_PROF_DATA -#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) -#else -#define INSTR_PROF_DATA_DEFINED -#endif -INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ - ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ - IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName())))) -INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ - ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ - Inc->getHash()->getZExtValue())) -INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ - ConstantExpr::getBitCast(CounterPtr, \ - llvm::Type::getInt64PtrTy(Ctx))) -/* This is used to map function pointers for the indirect call targets to - * function name hashes during the conversion from raw to merged profile - * data. - */ -INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \ - FunctionAddr) -INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ - ValuesPtrExpr) -INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ - ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) -INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ - ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) -#undef INSTR_PROF_DATA -/* INSTR_PROF_DATA end. */ - - -/* This is an internal data structure used by value profiler. It - * is defined here to allow serialization code sharing by LLVM - * to be used in unit test. - * - * typedef struct ValueProfNode { - * // InstrProfValueData VData; - * uint64_t Value; - * uint64_t Count; - * struct ValueProfNode *Next; - * } ValueProfNode; - */ -/* INSTR_PROF_VALUE_NODE start. */ -#ifndef INSTR_PROF_VALUE_NODE -#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) -#else -#define INSTR_PROF_DATA_DEFINED -#endif -INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \ - ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) -INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \ - ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) -INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ - ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0)) -#undef INSTR_PROF_VALUE_NODE -/* INSTR_PROF_VALUE_NODE end. */ - -/* INSTR_PROF_RAW_HEADER start */ -/* Definition of member fields of the raw profile header data structure. */ -#ifndef INSTR_PROF_RAW_HEADER -#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) -#else -#define INSTR_PROF_DATA_DEFINED -#endif -INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) -INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) -INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) -INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) -INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) -INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) -INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) -INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) -INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) -INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) -#undef INSTR_PROF_RAW_HEADER -/* INSTR_PROF_RAW_HEADER end */ - -/* VALUE_PROF_FUNC_PARAM start */ -/* Definition of parameter types of the runtime API used to do value profiling - * for a given value site. - */ -#ifndef VALUE_PROF_FUNC_PARAM -#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) -#define INSTR_PROF_COMMA -#else -#define INSTR_PROF_DATA_DEFINED -#define INSTR_PROF_COMMA , -#endif -VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ - INSTR_PROF_COMMA -VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA -#ifndef VALUE_RANGE_PROF -VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) -#else /* VALUE_RANGE_PROF */ -/* FIXME: This is to be removed after switching to the new memop value - * profiling. */ -VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \ - INSTR_PROF_COMMA -VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \ - INSTR_PROF_COMMA -VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeLast, Type::getInt64Ty(Ctx)) \ - INSTR_PROF_COMMA -VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) -#endif /*VALUE_RANGE_PROF */ -#undef VALUE_PROF_FUNC_PARAM -#undef INSTR_PROF_COMMA -/* VALUE_PROF_FUNC_PARAM end */ - -/* VALUE_PROF_KIND start */ -#ifndef VALUE_PROF_KIND -#define VALUE_PROF_KIND(Enumerator, Value, Descr) -#else -#define INSTR_PROF_DATA_DEFINED -#endif -/* For indirect function call value profiling, the addresses of the target - * functions are profiled by the instrumented code. The target addresses are - * written in the raw profile data and converted to target function name's MD5 - * hash by the profile reader during deserialization. Typically, this happens - * when the raw profile data is read during profile merging. - * - * For this remapping the ProfData is used. ProfData contains both the function - * name hash and the function address. - */ -VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") -/* For memory intrinsic functions size profiling. */ -VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") -/* These two kinds must be the last to be - * declared. This is to make sure the string - * array created with the template can be - * indexed with the kind value. - */ -VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") -VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") - -#undef VALUE_PROF_KIND -/* VALUE_PROF_KIND end */ - -#undef COVMAP_V2_OR_V3 -#ifdef COVMAP_V2 -#define COVMAP_V2_OR_V3 -#endif -#ifdef COVMAP_V3 -#define COVMAP_V2_OR_V3 -#endif - -/* COVMAP_FUNC_RECORD start */ -/* Definition of member fields of the function record structure in coverage - * map. - */ -#ifndef COVMAP_FUNC_RECORD -#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) -#else -#define INSTR_PROF_DATA_DEFINED -#endif -#ifdef COVMAP_V1 -COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ - NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ - llvm::Type::getInt8PtrTy(Ctx))) -COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ - llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ - NameValue.size())) -#endif -#ifdef COVMAP_V2_OR_V3 -COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ - llvm::ConstantInt::get( \ - llvm::Type::getInt64Ty(Ctx), NameHash)) -#endif -COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ - llvm::ConstantInt::get( \ - llvm::Type::getInt32Ty(Ctx), CoverageMapping.size())) -COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ - llvm::ConstantInt::get( \ - llvm::Type::getInt64Ty(Ctx), FuncHash)) -#ifdef COVMAP_V3 -COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \ - llvm::ConstantInt::get( \ - llvm::Type::getInt64Ty(Ctx), FilenamesRef)) -COVMAP_FUNC_RECORD(const char, \ - llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \ - CoverageMapping.size()), \ - CoverageMapping, - llvm::ConstantDataArray::getRaw( \ - CoverageMapping, CoverageMapping.size(), \ - llvm::Type::getInt8Ty(Ctx))) -#endif -#undef COVMAP_FUNC_RECORD -/* COVMAP_FUNC_RECORD end. */ - -/* COVMAP_HEADER start */ -/* Definition of member fields of coverage map header. - */ -#ifndef COVMAP_HEADER -#define COVMAP_HEADER(Type, LLVMType, Name, Initializer) -#else -#define INSTR_PROF_DATA_DEFINED -#endif -COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \ - llvm::ConstantInt::get(Int32Ty, NRecords)) -COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ - llvm::ConstantInt::get(Int32Ty, FilenamesSize)) -COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ - llvm::ConstantInt::get(Int32Ty, CoverageMappingSize)) -COVMAP_HEADER(uint32_t, Int32Ty, Version, \ - llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion)) -#undef COVMAP_HEADER -/* COVMAP_HEADER end. */ - - -#ifdef INSTR_PROF_SECT_ENTRY -#define INSTR_PROF_DATA_DEFINED -INSTR_PROF_SECT_ENTRY(IPSK_data, \ - INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ - INSTR_PROF_DATA_COFF, "__DATA,") -INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ - INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ - INSTR_PROF_CNTS_COFF, "__DATA,") -INSTR_PROF_SECT_ENTRY(IPSK_name, \ - INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ - INSTR_PROF_NAME_COFF, "__DATA,") -INSTR_PROF_SECT_ENTRY(IPSK_vals, \ - INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ - INSTR_PROF_VALS_COFF, "__DATA,") -INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ - INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ - INSTR_PROF_VNODES_COFF, "__DATA,") -INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ - INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ - INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") -INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ - INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \ - INSTR_PROF_COVFUN_COFF, "__LLVM_COV,") -INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ - INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") - -#undef INSTR_PROF_SECT_ENTRY -#endif - - -#ifdef INSTR_PROF_VALUE_PROF_DATA -#define INSTR_PROF_DATA_DEFINED - -#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255 -/*! - * This is the header of the data structure that defines the on-disk - * layout of the value profile data of a particular kind for one function. - */ -typedef struct ValueProfRecord { - /* The kind of the value profile record. */ - uint32_t Kind; - /* - * The number of value profile sites. It is guaranteed to be non-zero; - * otherwise the record for this kind won't be emitted. - */ - uint32_t NumValueSites; - /* - * The first element of the array that stores the number of profiled - * values for each value site. The size of the array is NumValueSites. - * Since NumValueSites is greater than zero, there is at least one - * element in the array. - */ - uint8_t SiteCountArray[1]; - - /* - * The fake declaration is for documentation purpose only. - * Align the start of next field to be on 8 byte boundaries. - uint8_t Padding[X]; - */ - - /* The array of value profile data. The size of the array is the sum - * of all elements in SiteCountArray[]. - InstrProfValueData ValueData[]; - */ - -#ifdef __cplusplus - /*! - * Return the number of value sites. - */ - uint32_t getNumValueSites() const { return NumValueSites; } - /*! - * Read data from this record and save it to Record. - */ - void deserializeTo(InstrProfRecord &Record, - InstrProfSymtab *SymTab); - /* - * In-place byte swap: - * Do byte swap for this instance. \c Old is the original order before - * the swap, and \c New is the New byte order. - */ - void swapBytes(support::endianness Old, support::endianness New); -#endif -} ValueProfRecord; - -/*! - * Per-function header/control data structure for value profiling - * data in indexed format. - */ -typedef struct ValueProfData { - /* - * Total size in bytes including this field. It must be a multiple - * of sizeof(uint64_t). - */ - uint32_t TotalSize; - /* - *The number of value profile kinds that has value profile data. - * In this implementation, a value profile kind is considered to - * have profile data if the number of value profile sites for the - * kind is not zero. More aggressively, the implementation can - * choose to check the actual data value: if none of the value sites - * has any profiled values, the kind can be skipped. - */ - uint32_t NumValueKinds; - - /* - * Following are a sequence of variable length records. The prefix/header - * of each record is defined by ValueProfRecord type. The number of - * records is NumValueKinds. - * ValueProfRecord Record_1; - * ValueProfRecord Record_N; - */ - -#if __cplusplus - /*! - * Return the total size in bytes of the on-disk value profile data - * given the data stored in Record. - */ - static uint32_t getSize(const InstrProfRecord &Record); - /*! - * Return a pointer to \c ValueProfData instance ready to be streamed. - */ - static std::unique_ptr - serializeFrom(const InstrProfRecord &Record); - /*! - * Check the integrity of the record. - */ - Error checkIntegrity(); - /*! - * Return a pointer to \c ValueProfileData instance ready to be read. - * All data in the instance are properly byte swapped. The input - * data is assumed to be in little endian order. - */ - static Expected> - getValueProfData(const unsigned char *SrcBuffer, - const unsigned char *const SrcBufferEnd, - support::endianness SrcDataEndianness); - /*! - * Swap byte order from \c Endianness order to host byte order. - */ - void swapBytesToHost(support::endianness Endianness); - /*! - * Swap byte order from host byte order to \c Endianness order. - */ - void swapBytesFromHost(support::endianness Endianness); - /*! - * Return the total size of \c ValueProfileData. - */ - uint32_t getSize() const { return TotalSize; } - /*! - * Read data from this data and save it to \c Record. - */ - void deserializeTo(InstrProfRecord &Record, - InstrProfSymtab *SymTab); - void operator delete(void *ptr) { ::operator delete(ptr); } -#endif -} ValueProfData; - -/* - * The closure is designed to abstact away two types of value profile data: - * - InstrProfRecord which is the primary data structure used to - * represent profile data in host tools (reader, writer, and profile-use) - * - value profile runtime data structure suitable to be used by C - * runtime library. - * - * Both sources of data need to serialize to disk/memory-buffer in common - * format: ValueProfData. The abstraction allows compiler-rt's raw profiler - * writer to share the same format and code with indexed profile writer. - * - * For documentation of the member methods below, refer to corresponding methods - * in class InstrProfRecord. - */ -typedef struct ValueProfRecordClosure { - const void *Record; - uint32_t (*GetNumValueKinds)(const void *Record); - uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); - uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); - uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); - - /* - * After extracting the value profile data from the value profile record, - * this method is used to map the in-memory value to on-disk value. If - * the method is null, value will be written out untranslated. - */ - uint64_t (*RemapValueData)(uint32_t, uint64_t Value); - void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, - uint32_t S); - ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); -} ValueProfRecordClosure; - -INSTR_PROF_VISIBILITY ValueProfRecord * -getFirstValueProfRecord(ValueProfData *VPD); -INSTR_PROF_VISIBILITY ValueProfRecord * -getValueProfRecordNext(ValueProfRecord *VPR); -INSTR_PROF_VISIBILITY InstrProfValueData * -getValueProfRecordValueData(ValueProfRecord *VPR); -INSTR_PROF_VISIBILITY uint32_t -getValueProfRecordHeaderSize(uint32_t NumValueSites); - -#undef INSTR_PROF_VALUE_PROF_DATA -#endif /* INSTR_PROF_VALUE_PROF_DATA */ - - -#ifdef INSTR_PROF_COMMON_API_IMPL -#define INSTR_PROF_DATA_DEFINED -#ifdef __cplusplus -#define INSTR_PROF_INLINE inline -#define INSTR_PROF_NULLPTR nullptr -#else -#define INSTR_PROF_INLINE -#define INSTR_PROF_NULLPTR NULL -#endif - -#ifndef offsetof -#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) -#endif - -/*! - * Return the \c ValueProfRecord header size including the - * padding bytes. - */ -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { - uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + - sizeof(uint8_t) * NumValueSites; - /* Round the size to multiple of 8 bytes. */ - Size = (Size + 7) & ~7; - return Size; -} - -/*! - * Return the total size of the value profile record including the - * header and the value data. - */ -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -uint32_t getValueProfRecordSize(uint32_t NumValueSites, - uint32_t NumValueData) { - return getValueProfRecordHeaderSize(NumValueSites) + - sizeof(InstrProfValueData) * NumValueData; -} - -/*! - * Return the pointer to the start of value data array. - */ -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { - return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( - This->NumValueSites)); -} - -/*! - * Return the total number of value data for \c This record. - */ -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { - uint32_t NumValueData = 0; - uint32_t I; - for (I = 0; I < This->NumValueSites; I++) - NumValueData += This->SiteCountArray[I]; - return NumValueData; -} - -/*! - * Use this method to advance to the next \c This \c ValueProfRecord. - */ -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { - uint32_t NumValueData = getValueProfRecordNumValueData(This); - return (ValueProfRecord *)((char *)This + - getValueProfRecordSize(This->NumValueSites, - NumValueData)); -} - -/*! - * Return the first \c ValueProfRecord instance. - */ -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { - return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); -} - -/* Closure based interfaces. */ - -/*! - * Return the total size in bytes of the on-disk value profile data - * given the data stored in Record. - */ -INSTR_PROF_VISIBILITY uint32_t -getValueProfDataSize(ValueProfRecordClosure *Closure) { - uint32_t Kind; - uint32_t TotalSize = sizeof(ValueProfData); - const void *Record = Closure->Record; - - for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { - uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); - if (!NumValueSites) - continue; - TotalSize += getValueProfRecordSize(NumValueSites, - Closure->GetNumValueData(Record, Kind)); - } - return TotalSize; -} - -/*! - * Extract value profile data of a function for the profile kind \c ValueKind - * from the \c Closure and serialize the data into \c This record instance. - */ -INSTR_PROF_VISIBILITY void -serializeValueProfRecordFrom(ValueProfRecord *This, - ValueProfRecordClosure *Closure, - uint32_t ValueKind, uint32_t NumValueSites) { - uint32_t S; - const void *Record = Closure->Record; - This->Kind = ValueKind; - This->NumValueSites = NumValueSites; - InstrProfValueData *DstVD = getValueProfRecordValueData(This); - - for (S = 0; S < NumValueSites; S++) { - uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); - This->SiteCountArray[S] = ND; - Closure->GetValueForSite(Record, DstVD, ValueKind, S); - DstVD += ND; - } -} - -/*! - * Extract value profile data of a function from the \c Closure - * and serialize the data into \c DstData if it is not NULL or heap - * memory allocated by the \c Closure's allocator method. If \c - * DstData is not null, the caller is expected to set the TotalSize - * in DstData. - */ -INSTR_PROF_VISIBILITY ValueProfData * -serializeValueProfDataFrom(ValueProfRecordClosure *Closure, - ValueProfData *DstData) { - uint32_t Kind; - uint32_t TotalSize = - DstData ? DstData->TotalSize : getValueProfDataSize(Closure); - - ValueProfData *VPD = - DstData ? DstData : Closure->AllocValueProfData(TotalSize); - - VPD->TotalSize = TotalSize; - VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); - ValueProfRecord *VR = getFirstValueProfRecord(VPD); - for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { - uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); - if (!NumValueSites) - continue; - serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); - VR = getValueProfRecordNext(VR); - } - return VPD; -} - -#undef INSTR_PROF_COMMON_API_IMPL -#endif /* INSTR_PROF_COMMON_API_IMPL */ - -/*============================================================================*/ - -#ifndef INSTR_PROF_DATA_DEFINED - -#ifndef INSTR_PROF_DATA_INC -#define INSTR_PROF_DATA_INC - -/* Helper macros. */ -#define INSTR_PROF_SIMPLE_QUOTE(x) #x -#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) -#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y -#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) - -/* Magic number to detect file format and endianness. - * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, - * so that utilities, like strings, don't grab it as a string. 129 is also - * invalid UTF-8, and high enough to be interesting. - * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" - * for 32-bit platforms. - */ -#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ - (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ - (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 -#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ - (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ - (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 - -/* Raw profile format version (start from 1). */ -#define INSTR_PROF_RAW_VERSION 5 -/* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 6 -/* Coverage mapping format version (start from 0). */ -#define INSTR_PROF_COVMAP_VERSION 3 - -/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the - * version for other variants of profile. We set the lowest bit of the upper 8 - * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton - * generated profile, and 0 if this is a Clang FE generated profile. - * 1 in bit 57 indicates there are context-sensitive records in the profile. - */ -#define VARIANT_MASKS_ALL 0xff00000000000000ULL -#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) -#define VARIANT_MASK_IR_PROF (0x1ULL << 56) -#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) -#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version -#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime - -/* The variable that holds the name of the profile data - * specified via command line. */ -#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename - -/* section name strings common to all targets other - than WIN32 */ -#define INSTR_PROF_DATA_COMMON __llvm_prf_data -#define INSTR_PROF_NAME_COMMON __llvm_prf_names -#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts -#define INSTR_PROF_VALS_COMMON __llvm_prf_vals -#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds -#define INSTR_PROF_COVMAP_COMMON __llvm_covmap -#define INSTR_PROF_COVFUN_COMMON __llvm_covfun -#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile -/* Windows section names. Because these section names contain dollar characters, - * they must be quoted. - */ -#define INSTR_PROF_DATA_COFF ".lprfd$M" -#define INSTR_PROF_NAME_COFF ".lprfn$M" -#define INSTR_PROF_CNTS_COFF ".lprfc$M" -#define INSTR_PROF_VALS_COFF ".lprfv$M" -#define INSTR_PROF_VNODES_COFF ".lprfnd$M" -#define INSTR_PROF_COVMAP_COFF ".lcovmap$M" -#define INSTR_PROF_COVFUN_COFF ".lcovfun$M" -#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" - -#ifdef _WIN32 -/* Runtime section names and name strings. */ -#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF -#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF -#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF -/* Array of pointers. Each pointer points to a list - * of value nodes associated with one value site. - */ -#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF -/* Value profile nodes section. */ -#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF -#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF -#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF -#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF -#else -/* Runtime section names and name strings. */ -#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) -#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) -#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) -/* Array of pointers. Each pointer points to a list - * of value nodes associated with one value site. - */ -#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) -/* Value profile nodes section. */ -#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) -#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) -#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) -/* Order file instrumentation. */ -#define INSTR_PROF_ORDERFILE_SECT_NAME \ - INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) -#endif - -#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer -#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) -#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx -#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) - -/* Macros to define start/stop section symbol for a given - * section on Linux. For instance - * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will - * expand to __start___llvm_prof_data - */ -#define INSTR_PROF_SECT_START(Sect) \ - INSTR_PROF_CONCAT(__start_,Sect) -#define INSTR_PROF_SECT_STOP(Sect) \ - INSTR_PROF_CONCAT(__stop_,Sect) - -/* Value Profiling API linkage name. */ -#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target -#define INSTR_PROF_VALUE_PROF_FUNC_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) -/* FIXME: This is to be removed after switching to the new memop value - * profiling. */ -#define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range -#define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC) -#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop -#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC) - -/* InstrProfile per-function control data alignment. */ -#define INSTR_PROF_DATA_ALIGNMENT 8 - -/* The data structure that represents a tracked value by the - * value profiler. - */ -typedef struct InstrProfValueData { - /* Profiled value. */ - uint64_t Value; - /* Number of times the value appears in the training run. */ - uint64_t Count; -} InstrProfValueData; - -#endif /* INSTR_PROF_DATA_INC */ - -#ifndef INSTR_ORDER_FILE_INC -/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */ -#define INSTR_ORDER_FILE_BUFFER_SIZE 131072 -#define INSTR_ORDER_FILE_BUFFER_BITS 17 -#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff -#endif /* INSTR_ORDER_FILE_INC */ -#else -#undef INSTR_PROF_DATA_DEFINED -#endif - -#undef COVMAP_V2_OR_V3 - -#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API - -#ifdef __cplusplus -#define INSTR_PROF_INLINE inline -#else -#define INSTR_PROF_INLINE -#endif - -/* The value range buckets (22 buckets) for the memop size value profiling looks - * like: - * - * [0, 0] - * [1, 1] - * [2, 2] - * [3, 3] - * [4, 4] - * [5, 5] - * [6, 6] - * [7, 7] - * [8, 8] - * [9, 15] - * [16, 16] - * [17, 31] - * [32, 32] - * [33, 63] - * [64, 64] - * [65, 127] - * [128, 128] - * [129, 255] - * [256, 256] - * [257, 511] - * [512, 512] - * [513, UINT64_MAX] - * - * Each range has a 'representative value' which is the lower end value of the - * range and used to store in the runtime profile data records and the VP - * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127]. - */ - -/* - * Clz and Popcount. This code was copied from - * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and - * llvm/include/llvm/Support/MathExtras.h. - */ -#if defined(_MSC_VER) && !defined(__clang__) - -#include -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -int InstProfClzll(unsigned long long X) { - unsigned long LeadZeroIdx = 0; -#if !defined(_M_ARM64) && !defined(_M_X64) - // Scan the high 32 bits. - if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32))) - return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset - // from the MSB. - // Scan the low 32 bits. - if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X))) - return (int)(63 - LeadZeroIdx); -#else - if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx; -#endif - return 64; -} -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -int InstProfPopcountll(unsigned long long X) { - // This code originates from https://reviews.llvm.org/rG30626254510f. - unsigned long long v = X; - v = v - ((v >> 1) & 0x5555555555555555ULL); - v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); - v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; - return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56); -} - -#else - -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); } -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); } - -#endif /* defined(_MSC_VER) && !defined(__clang__) */ - -/* Map an (observed) memop size value to the representative value of its range. - * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */ -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t -InstrProfGetRangeRepValue(uint64_t Value) { - if (Value <= 8) - // The first ranges are individually tracked. Use the value as is. - return Value; - else if (Value >= 513) - // The last range is mapped to its lowest value. - return 513; - else if (InstProfPopcountll(Value) == 1) - // If it's a power of two, use it as is. - return Value; - else - // Otherwise, take to the previous power of two + 1. - return (1 << (64 - InstProfClzll(Value) - 1)) + 1; -} - -/* Return true if the range that an (observed) memop size value belongs to has - * only a single value in the range. For example, 0 -> true, 8 -> true, 10 -> - * false, 64 -> true, 100 -> false, 513 -> false. */ -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned -InstrProfIsSingleValRange(uint64_t Value) { - if (Value <= 8) - // The first ranges are individually tracked. - return 1; - else if (InstProfPopcountll(Value) == 1) - // If it's a power of two, there's only one value. - return 1; - else - // Otherwise, there's more than one value in the range. - return 0; -} - -#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */ Index: compiler-rt/lib/profile/InstrProfiling.h =================================================================== --- compiler-rt/lib/profile/InstrProfiling.h +++ compiler-rt/lib/profile/InstrProfiling.h @@ -13,29 +13,29 @@ #include #define INSTR_PROF_VISIBILITY COMPILER_RT_VISIBILITY -#include "profile/InstrProfData.inc" +#include "../../llvm/include/llvm/ProfileData/InstrProfData.inc" enum ValueKind { #define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value, -#include "profile/InstrProfData.inc" +#include "../../llvm/include/llvm/ProfileData/InstrProfData.inc" }; typedef void *IntPtrT; typedef struct COMPILER_RT_ALIGNAS(INSTR_PROF_DATA_ALIGNMENT) __llvm_profile_data { #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) Type Name; -#include "profile/InstrProfData.inc" +#include "../../llvm/include/llvm/ProfileData/InstrProfData.inc" } __llvm_profile_data; typedef struct __llvm_profile_header { #define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) Type Name; -#include "profile/InstrProfData.inc" +#include "../../llvm/include/llvm/ProfileData/InstrProfData.inc" } __llvm_profile_header; typedef struct ValueProfNode * PtrToNodeT; typedef struct ValueProfNode { #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) Type Name; -#include "profile/InstrProfData.inc" +#include "../../llvm/include/llvm/ProfileData/InstrProfData.inc" } ValueProfNode; /*! @@ -129,7 +129,7 @@ */ void INSTR_PROF_VALUE_PROF_FUNC( #define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) ArgType ArgName -#include "profile/InstrProfData.inc" +#include "../../llvm/include/llvm/ProfileData/InstrProfData.inc" ); void __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data, Index: compiler-rt/lib/profile/InstrProfiling.c =================================================================== --- compiler-rt/lib/profile/InstrProfiling.c +++ compiler-rt/lib/profile/InstrProfiling.c @@ -18,7 +18,7 @@ #include "InstrProfilingInternal.h" #define INSTR_PROF_VALUE_PROF_DATA -#include "profile/InstrProfData.inc" +#include "../../llvm/include/llvm/ProfileData/InstrProfData.inc" COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_magic(void) { return sizeof(void *) == sizeof(uint64_t) ? (INSTR_PROF_RAW_MAGIC_64) Index: compiler-rt/lib/profile/InstrProfilingMerge.c =================================================================== --- compiler-rt/lib/profile/InstrProfilingMerge.c +++ compiler-rt/lib/profile/InstrProfilingMerge.c @@ -14,7 +14,7 @@ #include "InstrProfilingUtil.h" #define INSTR_PROF_VALUE_PROF_DATA -#include "profile/InstrProfData.inc" +#include "../../llvm/include/llvm/ProfileData/InstrProfData.inc" COMPILER_RT_VISIBILITY void (*VPMergeHook)(ValueProfData *, __llvm_profile_data *); Index: compiler-rt/lib/profile/InstrProfilingMergeFile.c =================================================================== --- compiler-rt/lib/profile/InstrProfilingMergeFile.c +++ compiler-rt/lib/profile/InstrProfilingMergeFile.c @@ -16,7 +16,7 @@ #include "InstrProfilingUtil.h" #define INSTR_PROF_VALUE_PROF_DATA -#include "profile/InstrProfData.inc" +#include "../../llvm/include/llvm/ProfileData/InstrProfData.inc" /* Merge value profile data pointed to by SrcValueProfData into * in-memory profile counters pointed by to DstData. */ Index: compiler-rt/lib/profile/InstrProfilingValue.c =================================================================== --- compiler-rt/lib/profile/InstrProfilingValue.c +++ compiler-rt/lib/profile/InstrProfilingValue.c @@ -19,7 +19,7 @@ #define INSTR_PROF_VALUE_PROF_DATA #define INSTR_PROF_COMMON_API_IMPL #define INSTR_PROF_VALUE_PROF_MEMOP_API -#include "profile/InstrProfData.inc" +#include "../../llvm/include/llvm/ProfileData/InstrProfData.inc" static int hasStaticCounters = 1; static int OutOfNodesWarnings = 0; Index: compiler-rt/lib/profile/InstrProfilingWriter.c =================================================================== --- compiler-rt/lib/profile/InstrProfilingWriter.c +++ compiler-rt/lib/profile/InstrProfilingWriter.c @@ -20,7 +20,7 @@ #include "InstrProfilingPort.h" #define INSTR_PROF_VALUE_PROF_DATA -#include "profile/InstrProfData.inc" +#include "../../llvm/include/llvm/ProfileData/InstrProfData.inc" COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL; static ProfBufferIO TheBufferIO; @@ -281,7 +281,7 @@ /* Initialize header structure. */ #define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init; -#include "profile/InstrProfData.inc" +#include "../../llvm/include/llvm/ProfileData/InstrProfData.inc" /* Write the data. */ ProfDataIOVec IOVec[] = { Index: llvm/include/llvm/ProfileData/InstrProfData.inc =================================================================== --- llvm/include/llvm/ProfileData/InstrProfData.inc +++ llvm/include/llvm/ProfileData/InstrProfData.inc @@ -6,18 +6,12 @@ |* \*===----------------------------------------------------------------------===*/ /* - * This is the master file that defines all the data structure, signature, + * This file defines all the data structure, signature, * constant literals that are shared across profiling runtime library, * compiler (instrumentation), and host tools (reader/writer). The entities * defined in this file affect the profile runtime ABI, the raw profile format, * or both. * - * The file has two identical copies. The master copy lives in LLVM and - * the other one sits in compiler-rt/lib/profile directory. To make changes - * in this file, first modify the master copy and copy it over to compiler-rt. - * Testing of any change in this file can start only after the two copies are - * synced up. - * * The first part of the file includes macros that defines types, names, and * initializers for the member fields of the core data structures. The field * declarations for one structure is enabled by defining the field activation