Index: include/llvm/ProfileData/InstrProfData.inc =================================================================== --- include/llvm/ProfileData/InstrProfData.inc +++ include/llvm/ProfileData/InstrProfData.inc @@ -97,6 +97,7 @@ #endif INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) +INSTR_PROF_RAW_HEADER(uint64_t, CounterSectionAlignment, CounterSectionAlignment) INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) @@ -711,7 +712,7 @@ (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 /* Raw profile format version (start from 1). */ -#define INSTR_PROF_RAW_VERSION 3 +#define INSTR_PROF_RAW_VERSION 4 /* Indexed profile format version (start from 1). */ #define INSTR_PROF_INDEX_VERSION 4 /* Coverage mapping format vresion (start from 0). */ Index: include/llvm/ProfileData/InstrProfReader.h =================================================================== --- include/llvm/ProfileData/InstrProfReader.h +++ include/llvm/ProfileData/InstrProfReader.h @@ -168,6 +168,7 @@ const RawInstrProf::ProfileData *Data; const RawInstrProf::ProfileData *DataEnd; const uint64_t *CountersStart; + const uint64_t *CountersEnd; const char *NamesStart; uint64_t NamesSize; const uint8_t *ValueDataStart; Index: include/llvm/Transforms/InstrProfiling.h =================================================================== --- include/llvm/Transforms/InstrProfiling.h +++ include/llvm/Transforms/InstrProfiling.h @@ -43,6 +43,7 @@ memset(NumValueSites, 0, sizeof(uint32_t) * (IPVK_Last + 1)); } }; + uint64_t TotalNumCounters; DenseMap ProfileDataMap; std::vector UsedVars; std::vector ReferencedNames; @@ -84,6 +85,9 @@ /// Emit the section with compressed function names. void emitNameData(); + /// Emit padding to attempt to page-align the counters section. + void emitCounterPadding(); + /// Emit runtime registration functions for each profile data variable. void emitRegistration(); Index: lib/ProfileData/InstrProfReader.cpp =================================================================== --- lib/ProfileData/InstrProfReader.cpp +++ lib/ProfileData/InstrProfReader.cpp @@ -14,6 +14,7 @@ #include "llvm/ProfileData/InstrProfReader.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/Support/MathExtras.h" #include using namespace llvm; @@ -317,6 +318,11 @@ CountersDelta = swap(Header.CountersDelta); NamesDelta = swap(Header.NamesDelta); + + auto CounterSectionAlignment = swap(Header.CounterSectionAlignment); + if (!isPowerOf2_64(CounterSectionAlignment)) + return error(instrprof_error::malformed); + auto DataSize = swap(Header.DataSize); auto CountersSize = swap(Header.CountersSize); NamesSize = swap(Header.NamesSize); @@ -325,10 +331,13 @@ auto DataSizeInBytes = DataSize * sizeof(RawInstrProf::ProfileData); auto PaddingSize = getNumPaddingBytes(NamesSize); + auto CountersSizeInBytes = CountersSize * sizeof(uint64_t); ptrdiff_t DataOffset = sizeof(RawInstrProf::Header); - ptrdiff_t CountersOffset = DataOffset + DataSizeInBytes; - ptrdiff_t NamesOffset = CountersOffset + sizeof(uint64_t) * CountersSize; + ptrdiff_t CountersOffset = + alignTo(DataOffset + DataSizeInBytes, CounterSectionAlignment); + ptrdiff_t NamesOffset = + alignTo(CountersOffset + CountersSizeInBytes, CounterSectionAlignment); ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize; size_t ProfileSize = ValueDataOffset + ValueDataSize; @@ -336,10 +345,15 @@ if (Start + ProfileSize > DataBuffer->getBufferEnd()) return error(instrprof_error::bad_header); + // The writer emits each profile at an aligned offset. + if ((Start - DataBuffer->getBufferStart()) % CounterSectionAlignment) + return error(instrprof_error::malformed); + Data = reinterpret_cast *>( Start + DataOffset); DataEnd = Data + DataSize; CountersStart = reinterpret_cast(Start + CountersOffset); + CountersEnd = CountersStart + CountersSize; NamesStart = Start + NamesOffset; ValueDataStart = reinterpret_cast(Start + ValueDataOffset); ProfileEnd = Start + ProfileSize; @@ -372,11 +386,10 @@ return error(instrprof_error::malformed); auto RawCounts = makeArrayRef(getCounter(CounterPtr), NumCounters); - auto *NamesStartAsCounter = reinterpret_cast(NamesStart); // Check bounds. if (RawCounts.data() < CountersStart || - RawCounts.data() + RawCounts.size() > NamesStartAsCounter) + RawCounts.data() + RawCounts.size() > CountersEnd) return error(instrprof_error::malformed); if (ShouldSwapBytes) { Index: lib/Transforms/Instrumentation/InstrProfiling.cpp =================================================================== --- lib/Transforms/Instrumentation/InstrProfiling.cpp +++ lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -19,6 +19,7 @@ #include "llvm/IR/Module.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/Transforms/InstrProfiling.h" +#include "llvm/Support/Process.h" #include "llvm/Transforms/Utils/ModuleUtils.h" using namespace llvm; @@ -98,6 +99,7 @@ this->M = &M; NamesVar = nullptr; NamesSize = 0; + TotalNumCounters = 0; ProfileDataMap.clear(); UsedVars.clear(); @@ -142,6 +144,7 @@ return false; emitNameData(); + emitCounterPadding(); emitRegistration(); emitRuntimeHook(); emitUses(); @@ -305,6 +308,7 @@ uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); LLVMContext &Ctx = M->getContext(); ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters); + TotalNumCounters += NumCounters; // Create the counters variable. auto *CounterPtr = @@ -383,6 +387,21 @@ UsedVars.push_back(NamesVar); } +void InstrProfiling::emitCounterPadding() { + auto PageSize = llvm::sys::Process::getPageSize(); + auto NumPadCounters = + alignTo(TotalNumCounters, PageSize / sizeof(uint64_t)) - TotalNumCounters; + LLVMContext &Ctx = M->getContext(); + ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumPadCounters); + auto *PadCountersPtr = + new GlobalVariable(*M, CounterTy, false, GlobalValue::PrivateLinkage, + Constant::getNullValue(CounterTy), + getInstrProfCountersVarPrefix() + "pad"); + PadCountersPtr->setAlignment(8); + PadCountersPtr->setSection(getCountersSection()); + UsedVars.push_back(PadCountersPtr); +} + void InstrProfiling::emitRegistration() { // Don't do this for Darwin. compiler-rt uses linker magic. if (Triple(M->getTargetTriple()).isOSDarwin()) Index: test/tools/llvm-profdata/raw-32-bits-be.test =================================================================== --- test/tools/llvm-profdata/raw-32-bits-be.test +++ test/tools/llvm-profdata/raw-32-bits-be.test @@ -1,5 +1,6 @@ RUN: printf '\377lprofR\201' > %t -RUN: printf '\0\0\0\0\0\0\0\3' >> %t +RUN: printf '\0\0\0\0\0\0\0\4' >> %t +RUN: printf '\0\0\0\0\0\0\0\1' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t RUN: printf '\0\0\0\0\0\0\0\3' >> %t RUN: printf '\0\0\0\0\0\0\0\20' >> %t Index: test/tools/llvm-profdata/raw-32-bits-le.test =================================================================== --- test/tools/llvm-profdata/raw-32-bits-le.test +++ test/tools/llvm-profdata/raw-32-bits-le.test @@ -1,5 +1,6 @@ RUN: printf '\201Rforpl\377' > %t -RUN: printf '\3\0\0\0\0\0\0\0' >> %t +RUN: printf '\4\0\0\0\0\0\0\0' >> %t +RUN: printf '\1\0\0\0\0\0\0\0' >> %t RUN: printf '\2\0\0\0\0\0\0\0' >> %t RUN: printf '\3\0\0\0\0\0\0\0' >> %t RUN: printf '\20\0\0\0\0\0\0\0' >> %t Index: test/tools/llvm-profdata/raw-64-bits-be.test =================================================================== --- test/tools/llvm-profdata/raw-64-bits-be.test +++ test/tools/llvm-profdata/raw-64-bits-be.test @@ -1,5 +1,6 @@ RUN: printf '\377lprofr\201' > %t -RUN: printf '\0\0\0\0\0\0\0\3' >> %t +RUN: printf '\0\0\0\0\0\0\0\4' >> %t +RUN: printf '\0\0\0\0\0\0\0\1' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t RUN: printf '\0\0\0\0\0\0\0\3' >> %t RUN: printf '\0\0\0\0\0\0\0\20' >> %t Index: test/tools/llvm-profdata/raw-64-bits-le.test =================================================================== --- test/tools/llvm-profdata/raw-64-bits-le.test +++ test/tools/llvm-profdata/raw-64-bits-le.test @@ -1,5 +1,6 @@ RUN: printf '\201rforpl\377' > %t -RUN: printf '\3\0\0\0\0\0\0\0' >> %t +RUN: printf '\4\0\0\0\0\0\0\0' >> %t +RUN: printf '\1\0\0\0\0\0\0\0' >> %t RUN: printf '\2\0\0\0\0\0\0\0' >> %t RUN: printf '\3\0\0\0\0\0\0\0' >> %t RUN: printf '\20\0\0\0\0\0\0\0' >> %t Index: test/tools/llvm-profdata/raw-two-profiles.test =================================================================== --- test/tools/llvm-profdata/raw-two-profiles.test +++ test/tools/llvm-profdata/raw-two-profiles.test @@ -1,5 +1,6 @@ RUN: printf '\201rforpl\377' > %t-foo.profraw -RUN: printf '\3\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\4\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t-foo.profraw @@ -20,7 +21,8 @@ RUN: printf '\3\0foo\0\0\0' >> %t-foo.profraw RUN: printf '\201rforpl\377' > %t-bar.profraw -RUN: printf '\3\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\4\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\1\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\2\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t-bar.profraw