[profile] Support dynamically allocated counters

[phosek]

Summary

Rather than allocating counter statically at compile time, it's the
responsiblity of the profile runtime to allocate them at run time
by mmapping the profile and updating all the pointers. This allows
writing counters on-the-fly rather than at exit.

See discussion on "D68351: [profile] Add a mode to continuously sync
counter updates to a file" for more details.

[phosek]

Abandoned in favor of D69740.

compiler-rt/lib/profile/InstrProfData.inc

 #define INSTR_PROF_DATA_DEFINED
 #endif
 INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
                 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
                 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName()))))
 INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
                 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
                 Inc->getHash()->getZExtValue()))
-INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \
+INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \
                 ConstantExpr::getBitCast(CounterPtr, \
                 llvm::Type::getInt64PtrTy(Ctx)))
 /* This is used to map function pointers for the indirect call targets to
  * function name hashes during the conversion from raw to merged profile
  * data.
  */
 INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \
                 FunctionAddr)

compiler-rt/lib/profile/InstrProfilingFile.c

typedef struct lprofFilename {
    * can only appear once at the end of the name pattern. */
   unsigned MergePoolSize;
   ProfileNameSpecifier PNS;
 } lprofFilename;
 
 static lprofFilename lprofCurFilename = {0, 0, 0, {0},        {0},
                                          0, 0, 0, PNS_unknown};
 
+static int RuntimeAllocatedCounters = 0;
 static int ProfileMergeRequested = 0;
 static int isProfileMergeRequested() { return ProfileMergeRequested; }
 static void setProfileMergeRequested(int EnableMerge) {
   ProfileMergeRequested = EnableMerge;
 }
 
 static FILE *ProfileFile = NULL;
 static FILE *getProfileFile() { return ProfileFile; }
 static void setProfileFile(FILE *File) { ProfileFile = File; }
 
 COMPILER_RT_VISIBILITY void __llvm_profile_set_file_object(FILE *File,
                                                            int EnableMerge) {
+  if (RuntimeAllocatedCounters)
+    return;
   setProfileFile(File);
   setProfileMergeRequested(EnableMerge);
 }
 
 static int getCurFilenameLength();
 static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf);
 static unsigned doMerging() {
   return lprofCurFilename.MergePoolSize || isProfileMergeRequested();
static void setupIOBuffer() {
   const char *BufferSzStr = 0;
   BufferSzStr = getenv("LLVM_VP_BUFFER_SIZE");
   if (BufferSzStr && BufferSzStr[0]) {
     VPBufferSize = atoi(BufferSzStr);
     DynamicBufferIOBuffer = (uint8_t *)calloc(VPBufferSize, 1);
   }
 }
 
-/* Read profile data in \c ProfileFile and merge with in-memory
-   profile counters. Returns -1 if there is fatal error, otheriwse
-   0 is returned. Returning 0 does not mean merge is actually
-   performed. If merge is actually done, *MergeDone is set to 1.
-*/
-static int doProfileMerging(FILE *ProfileFile, int *MergeDone) {
-  uint64_t ProfileFileSize;
-  char *ProfileBuffer;
-
+/// Get the size of the profile file. If there are any errors, print the
+/// message under the assumption that the profile is being read for merging
+/// purposes, and return -1. Otherwise return the file size in the inout param
+/// \p ProfileFileSize.
+static int getProfileFileSizeForMerging(FILE *ProfileFile,
+                                        uint64_t *ProfileFileSize) {
   if (fseek(ProfileFile, 0L, SEEK_END) == -1) {
     PROF_ERR("Unable to merge profile data, unable to get size: %s\n",
              strerror(errno));
     return -1;
   }
-  ProfileFileSize = ftell(ProfileFile);
+  *ProfileFileSize = ftell(ProfileFile);
 
-  /* Restore file offset.  */
   if (fseek(ProfileFile, 0L, SEEK_SET) == -1) {
     PROF_ERR("Unable to merge profile data, unable to rewind: %s\n",
              strerror(errno));
     return -1;
   }
 
-  /* Nothing to merge.  */
-  if (ProfileFileSize < sizeof(__llvm_profile_header)) {
-    if (ProfileFileSize)
-      PROF_WARN("Unable to merge profile data: %s\n",
-                "source profile file is too small.");
-    return 0;
-  }
+  if (*ProfileFileSize > 0 &&
+      *ProfileFileSize < sizeof(__llvm_profile_header)) {
+    return -1;
+  }
+
 
-  ProfileBuffer = mmap(NULL, ProfileFileSize, PROT_READ, MAP_SHARED | MAP_FILE,
-                       fileno(ProfileFile), 0);
-  if (ProfileBuffer == MAP_FAILED) {
+/// mmap() \p ProfileFile for profile merging purposes, assuming that an
+/// exclusive lock is held on the file and that \p ProfileFileSize is the
+/// length of the file. Return the mmap'd buffer in the inout variable
+/// \p ProfileBuffer. Returns -1 on failure. On success, the caller is
+/// responsible for unmapping the mmap'd buffer in \p ProfileBuffer.
+static int mmapProfileForMerging(FILE *ProfileFile, uint64_t ProfileFileSize,
+                                 char **ProfileBuffer) {
+  *ProfileBuffer = mmap(NULL, ProfileFileSize, PROT_READ | PROT_WRITE,
+                        MAP_SHARED | MAP_FILE, fileno(ProfileFile), 0);
+  if (*ProfileBuffer == MAP_FAILED) {
     PROF_ERR("Unable to merge profile data, mmap failed: %s\n",
              strerror(errno));
     return -1;
   }
 
-  if (__llvm_profile_check_compatibility(ProfileBuffer, ProfileFileSize)) {
+  if (__llvm_profile_check_compatibility(*ProfileBuffer, ProfileFileSize)) {
     (void)munmap(ProfileBuffer, ProfileFileSize);
     PROF_WARN("Unable to merge profile data: %s\n",
               "source profile file is not compatible.");
-    return 0;
-  }
+    return -1;
+  }
+
 
+/* Read profile data in \c ProfileFile and merge with in-memory
+   profile counters. Returns -1 if there is fatal error, otheriwse
+   0 is returned. Returning 0 does not mean merge is actually
+   performed. If merge is actually done, *MergeDone is set to 1.
+*/
+static int doProfileMerging(FILE *ProfileFile, int *MergeDone) {
+  uint64_t ProfileFileSize;
+  char *ProfileBuffer;
+
+  /* Get the size of the profile on disk. */
+  if (getProfileFileSizeForMerging(ProfileFile, &ProfileFileSize) == -1)
+    return -1;
+
+  /* Nothing to merge.  */
+  if (!ProfileFileSize)
+    return 0;
+
+  /* mmap() the profile and check that it is compatible with the data in
+   * the current image. */
+  if (mmapProfileForMerging(ProfileFile, ProfileFileSize, &ProfileBuffer) == -1)
+    return -1;
+
   /* Now start merging */
   __llvm_profile_merge_from_buffer(ProfileBuffer, ProfileFileSize);
 
   // Truncate the file in case merging of value profile did not happend to
   // prevent from leaving garbage data at the end of the profile file.
   COMPILER_RT_FTRUNCATE(ProfileFile, __llvm_profile_get_size_for_buffer());
 
   (void)munmap(ProfileBuffer, ProfileFileSize);
static void truncateCurrentFile(void) {
 
   /* Truncate the file.  Later we'll reopen and append. */
   File = fopen(Filename, "w");
   if (!File)
     return;
   fclose(File);
 }
 
+static int createProfileWithAllocatedCounters(FILE *OutputFile) {
+  /* Get the sizes of various profile data sections. Taken from
+   * __llvm_profile_get_size_for_buffer(). */
+  const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
+  const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
+  const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
+
+  setupIOBuffer();
+  ProfDataWriter fileWriter;
+  initFileWriter(&fileWriter, OutputFile);
+  if (lprofWriteData(&fileWriter, NULL, 0)) {
+    PROF_ERR("Failed to write profile: %s\n", strerror(errno));
+    return -1;
+  }
+  fflush(OutputFile);
+
+  uint64_t FileSize = ftell(OutputFile);
+
+  char *ProfileBuffer = mmap(NULL, FileSize, PROT_READ | PROT_WRITE,
+                             MAP_SHARED | MAP_FILE, fileno(OutputFile), 0);
+  if (ProfileBuffer == MAP_FAILED) {
+    PROF_ERR("Unable to mmap profile: %s\n", strerror(errno));
+    return -1;
+  }
+
+  /* Update the profile fields based on the current mapping. */
+  __llvm_profile_header *Header = (__llvm_profile_header *)ProfileBuffer;
+  __llvm_profile_data *Data = (__llvm_profile_data *)(
+      ProfileBuffer + sizeof(__llvm_profile_header));
+  uint64_t *Counters = (uint64_t *)(ProfileBuffer +
+      sizeof(__llvm_profile_header) + sizeof(__llvm_profile_data) * DataSize);
+  Header->CountersDelta = (uint64_t)Counters;
+  for (const __llvm_profile_data *DI = DataBegin; DI != DataEnd; DI++) {
+    ((__llvm_profile_data *)DI)->CounterPtr = Counters;
+    Data->CounterPtr = Counters;
+    Data++;
+    Counters += DI->NumCounters;
+  }
+
+  return 0;
+}
+
+static void allocateCounters(void) {
+  /* Get the sizes of various profile data sections. Taken from
+   * __llvm_profile_get_size_for_buffer(). */
+  const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
+  const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
+  uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
+
+  /* Open the raw profile in append mode. */
+  int Length = getCurFilenameLength();
+  char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
+  const char *Filename = getCurFilename(FilenameBuf, 0);
+  if (!Filename)
+    return;
+
+  FILE *File = NULL;
+  char *ProfileBuffer = NULL;
+
+  if (!doMerging()) {
+    File = lprofOpenFile(Filename);
+    if (!File)
+      return;
+
+    createProfileWithAllocatedCounters(File);
+  } else {
+    File = lprofOpenFileEx(Filename);
+    if (!File)
+      return;
+
+    uint64_t ProfileFileSize = 0;
+    if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) {
+      lprofUnlockFileHandle(File);
+      return;
+    }
+
+    if (!ProfileFileSize) {
+      createProfileWithAllocatedCounters(File);
+    } else {
+      /* The merged profile has a non-zero length. Check that it is compatible
+       * with the data in this process. */
+      if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1) {
+        lprofUnlockFileHandle(File);
+        return;
+      }
+
+      /* Update the profile fields based on the current mapping. */
+      uint64_t *Counters = (uint64_t *)(ProfileBuffer +
+          sizeof(__llvm_profile_header) + sizeof(__llvm_profile_data) * DataSize);
+      for (const __llvm_profile_data *DI = DataBegin; DI != DataEnd; DI++) {
+        ((__llvm_profile_data *)DI)->CounterPtr = Counters;
+        Counters += DI->NumCounters;
+      }
+    }
+
+    lprofUnlockFileHandle(File);
+  }
+}
+
 static const char *DefaultProfileName = "default.profraw";
 static void resetFilenameToDefault(void) {
   if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) {
     free((void *)lprofCurFilename.FilenamePat);
   }
   memset(&lprofCurFilename, 0, sizeof(lprofCurFilename));
   lprofCurFilename.FilenamePat = DefaultProfileName;
   lprofCurFilename.PNS = PNS_default;

 
 static void parseAndSetFilename(const char *FilenamePat,
                                 ProfileNameSpecifier PNS,
                                 unsigned CopyFilenamePat) {
 
   const char *OldFilenamePat = lprofCurFilename.FilenamePat;
   ProfileNameSpecifier OldPNS = lprofCurFilename.PNS;
 
+  /* The old profile name specifier takes precedence over the old one. */
   if (PNS < OldPNS)
     return;
 
   if (!FilenamePat)
     FilenamePat = DefaultProfileName;
 
   if (OldFilenamePat && !strcmp(OldFilenamePat, FilenamePat)) {
     lprofCurFilename.PNS = PNS;
static void parseAndSetFilename(const char *FilenamePat,
   } else {
     if (getenv("LLVM_PROFILE_VERBOSE"))
       PROF_NOTE("Override old profile path \"%s\" via %s to \"%s\" via %s.\n",
                 OldFilenamePat, getPNSStr(OldPNS), lprofCurFilename.FilenamePat,
                 getPNSStr(PNS));
   }
 
   truncateCurrentFile();
+
+  if (RuntimeAllocatedCounters)
+    allocateCounters();
 }
 
 /* Return buffer length that is required to store the current profile
  * filename with PID and hostname substitutions. */
 /* The length to hold uint64_t followed by 2 digit pool id including '_' */
 #define SIGLEN 24
 static int getCurFilenameLength() {
   int Len;

  * environment variable can override this default value. */
 COMPILER_RT_VISIBILITY
 void __llvm_profile_initialize_file(void) {
   const char *EnvFilenamePat;
   const char *SelectedPat = NULL;
   ProfileNameSpecifier PNS = PNS_unknown;
   int hasCommandLineOverrider = (INSTR_PROF_PROFILE_NAME_VAR[0] != 0);
 
+  // TODO: This is potentially error prone, maybe we should extend the
+  // header and include a field that indicates whether to allocate counters.
+  if (__llvm_profile_begin_data() != __llvm_profile_end_data() &&
+      __llvm_profile_begin_counters() == __llvm_profile_end_counters())
+    RuntimeAllocatedCounters = 1;
+
   EnvFilenamePat = getFilenamePatFromEnv();
   if (EnvFilenamePat) {
     /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid
        at the  moment when __llvm_profile_write_file() gets executed. */
     parseAndSetFilename(EnvFilenamePat, PNS_environment, 1);
     return;
   } else if (hasCommandLineOverrider) {
     SelectedPat = INSTR_PROF_PROFILE_NAME_VAR;
     PNS = PNS_command_line;
   } else {
     SelectedPat = NULL;
     PNS = PNS_default;
   }
 
   parseAndSetFilename(SelectedPat, PNS, 0);
 }
 
 /* This API is directly called by the user application code. It has the
  * highest precedence compared with LLVM_PROFILE_FILE environment variable
  * and command line option -fprofile-instr-generate=<profile_name>.
  */
 COMPILER_RT_VISIBILITY
 void __llvm_profile_set_filename(const char *FilenamePat) {
+  if (RuntimeAllocatedCounters)
+    return;
   parseAndSetFilename(FilenamePat, PNS_runtime_api, 1);
 }
 
 /* The public API for writing profile data into the file with name
  * set by previous calls to __llvm_profile_set_filename or
  * __llvm_profile_override_default_filename or
  * __llvm_profile_initialize_file. */
 COMPILER_RT_VISIBILITY
 int __llvm_profile_write_file(void) {
   int rc, Length;
   const char *Filename;
   char *FilenameBuf;
   int PDeathSig = 0;
 
-  if (lprofProfileDumped()) {
+  if (lprofProfileDumped() || RuntimeAllocatedCounters) {
     PROF_NOTE("Profile data not written to file: %s.\n", "already written");
     return 0;
   }
 
   Length = getCurFilenameLength();
   FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
   Filename = getCurFilename(FilenameBuf, 0);
 

 
 COMPILER_RT_VISIBILITY
 int __llvm_profile_register_write_file_atexit(void) {
   static int HasBeenRegistered = 0;
 
   if (HasBeenRegistered)
     return 0;
 
+  if (RuntimeAllocatedCounters)
+    return 0;
+
   lprofSetupValueProfiler();
 
   HasBeenRegistered = 1;
   return atexit(writeFileWithoutReturn);
 }
 
 #endif

compiler-rt/lib/profile/InstrProfilingMerge.c

if (ProfileSize < sizeof(__llvm_profile_header))
     return 1;
 
   /* Check the header first.  */
   if (Header->Magic != __llvm_profile_get_magic() ||
       Header->Version != __llvm_profile_get_version() ||
       Header->DataSize !=
           __llvm_profile_get_data_size(__llvm_profile_begin_data(),
                                        __llvm_profile_end_data()) ||
-      Header->CountersSize != (uint64_t)(__llvm_profile_end_counters() -
-                                         __llvm_profile_begin_counters()) ||
       Header->NamesSize != (uint64_t)(__llvm_profile_end_names() -
                                       __llvm_profile_begin_names()) ||
       Header->ValueKindLast != IPVK_Last)
     return 1;
 
+  if (__llvm_profile_begin_counters() != __llvm_profile_end_counters() &&
+      Header->CountersSize != (uint64_t)(__llvm_profile_end_counters() -
+                                         __llvm_profile_begin_counters()))
+      return 1;
+
   if (ProfileSize < sizeof(__llvm_profile_header) +
                         Header->DataSize * sizeof(__llvm_profile_data) +
                         Header->NamesSize + Header->CountersSize)
     return 1;
 
   for (SrcData = SrcDataStart,
        DstData = (__llvm_profile_data *)__llvm_profile_begin_data();
        SrcData < SrcDataEnd; ++SrcData, ++DstData) {

compiler-rt/lib/profile/InstrProfilingRuntime.cpp

 COMPILER_RT_VISIBILITY int INSTR_PROF_PROFILE_RUNTIME_VAR;
 }
 
 namespace {
 
 class RegisterRuntime {
 public:
   RegisterRuntime() {
-    __llvm_profile_register_write_file_atexit();
     __llvm_profile_initialize_file();
+    __llvm_profile_register_write_file_atexit();
   }
 };
 
-RegisterRuntime Registration;
+RegisterRuntime Registration __attribute__((init_priority(101)));
 
 }

compiler-rt/lib/profile/InstrProfilingWriter.c

 }
 
 COMPILER_RT_VISIBILITY int
 lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
                    const __llvm_profile_data *DataEnd,
                    const uint64_t *CountersBegin, const uint64_t *CountersEnd,
                    VPDataReaderType *VPDataReader, const char *NamesBegin,
                    const char *NamesEnd, int SkipNameDataWrite) {
-
   /* Calculate size of sections. */
   const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
-  const uint64_t CountersSize = CountersEnd - CountersBegin;
+  uint64_t CountersSize = 0;
+  int SkipCounterDataWrite = 0;
+  if (CountersBegin != CountersEnd) {
+    CountersSize = CountersEnd - CountersBegin;
+  } else {
+    for (const __llvm_profile_data *Data = DataBegin; Data != DataEnd; Data++)
+      CountersSize += Data->NumCounters;
+    SkipCounterDataWrite = 1;
+  }
   const uint64_t NamesSize = NamesEnd - NamesBegin;
   const uint64_t Padding = __llvm_profile_get_num_padding_bytes(NamesSize);
 
-  /* Enough zeroes for padding. */
-  const char Zeroes[sizeof(uint64_t)] = {0};
-
   /* Create the header. */
   __llvm_profile_header Header;
 
   if (!DataSize)
     return 0;
 
 /* Initialize header structure.  */
 #define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init;
 #include "InstrProfData.inc"
 
+  /* Enough zeroes for padding. */
+  const char Zeroes[sizeof(uint64_t)] = {0};
+
   /* Write the data. */
   ProfDataIOVec IOVec[] = {
       {&Header, sizeof(__llvm_profile_header), 1},
       {DataBegin, sizeof(__llvm_profile_data), DataSize},
-      {CountersBegin, sizeof(uint64_t), CountersSize},
+      {SkipCounterDataWrite ? NULL : CountersBegin, sizeof(uint64_t), CountersSize},
       {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize},
       {Zeroes, sizeof(uint8_t), Padding}};
   if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec)))
     return -1;
 
+  /* Value profiling is not yet supported when writing counters directly. */
+  if (SkipCounterDataWrite)
+    return 0;
+
   return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd);
 }

compiler-rt/test/profile/instrprof-basic.c

 // RUN: cd %t.dir4
 // RUN: %run %t.dir4/merge4
 // RUN: %run %t.dir4/merge4
 // RUN: %run %t.dir4/merge4
 // RUN: %run %t.dir4/merge4
 // RUN: rm -f %t.dir4/merge4
 // RUN: llvm-profdata merge -o %t.m4.profdata ./
 // RUN: %clang_profuse=%t.m4.profdata -O0 -o - -S -emit-llvm %s | FileCheck %s --check-prefix=COMMON  --check-prefix=PGOMERGE
+//
+// Test runtime allocated counters
+// RUN: %clang_profgen -o %t -O3 -mllvm -counter-static-alloc=false %s
+// RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t
+// RUN: llvm-profdata merge -o %t.profdata %t.profraw
+// RUN: %clang_profuse=%t.profdata -o - -S -emit-llvm %s | FileCheck %s --check-prefix=COMMON --check-prefix=RUNTIME
 
 int begin(int i) {
   // COMMON: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}, !prof ![[PD1:[0-9]+]]
   if (i)
     return 0;
   return 1;
 }
 

llvm/include/llvm/ProfileData/InstrProfData.inc

 #define INSTR_PROF_DATA_DEFINED
 #endif
 INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
                 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
                 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName()))))
 INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
                 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
                 Inc->getHash()->getZExtValue()))
-INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \
-                ConstantExpr::getBitCast(CounterPtr, \
-                llvm::Type::getInt64PtrTy(Ctx)))
+INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \
+                CounterPtr)
 /* This is used to map function pointers for the indirect call targets to
  * function name hashes during the conversion from raw to merged profile
  * data.
  */
 INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \
                 FunctionAddr)
 INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \
                 ValuesPtrExpr)

llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h

 
 private:
   InstrProfOptions Options;
   Module *M;
   Triple TT;
   std::function<const TargetLibraryInfo &(Function &F)> GetTLI;
   struct PerFunctionProfileData {
     uint32_t NumValueSites[IPVK_Last + 1];
-    GlobalVariable *RegionCounters = nullptr;
+    Value *RegionCounters = nullptr;
     GlobalVariable *DataVar = nullptr;
 
     PerFunctionProfileData() {
       memset(NumValueSites, 0, sizeof(uint32_t) * (IPVK_Last + 1));
     }
   };
   DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap;
   std::vector<GlobalValue *> UsedVars;
private:
 
   /// Force emitting of name vars for unused functions.
   void lowerCoverageData(GlobalVariable *CoverageNamesVar);
 
   /// Get the region counters for an increment, creating them if necessary.
   ///
   /// If the counter array doesn't yet exist, the profile data variables
   /// referring to them will also be created.
-  GlobalVariable *getOrCreateRegionCounters(InstrProfIncrementInst *Inc);
+  Value *getOrCreateRegionCounters(InstrProfIncrementInst *Inc);
 
   /// Emit the section with compressed function names.
   void emitNameData();
 
   /// Emit value nodes section for value profiling.
   void emitVNodes();
 
   /// Emit runtime registration functions for each profile data variable.

llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp

cl::opt<bool> DoNameCompression("enable-name-compression",
                                 cl::desc("Enable name string compression"),
                                 cl::init(true));
 
 cl::opt<bool> DoHashBasedCounterSplit(
     "hash-based-counter-split",
     cl::desc("Rename counter variable of a comdat function based on cfg hash"),
     cl::init(true));
 
+cl::opt<bool> CounterStaticAlloc(
+    "counter-static-alloc",
+    cl::desc("Do static counter allocation."),
+    cl::init(true));
+
 cl::opt<bool> ValueProfileStaticAlloc(
     "vp-static-alloc",
     cl::desc("Do static counter allocation for value profiler"),
     cl::init(true));
 
 cl::opt<double> NumCountersPerValueSite(
     "vp-counters-per-site",
     cl::desc("The average number of profile counters allocated "
void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
   }
   if (auto AK = TLI->getExtAttrForI32Param(false))
     Call->addParamAttr(2, AK);
   Ind->replaceAllUsesWith(Call);
   Ind->eraseFromParent();
 }
 
 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
-  GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
+  Value *Counters = getOrCreateRegionCounters(Inc);
 
   IRBuilder<> Builder(Inc);
+  LLVMContext &Ctx = M->getContext();
   uint64_t Index = Inc->getIndex()->getZExtValue();
-  Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters->getValueType(),
-                                                   Counters, 0, Index);
+  Value *Addr = Builder.CreateConstInBoundsGEP1_64(Type::getInt64Ty(Ctx),
+                                                   Counters, Index);
 
   if (Options.Atomic || AtomicCounterUpdateAll) {
     Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
                             AtomicOrdering::Monotonic);
   } else {
     Value *IncStep = Inc->getStep();
     Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
     auto *Count = Builder.CreateAdd(Load, Inc->getStep());
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
   if (TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
       TT.isOSSolaris() || TT.isOSFuchsia() || TT.isPS4CPU() ||
       TT.isOSWindows())
     return false;
 
   return true;
 }
 
-GlobalVariable *
+Value *
 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
   GlobalVariable *NamePtr = Inc->getName();
   auto It = ProfileDataMap.find(NamePtr);
   PerFunctionProfileData PD;
   if (It != ProfileDataMap.end()) {
     if (It->second.RegionCounters)
       return It->second.RegionCounters;
     PD = It->second;
InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
   }
   auto MaybeSetComdat = [=](GlobalVariable *GV) {
     if (NeedComdat)
       GV->setComdat(M->getOrInsertComdat(GV->getName()));
   };
 
   uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
   LLVMContext &Ctx = M->getContext();
-  ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
 
+  auto *Int64PtrTy = Type::getInt64PtrTy(Ctx);
   // Create the counters variable.
-  auto *CounterPtr =
+  Constant *CounterPtr = ConstantPointerNull::get(Int64PtrTy);
+  if (CounterStaticAlloc) {
+    ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
+    auto *CounterVar =
         new GlobalVariable(*M, CounterTy, false, Linkage,
                            Constant::getNullValue(CounterTy),
                            getVarName(Inc, getInstrProfCountersVarPrefix()));
-  CounterPtr->setVisibility(Visibility);
-  CounterPtr->setSection(
+    CounterVar->setVisibility(Visibility);
+    CounterVar->setSection(
         getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
-  CounterPtr->setAlignment(Align(8));
-  MaybeSetComdat(CounterPtr);
-  CounterPtr->setLinkage(Linkage);
+    CounterVar->setAlignment(Align(8));
+    MaybeSetComdat(CounterVar);
+    CounterVar->setLinkage(Linkage);
+    CounterPtr =
+        ConstantExpr::getBitCast(CounterVar, Type::getInt64PtrTy(Ctx));
+  }
 
   auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
   // Allocate statically the array of pointers to value profile nodes for
   // the current function.
   Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
   if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(TT)) {
     uint64_t NS = 0;
     for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
auto *Data = new GlobalVariable(*M, DataTy, false, Linkage,
                                   ConstantStruct::get(DataTy, DataVals),
                                   getVarName(Inc, getInstrProfDataVarPrefix()));
   Data->setVisibility(Visibility);
   Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
   Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
   MaybeSetComdat(Data);
   Data->setLinkage(Linkage);
 
+  IRBuilder<> Builder(Inc);
+
+  if (CounterStaticAlloc) {
     PD.RegionCounters = CounterPtr;
+  } else {
+    auto *Addr = Builder.CreateConstInBoundsGEP2_32(DataTy, Data, 0, 2);
+    PD.RegionCounters = Builder.CreateLoad(llvm::Type::getInt64PtrTy(Ctx), Addr);
+  }
   PD.DataVar = Data;
   ProfileDataMap[NamePtr] = PD;
 
   // Mark the data variable as used so that it isn't stripped out.
   UsedVars.push_back(Data);
   // Now that the linkage set by the FE has been passed to the data and counter
   // variables, reset Name variable's linkage and visibility to private so that
   // it can be removed later by the compiler.