@@ -144,119 +144,181 @@ bool PGOProfileData::getFunctionCounts(StringRef FuncName,
144
144
}
145
145
146
146
void CodeGenPGO::setFuncName (llvm::Function *Fn) {
147
- StringRef Func = Fn->getName ();
147
+ RawFuncName = Fn->getName ();
148
148
149
149
// Function names may be prefixed with a binary '1' to indicate
150
150
// that the backend should not modify the symbols due to any platform
151
151
// naming convention. Do not include that '1' in the PGO profile name.
152
- if (Func [0 ] == ' \1 ' )
153
- Func = Func .substr (1 );
152
+ if (RawFuncName [0 ] == ' \1 ' )
153
+ RawFuncName = RawFuncName .substr (1 );
154
154
155
155
if (!Fn->hasLocalLinkage ()) {
156
- FuncName = new std::string (Func );
156
+ PrefixedFuncName = new std::string (RawFuncName );
157
157
return ;
158
158
}
159
159
160
160
// For local symbols, prepend the main file name to distinguish them.
161
161
// Do not include the full path in the file name since there's no guarantee
162
162
// that it will stay the same, e.g., if the files are checked out from
163
163
// version control in different locations.
164
- FuncName = new std::string (CGM.getCodeGenOpts ().MainFileName );
165
- if (FuncName ->empty ())
166
- FuncName ->assign (" <unknown>" );
167
- FuncName ->append (" :" );
168
- FuncName ->append (Func );
164
+ PrefixedFuncName = new std::string (CGM.getCodeGenOpts ().MainFileName );
165
+ if (PrefixedFuncName ->empty ())
166
+ PrefixedFuncName ->assign (" <unknown>" );
167
+ PrefixedFuncName ->append (" :" );
168
+ PrefixedFuncName ->append (RawFuncName );
169
169
}
170
170
171
- void CodeGenPGO::emitWriteoutFunction () {
172
- if (!CGM.getCodeGenOpts ().ProfileInstrGenerate )
173
- return ;
174
-
175
- llvm::LLVMContext &Ctx = CGM.getLLVMContext ();
171
+ static llvm::Function *getRegisterFunc (CodeGenModule &CGM) {
172
+ return CGM.getModule ().getFunction (" __llvm_pgo_register_functions" );
173
+ }
176
174
177
- llvm::Type *Int32Ty = llvm::Type::getInt32Ty (Ctx);
178
- llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy (Ctx);
179
-
180
- llvm::Function *WriteoutF =
181
- CGM. getModule (). getFunction ( " __llvm_pgo_writeout " );
182
- if (!WriteoutF) {
183
- llvm::FunctionType *WriteoutFTy =
184
- llvm::FunctionType::get (llvm::Type::getVoidTy (Ctx) , false );
185
- WriteoutF = llvm::Function::Create (WriteoutFTy ,
186
- llvm::GlobalValue::InternalLinkage,
187
- " __llvm_pgo_writeout " , &CGM. getModule ());
188
- }
189
- WriteoutF ->setUnnamedAddr (true );
190
- WriteoutF ->addFnAttr (llvm::Attribute::NoInline);
175
+ static llvm::BasicBlock * getOrInsertRegisterBB (CodeGenModule &CGM) {
176
+ // Only need to insert this once per module.
177
+ if (llvm::Function *RegisterF = getRegisterFunc (CGM))
178
+ return &RegisterF-> getEntryBlock ();
179
+
180
+ // Construct the function.
181
+ auto *VoidTy = llvm::Type::getVoidTy (CGM. getLLVMContext ());
182
+ auto *RegisterFTy = llvm::FunctionType::get (VoidTy , false );
183
+ auto *RegisterF = llvm::Function::Create (RegisterFTy ,
184
+ llvm::GlobalValue::InternalLinkage,
185
+ " __llvm_pgo_register_functions " ,
186
+ &CGM. getModule ());
187
+ RegisterF ->setUnnamedAddr (true );
188
+ RegisterF ->addFnAttr (llvm::Attribute::NoInline);
191
189
if (CGM.getCodeGenOpts ().DisableRedZone )
192
- WriteoutF->addFnAttr (llvm::Attribute::NoRedZone);
190
+ RegisterF->addFnAttr (llvm::Attribute::NoRedZone);
191
+
192
+ // Construct and return the entry block.
193
+ auto *BB = llvm::BasicBlock::Create (CGM.getLLVMContext (), " " , RegisterF);
194
+ CGBuilderTy Builder (BB);
195
+ Builder.CreateRetVoid ();
196
+ return BB;
197
+ }
193
198
194
- llvm::BasicBlock *BB = WriteoutF->empty () ?
195
- llvm::BasicBlock::Create (Ctx, " " , WriteoutF) : &WriteoutF->getEntryBlock ();
199
+ static llvm::Constant *getOrInsertRuntimeRegister (CodeGenModule &CGM) {
200
+ auto *VoidTy = llvm::Type::getVoidTy (CGM.getLLVMContext ());
201
+ auto *VoidPtrTy = llvm::Type::getInt8PtrTy (CGM.getLLVMContext ());
202
+ auto *RuntimeRegisterTy = llvm::FunctionType::get (VoidTy, VoidPtrTy, false );
203
+ return CGM.getModule ().getOrInsertFunction (" __llvm_pgo_register_function" ,
204
+ RuntimeRegisterTy);
205
+ }
206
+
207
+ static llvm::Constant *getOrInsertRuntimeWriteAtExit (CodeGenModule &CGM) {
208
+ // TODO: make this depend on a command-line option.
209
+ auto *VoidTy = llvm::Type::getVoidTy (CGM.getLLVMContext ());
210
+ auto *WriteAtExitTy = llvm::FunctionType::get (VoidTy, false );
211
+ return CGM.getModule ().getOrInsertFunction (" __llvm_pgo_register_write_atexit" ,
212
+ WriteAtExitTy);
213
+ }
214
+
215
+ static StringRef getCountersSection (const CodeGenModule &CGM) {
216
+ if (CGM.getTarget ().getTriple ().getObjectFormat () == llvm::Triple::MachO)
217
+ return " __DATA,__llvm_pgo_cnts" ;
218
+ else
219
+ return " __llvm_pgo_cnts" ;
220
+ }
196
221
197
- CGBuilderTy PGOBuilder (BB);
222
+ static StringRef getNameSection (const CodeGenModule &CGM) {
223
+ if (CGM.getTarget ().getTriple ().getObjectFormat () == llvm::Triple::MachO)
224
+ return " __DATA,__llvm_pgo_names" ;
225
+ else
226
+ return " __llvm_pgo_names" ;
227
+ }
198
228
199
- llvm::Instruction *I = BB->getTerminator ();
200
- if (!I)
201
- I = PGOBuilder.CreateRetVoid ();
202
- PGOBuilder.SetInsertPoint (I);
229
+ static StringRef getDataSection (const CodeGenModule &CGM) {
230
+ if (CGM.getTarget ().getTriple ().getObjectFormat () == llvm::Triple::MachO)
231
+ return " __DATA,__llvm_pgo_data" ;
232
+ else
233
+ return " __llvm_pgo_data" ;
234
+ }
203
235
204
- llvm::Type *Int64PtrTy = llvm::Type::getInt64PtrTy (Ctx);
205
- llvm::Type *Args[] = {
206
- Int8PtrTy, // const char *FuncName
207
- Int32Ty, // uint32_t NumCounters
208
- Int64PtrTy // uint64_t *Counters
236
+ llvm::GlobalVariable *CodeGenPGO::buildDataVar () {
237
+ // Create name variable.
238
+ llvm::LLVMContext &Ctx = CGM.getLLVMContext ();
239
+ auto *VarName = llvm::ConstantDataArray::getString (Ctx, getFuncName (),
240
+ false );
241
+ auto *Name = new llvm::GlobalVariable (CGM.getModule (), VarName->getType (),
242
+ true , FuncLinkage, VarName,
243
+ getFuncVarName (" name" ));
244
+ Name->setSection (getNameSection (CGM));
245
+ Name->setAlignment (1 );
246
+
247
+ // Create data variable.
248
+ auto *Int32Ty = llvm::Type::getInt32Ty (Ctx);
249
+ auto *Int8PtrTy = llvm::Type::getInt8PtrTy (Ctx);
250
+ auto *Int64PtrTy = llvm::Type::getInt64PtrTy (Ctx);
251
+ llvm::Type *DataTypes[] = {
252
+ Int32Ty, Int32Ty, Int8PtrTy, Int64PtrTy
209
253
};
210
- llvm::FunctionType *FTy =
211
- llvm::FunctionType::get (PGOBuilder.getVoidTy (), Args, false );
212
- llvm::Constant *EmitFunc =
213
- CGM.getModule ().getOrInsertFunction (" llvm_pgo_emit" , FTy);
214
-
215
- llvm::Constant *NameString =
216
- CGM.GetAddrOfConstantCString (getFuncName (), " __llvm_pgo_name" );
217
- NameString = llvm::ConstantExpr::getBitCast (NameString, Int8PtrTy);
218
- PGOBuilder.CreateCall3 (EmitFunc, NameString,
219
- PGOBuilder.getInt32 (NumRegionCounters),
220
- PGOBuilder.CreateBitCast (RegionCounters, Int64PtrTy));
254
+ auto *DataTy = llvm::StructType::get (Ctx, makeArrayRef (DataTypes));
255
+ llvm::Constant *DataVals[] = {
256
+ llvm::ConstantInt::get (Int32Ty, getFuncName ().size ()),
257
+ llvm::ConstantInt::get (Int32Ty, NumRegionCounters),
258
+ llvm::ConstantExpr::getBitCast (Name, Int8PtrTy),
259
+ llvm::ConstantExpr::getBitCast (RegionCounters, Int64PtrTy)
260
+ };
261
+ auto *Data =
262
+ new llvm::GlobalVariable (CGM.getModule (), DataTy, true , FuncLinkage,
263
+ llvm::ConstantStruct::get (DataTy, DataVals),
264
+ getFuncVarName (" data" ));
265
+
266
+ // All the data should be packed into an array in its own section.
267
+ Data->setSection (getDataSection (CGM));
268
+ Data->setAlignment (8 );
269
+
270
+ // Make sure the data doesn't get deleted.
271
+ CGM.addUsedGlobal (Data);
272
+ return Data;
273
+ }
274
+
275
+ void CodeGenPGO::emitInstrumentationData () {
276
+ if (!CGM.getCodeGenOpts ().ProfileInstrGenerate )
277
+ return ;
278
+
279
+ // Build the data.
280
+ auto *Data = buildDataVar ();
281
+
282
+ // Register the data.
283
+ //
284
+ // TODO: only register when static initialization is required.
285
+ CGBuilderTy Builder (getOrInsertRegisterBB (CGM)->getTerminator ());
286
+ auto *VoidPtrTy = llvm::Type::getInt8PtrTy (CGM.getLLVMContext ());
287
+ Builder.CreateCall (getOrInsertRuntimeRegister (CGM),
288
+ Builder.CreateBitCast (Data, VoidPtrTy));
221
289
}
222
290
223
291
llvm::Function *CodeGenPGO::emitInitialization (CodeGenModule &CGM) {
224
- llvm::Function *WriteoutF =
225
- CGM.getModule ().getFunction (" __llvm_pgo_writeout" );
226
- if (!WriteoutF)
227
- return NULL ;
292
+ if (!CGM.getCodeGenOpts ().ProfileInstrGenerate )
293
+ return 0 ;
228
294
229
- // Create a small bit of code that registers the "__llvm_pgo_writeout" to
230
- // be executed at exit.
231
- llvm::Function *F = CGM.getModule ().getFunction (" __llvm_pgo_init" );
232
- if (F)
233
- return NULL ;
295
+ // Only need to create this once per module.
296
+ if (CGM.getModule ().getFunction (" __llvm_pgo_init" ))
297
+ return 0 ;
234
298
235
- llvm::LLVMContext &Ctx = CGM.getLLVMContext ();
236
- llvm::FunctionType *FTy = llvm::FunctionType::get (llvm::Type::getVoidTy (Ctx),
237
- false );
238
- F = llvm::Function::Create (FTy, llvm::GlobalValue::InternalLinkage,
239
- " __llvm_pgo_init" , &CGM.getModule ());
299
+ // Get the functions to call at initialization.
300
+ llvm::Constant *RegisterF = getRegisterFunc (CGM);
301
+ llvm::Constant *WriteAtExitF = getOrInsertRuntimeWriteAtExit (CGM);
302
+ if (!RegisterF && !WriteAtExitF)
303
+ return 0 ;
304
+
305
+ // Create the initialization function.
306
+ auto *VoidTy = llvm::Type::getVoidTy (CGM.getLLVMContext ());
307
+ auto *F = llvm::Function::Create (llvm::FunctionType::get (VoidTy, false ),
308
+ llvm::GlobalValue::InternalLinkage,
309
+ " __llvm_pgo_init" , &CGM.getModule ());
240
310
F->setUnnamedAddr (true );
241
- F->setLinkage (llvm::GlobalValue::InternalLinkage);
242
311
F->addFnAttr (llvm::Attribute::NoInline);
243
312
if (CGM.getCodeGenOpts ().DisableRedZone )
244
313
F->addFnAttr (llvm::Attribute::NoRedZone);
245
314
246
- llvm::BasicBlock *BB = llvm::BasicBlock::Create (CGM.getLLVMContext (), " " , F);
247
- CGBuilderTy PGOBuilder (BB);
248
-
249
- FTy = llvm::FunctionType::get (PGOBuilder.getVoidTy (), false );
250
- llvm::Type *Params[] = {
251
- llvm::PointerType::get (FTy, 0 )
252
- };
253
- FTy = llvm::FunctionType::get (PGOBuilder.getVoidTy (), Params, false );
254
-
255
- // Inialize the environment and register the local writeout function.
256
- llvm::Constant *PGOInit =
257
- CGM.getModule ().getOrInsertFunction (" llvm_pgo_init" , FTy);
258
- PGOBuilder.CreateCall (PGOInit, WriteoutF);
259
- PGOBuilder.CreateRetVoid ();
315
+ // Add the basic block and the necessary calls.
316
+ CGBuilderTy Builder (llvm::BasicBlock::Create (CGM.getLLVMContext (), " " , F));
317
+ if (RegisterF)
318
+ Builder.CreateCall (RegisterF);
319
+ if (WriteAtExitF)
320
+ Builder.CreateCall (WriteAtExitF);
321
+ Builder.CreateRetVoid ();
260
322
261
323
return F;
262
324
}
@@ -764,6 +826,7 @@ void CodeGenPGO::assignRegionCounters(const Decl *D, llvm::Function *Fn) {
764
826
if (!D)
765
827
return ;
766
828
setFuncName (Fn);
829
+ FuncLinkage = Fn->getLinkage ();
767
830
mapRegionCounters (D);
768
831
if (InstrumentRegions)
769
832
emitCounterVariables ();
@@ -819,10 +882,11 @@ void CodeGenPGO::emitCounterVariables() {
819
882
llvm::ArrayType *CounterTy = llvm::ArrayType::get (llvm::Type::getInt64Ty (Ctx),
820
883
NumRegionCounters);
821
884
RegionCounters =
822
- new llvm::GlobalVariable (CGM.getModule (), CounterTy, false ,
823
- llvm::GlobalVariable::PrivateLinkage,
885
+ new llvm::GlobalVariable (CGM.getModule (), CounterTy, false , FuncLinkage,
824
886
llvm::Constant::getNullValue (CounterTy),
825
- " __llvm_pgo_ctr" );
887
+ getFuncVarName (" counters" ));
888
+ RegionCounters->setAlignment (8 );
889
+ RegionCounters->setSection (getCountersSection (CGM));
826
890
}
827
891
828
892
void CodeGenPGO::emitCounterIncrement (CGBuilderTy &Builder, unsigned Counter) {
0 commit comments