diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index 907d299ae0f3..ef1004b98d34 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -1,450 +1,450 @@ //===- llvm/InitializePasses.h - Initialize All Passes ----------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file contains the declarations for the pass initialization routines // for the entire LLVM project. // //===----------------------------------------------------------------------===// #ifndef LLVM_INITIALIZEPASSES_H #define LLVM_INITIALIZEPASSES_H namespace llvm { class PassRegistry; /// Initialize all passes linked into the TransformUtils library. void initializeCore(PassRegistry&); /// Initialize all passes linked into the TransformUtils library. void initializeTransformUtils(PassRegistry&); /// Initialize all passes linked into the ScalarOpts library. void initializeScalarOpts(PassRegistry&); /// Initialize all passes linked into the ObjCARCOpts library. void initializeObjCARCOpts(PassRegistry&); /// Initialize all passes linked into the Vectorize library. void initializeVectorization(PassRegistry&); /// Initialize all passes linked into the InstCombine library. void initializeInstCombine(PassRegistry&); /// Initialize all passes linked into the AggressiveInstCombine library. void initializeAggressiveInstCombine(PassRegistry&); /// Initialize all passes linked into the IPO library. void initializeIPO(PassRegistry&); /// Initialize all passes linked into the Instrumentation library. void initializeInstrumentation(PassRegistry&); /// Initialize all passes linked into the Analysis library. void initializeAnalysis(PassRegistry&); /// Initialize all passes linked into the Coroutines library. void initializeCoroutines(PassRegistry&); /// Initialize all passes linked into the CodeGen library. void initializeCodeGen(PassRegistry&); /// Initialize all passes linked into the GlobalISel library. void initializeGlobalISel(PassRegistry&); /// Initialize all passes linked into the CodeGen library. void initializeTarget(PassRegistry&); void initializeAAEvalLegacyPassPass(PassRegistry&); void initializeAAResultsWrapperPassPass(PassRegistry&); void initializeADCELegacyPassPass(PassRegistry&); void initializeAddDiscriminatorsLegacyPassPass(PassRegistry&); void initializeModuleAddressSanitizerLegacyPassPass(PassRegistry &); void initializeASanGlobalsMetadataWrapperPassPass(PassRegistry &); void initializeAddressSanitizerLegacyPassPass(PassRegistry &); void initializeAggressiveInstCombinerLegacyPassPass(PassRegistry&); void initializeAliasSetPrinterPass(PassRegistry&); void initializeAlignmentFromAssumptionsPass(PassRegistry&); void initializeAlwaysInlinerLegacyPassPass(PassRegistry&); void initializeAssumeSimplifyPassLegacyPassPass(PassRegistry &); void initializeAssumeBuilderPassLegacyPassPass(PassRegistry &); void initializeOpenMPOptLegacyPassPass(PassRegistry &); void initializeArgPromotionPass(PassRegistry&); void initializeAssumptionCacheTrackerPass(PassRegistry&); void initializeAtomicExpandPass(PassRegistry&); void initializeAttributorLegacyPassPass(PassRegistry&); void initializeAttributorCGSCCLegacyPassPass(PassRegistry &); void initializeBasicBlockSectionsPass(PassRegistry &); void initializeBDCELegacyPassPass(PassRegistry&); void initializeBarrierNoopPass(PassRegistry&); void initializeBasicAAWrapperPassPass(PassRegistry&); void initializeBlockExtractorPass(PassRegistry &); void initializeBlockFrequencyInfoWrapperPassPass(PassRegistry&); void initializeBoundsCheckingLegacyPassPass(PassRegistry&); void initializeBranchFolderPassPass(PassRegistry&); void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry&); void initializeBranchRelaxationPass(PassRegistry&); void initializeBreakCriticalEdgesPass(PassRegistry&); void initializeBreakFalseDepsPass(PassRegistry&); void initializeCanonicalizeAliasesLegacyPassPass(PassRegistry &); void initializeCanonicalizeFreezeInLoopsPass(PassRegistry &); void initializeCFGOnlyPrinterLegacyPassPass(PassRegistry&); void initializeCFGOnlyViewerLegacyPassPass(PassRegistry&); void initializeCFGPrinterLegacyPassPass(PassRegistry&); void initializeCFGSimplifyPassPass(PassRegistry&); void initializeCFGuardPass(PassRegistry&); void initializeCFGuardLongjmpPass(PassRegistry&); void initializeCFGViewerLegacyPassPass(PassRegistry&); void initializeCFIInstrInserterPass(PassRegistry&); void initializeCFLAndersAAWrapperPassPass(PassRegistry&); void initializeCFLSteensAAWrapperPassPass(PassRegistry&); void initializeCGProfileLegacyPassPass(PassRegistry &); void initializeCallGraphDOTPrinterPass(PassRegistry&); void initializeCallGraphPrinterLegacyPassPass(PassRegistry&); void initializeCallGraphViewerPass(PassRegistry&); void initializeCallGraphWrapperPassPass(PassRegistry&); void initializeCallSiteSplittingLegacyPassPass(PassRegistry&); void initializeCalledValuePropagationLegacyPassPass(PassRegistry &); void initializeCodeGenPreparePass(PassRegistry&); void initializeConstantHoistingLegacyPassPass(PassRegistry&); void initializeConstantMergeLegacyPassPass(PassRegistry&); void initializeConstraintEliminationPass(PassRegistry &); void initializeControlHeightReductionLegacyPassPass(PassRegistry&); void initializeCorrelatedValuePropagationPass(PassRegistry&); void initializeCostModelAnalysisPass(PassRegistry&); void initializeCrossDSOCFIPass(PassRegistry&); void initializeDAEPass(PassRegistry&); void initializeDAHPass(PassRegistry&); void initializeDCELegacyPassPass(PassRegistry&); void initializeDSELegacyPassPass(PassRegistry&); void initializeDataFlowSanitizerLegacyPassPass(PassRegistry &); void initializeDeadMachineInstructionElimPass(PassRegistry&); void initializeDebugifyMachineModulePass(PassRegistry &); void initializeDelinearizationPass(PassRegistry&); void initializeDemandedBitsWrapperPassPass(PassRegistry&); void initializeDependenceAnalysisPass(PassRegistry&); void initializeDependenceAnalysisWrapperPassPass(PassRegistry&); void initializeDetectDeadLanesPass(PassRegistry&); void initializeDivRemPairsLegacyPassPass(PassRegistry&); void initializeDomOnlyPrinterPass(PassRegistry&); void initializeDomOnlyViewerPass(PassRegistry&); void initializeDomPrinterPass(PassRegistry&); void initializeDomViewerPass(PassRegistry&); void initializeDominanceFrontierWrapperPassPass(PassRegistry&); void initializeDominatorTreeWrapperPassPass(PassRegistry&); void initializeDwarfEHPreparePass(PassRegistry&); void initializeEarlyCSELegacyPassPass(PassRegistry&); void initializeEarlyCSEMemSSALegacyPassPass(PassRegistry&); void initializeEarlyIfConverterPass(PassRegistry&); void initializeEarlyIfPredicatorPass(PassRegistry &); void initializeEarlyMachineLICMPass(PassRegistry&); void initializeEarlyTailDuplicatePass(PassRegistry&); void initializeEdgeBundlesPass(PassRegistry&); void initializeEliminateAvailableExternallyLegacyPassPass(PassRegistry&); void initializeEntryExitInstrumenterPass(PassRegistry&); void initializeExpandMemCmpPassPass(PassRegistry&); void initializeExpandPostRAPass(PassRegistry&); void initializeExpandReductionsPass(PassRegistry&); void initializeMakeGuardsExplicitLegacyPassPass(PassRegistry&); void initializeExternalAAWrapperPassPass(PassRegistry&); void initializeFEntryInserterPass(PassRegistry&); void initializeFinalizeISelPass(PassRegistry&); void initializeFinalizeMachineBundlesPass(PassRegistry&); void initializeFixIrreduciblePass(PassRegistry &); void initializeFixupStatepointCallerSavedPass(PassRegistry&); void initializeFlattenCFGPassPass(PassRegistry&); void initializeFloat2IntLegacyPassPass(PassRegistry&); void initializeForceFunctionAttrsLegacyPassPass(PassRegistry&); void initializeForwardControlFlowIntegrityPass(PassRegistry&); void initializeFuncletLayoutPass(PassRegistry&); void initializeFunctionImportLegacyPassPass(PassRegistry&); void initializeGCMachineCodeAnalysisPass(PassRegistry&); void initializeGCModuleInfoPass(PassRegistry&); void initializeGCOVProfilerLegacyPassPass(PassRegistry&); void initializeGVNHoistLegacyPassPass(PassRegistry&); void initializeGVNLegacyPassPass(PassRegistry&); void initializeGVNSinkLegacyPassPass(PassRegistry&); void initializeGlobalDCELegacyPassPass(PassRegistry&); void initializeGlobalMergePass(PassRegistry&); void initializeGlobalOptLegacyPassPass(PassRegistry&); void initializeGlobalSplitPass(PassRegistry&); void initializeGlobalsAAWrapperPassPass(PassRegistry&); void initializeGuardWideningLegacyPassPass(PassRegistry&); void initializeHardwareLoopsPass(PassRegistry&); void initializeMemProfilerLegacyPassPass(PassRegistry &); void initializeHotColdSplittingLegacyPassPass(PassRegistry&); void initializeHWAddressSanitizerLegacyPassPass(PassRegistry &); void initializeIPSCCPLegacyPassPass(PassRegistry&); void initializeIRCELegacyPassPass(PassRegistry&); void initializeIRSimilarityIdentifierWrapperPassPass(PassRegistry&); void initializeIRTranslatorPass(PassRegistry&); void initializeIVUsersWrapperPassPass(PassRegistry&); void initializeIfConverterPass(PassRegistry&); void initializeImmutableModuleSummaryIndexWrapperPassPass(PassRegistry&); void initializeImplicitNullChecksPass(PassRegistry&); void initializeIndVarSimplifyLegacyPassPass(PassRegistry&); void initializeIndirectBrExpandPassPass(PassRegistry&); void initializeInferAddressSpacesPass(PassRegistry&); void initializeInferFunctionAttrsLegacyPassPass(PassRegistry&); void initializeInjectTLIMappingsLegacyPass(PassRegistry &); void initializeInlineCostAnalysisPass(PassRegistry&); void initializeInstCountLegacyPassPass(PassRegistry &); void initializeInstNamerPass(PassRegistry&); void initializeInstSimplifyLegacyPassPass(PassRegistry &); void initializeInstrProfilingLegacyPassPass(PassRegistry&); void initializeInstrOrderFileLegacyPassPass(PassRegistry&); void initializeInstructionCombiningPassPass(PassRegistry&); void initializeInstructionSelectPass(PassRegistry&); void initializeInterleavedAccessPass(PassRegistry&); void initializeInterleavedLoadCombinePass(PassRegistry &); void initializeInternalizeLegacyPassPass(PassRegistry&); void initializeIntervalPartitionPass(PassRegistry&); void initializeJumpThreadingPass(PassRegistry&); void initializeLCSSAVerificationPassPass(PassRegistry&); void initializeLCSSAWrapperPassPass(PassRegistry&); void initializeLazyBlockFrequencyInfoPassPass(PassRegistry&); void initializeLazyBranchProbabilityInfoPassPass(PassRegistry&); void initializeLazyMachineBlockFrequencyInfoPassPass(PassRegistry&); void initializeLazyValueInfoPrinterPass(PassRegistry&); void initializeLazyValueInfoWrapperPassPass(PassRegistry&); void initializeLegacyDivergenceAnalysisPass(PassRegistry&); void initializeLegacyLICMPassPass(PassRegistry&); void initializeLegacyLoopSinkPassPass(PassRegistry&); void initializeLegalizerPass(PassRegistry&); void initializeGISelCSEAnalysisWrapperPassPass(PassRegistry &); void initializeGISelKnownBitsAnalysisPass(PassRegistry &); void initializeLibCallsShrinkWrapLegacyPassPass(PassRegistry&); void initializeLintLegacyPassPass(PassRegistry &); void initializeLiveDebugValuesPass(PassRegistry&); void initializeLiveDebugVariablesPass(PassRegistry&); void initializeLiveIntervalsPass(PassRegistry&); void initializeLiveRangeShrinkPass(PassRegistry&); void initializeLiveRegMatrixPass(PassRegistry&); void initializeLiveStacksPass(PassRegistry&); void initializeLiveVariablesPass(PassRegistry&); void initializeLoadStoreVectorizerLegacyPassPass(PassRegistry&); void initializeLoaderPassPass(PassRegistry&); void initializeLocalStackSlotPassPass(PassRegistry&); void initializeLocalizerPass(PassRegistry&); void initializeLoopAccessLegacyAnalysisPass(PassRegistry&); void initializeLoopDataPrefetchLegacyPassPass(PassRegistry&); void initializeLoopDeletionLegacyPassPass(PassRegistry&); void initializeLoopDistributeLegacyPass(PassRegistry&); void initializeLoopExtractorPass(PassRegistry&); void initializeLoopGuardWideningLegacyPassPass(PassRegistry&); void initializeLoopFuseLegacyPass(PassRegistry&); void initializeLoopIdiomRecognizeLegacyPassPass(PassRegistry&); void initializeLoopInfoWrapperPassPass(PassRegistry&); void initializeLoopInstSimplifyLegacyPassPass(PassRegistry&); void initializeLoopInterchangePass(PassRegistry&); void initializeLoopFlattenLegacyPassPass(PassRegistry&); void initializeLoopLoadEliminationPass(PassRegistry&); void initializeLoopPassPass(PassRegistry&); void initializeLoopPredicationLegacyPassPass(PassRegistry&); void initializeLoopRerollLegacyPassPass(PassRegistry &); void initializeLoopRotateLegacyPassPass(PassRegistry&); void initializeLoopSimplifyCFGLegacyPassPass(PassRegistry&); void initializeLoopSimplifyPass(PassRegistry&); void initializeLoopStrengthReducePass(PassRegistry&); void initializeLoopUnrollAndJamPass(PassRegistry&); void initializeLoopUnrollPass(PassRegistry&); void initializeLoopUnswitchPass(PassRegistry&); void initializeLoopVectorizePass(PassRegistry&); void initializeLoopVersioningLICMPass(PassRegistry&); void initializeLoopVersioningLegacyPassPass(PassRegistry &); void initializeLowerAtomicLegacyPassPass(PassRegistry&); void initializeLowerConstantIntrinsicsPass(PassRegistry&); void initializeLowerEmuTLSPass(PassRegistry&); void initializeLowerExpectIntrinsicPass(PassRegistry&); void initializeLowerGuardIntrinsicLegacyPassPass(PassRegistry&); void initializeLowerWidenableConditionLegacyPassPass(PassRegistry&); void initializeLowerIntrinsicsPass(PassRegistry&); void initializeLowerInvokeLegacyPassPass(PassRegistry&); void initializeLowerSwitchLegacyPassPass(PassRegistry &); void initializeLowerTypeTestsPass(PassRegistry&); void initializeLowerMatrixIntrinsicsLegacyPassPass(PassRegistry &); void initializeLowerMatrixIntrinsicsMinimalLegacyPassPass(PassRegistry &); void initializeMIRCanonicalizerPass(PassRegistry &); void initializeMIRNamerPass(PassRegistry &); void initializeMIRPrintingPassPass(PassRegistry&); void initializeMachineBlockFrequencyInfoPass(PassRegistry&); void initializeMachineBlockPlacementPass(PassRegistry&); void initializeMachineBlockPlacementStatsPass(PassRegistry&); void initializeMachineBranchProbabilityInfoPass(PassRegistry&); void initializeMachineCSEPass(PassRegistry&); void initializeMachineCombinerPass(PassRegistry&); void initializeMachineCopyPropagationPass(PassRegistry&); void initializeMachineDominanceFrontierPass(PassRegistry&); void initializeMachineDominatorTreePass(PassRegistry&); void initializeMachineFunctionPrinterPassPass(PassRegistry&); void initializeMachineFunctionSplitterPass(PassRegistry &); void initializeMachineLICMPass(PassRegistry&); void initializeMachineLoopInfoPass(PassRegistry&); void initializeMachineModuleInfoWrapperPassPass(PassRegistry &); void initializeMachineOptimizationRemarkEmitterPassPass(PassRegistry&); void initializeMachineOutlinerPass(PassRegistry&); void initializeMachinePipelinerPass(PassRegistry&); void initializeMachinePostDominatorTreePass(PassRegistry&); void initializeMachineRegionInfoPassPass(PassRegistry&); void initializeMachineSchedulerPass(PassRegistry&); void initializeMachineSinkingPass(PassRegistry&); void initializeMachineTraceMetricsPass(PassRegistry&); void initializeMachineVerifierPassPass(PassRegistry&); void initializeMemCpyOptLegacyPassPass(PassRegistry&); void initializeMemDepPrinterPass(PassRegistry&); void initializeMemDerefPrinterPass(PassRegistry&); void initializeMemoryDependenceWrapperPassPass(PassRegistry&); void initializeMemorySSAPrinterLegacyPassPass(PassRegistry&); void initializeMemorySSAWrapperPassPass(PassRegistry&); void initializeMemorySanitizerLegacyPassPass(PassRegistry&); void initializeMergeFunctionsLegacyPassPass(PassRegistry&); void initializeMergeICmpsLegacyPassPass(PassRegistry &); void initializeMergedLoadStoreMotionLegacyPassPass(PassRegistry&); void initializeMetaRenamerPass(PassRegistry&); void initializeModuleDebugInfoPrinterPass(PassRegistry&); void initializeModuleMemProfilerLegacyPassPass(PassRegistry &); void initializeModuleSummaryIndexWrapperPassPass(PassRegistry&); void initializeModuloScheduleTestPass(PassRegistry&); void initializeMustExecutePrinterPass(PassRegistry&); void initializeMustBeExecutedContextPrinterPass(PassRegistry&); void initializeNameAnonGlobalLegacyPassPass(PassRegistry&); void initializeUniqueInternalLinkageNamesLegacyPassPass(PassRegistry &); void initializeNaryReassociateLegacyPassPass(PassRegistry&); void initializeNewGVNLegacyPassPass(PassRegistry&); void initializeObjCARCAAWrapperPassPass(PassRegistry&); void initializeObjCARCAPElimPass(PassRegistry&); void initializeObjCARCContractLegacyPassPass(PassRegistry &); void initializeObjCARCExpandPass(PassRegistry&); void initializeObjCARCOptLegacyPassPass(PassRegistry &); void initializeOptimizationRemarkEmitterWrapperPassPass(PassRegistry&); void initializeOptimizePHIsPass(PassRegistry&); void initializePAEvalPass(PassRegistry&); void initializePEIPass(PassRegistry&); void initializePGOIndirectCallPromotionLegacyPassPass(PassRegistry&); void initializePGOInstrumentationGenLegacyPassPass(PassRegistry&); void initializePGOInstrumentationUseLegacyPassPass(PassRegistry&); void initializePGOInstrumentationGenCreateVarLegacyPassPass(PassRegistry&); void initializePGOMemOPSizeOptLegacyPassPass(PassRegistry&); void initializePHIEliminationPass(PassRegistry&); void initializePartialInlinerLegacyPassPass(PassRegistry&); void initializePartiallyInlineLibCallsLegacyPassPass(PassRegistry&); void initializePatchableFunctionPass(PassRegistry&); void initializePeepholeOptimizerPass(PassRegistry&); void initializePhiValuesWrapperPassPass(PassRegistry&); void initializePhysicalRegisterUsageInfoPass(PassRegistry&); void initializePlaceBackedgeSafepointsImplPass(PassRegistry&); void initializePlaceSafepointsPass(PassRegistry&); void initializePostDomOnlyPrinterPass(PassRegistry&); void initializePostDomOnlyViewerPass(PassRegistry&); void initializePostDomPrinterPass(PassRegistry&); void initializePostDomViewerPass(PassRegistry&); void initializePostDominatorTreeWrapperPassPass(PassRegistry&); void initializePostInlineEntryExitInstrumenterPass(PassRegistry&); void initializePostMachineSchedulerPass(PassRegistry&); void initializePostOrderFunctionAttrsLegacyPassPass(PassRegistry&); void initializePostRAHazardRecognizerPass(PassRegistry&); void initializePostRAMachineSinkingPass(PassRegistry&); void initializePostRASchedulerPass(PassRegistry&); void initializePreISelIntrinsicLoweringLegacyPassPass(PassRegistry&); void initializePredicateInfoPrinterLegacyPassPass(PassRegistry&); void initializePrintFunctionPassWrapperPass(PassRegistry&); void initializePrintModulePassWrapperPass(PassRegistry&); void initializeProcessImplicitDefsPass(PassRegistry&); void initializeProfileSummaryInfoWrapperPassPass(PassRegistry&); void initializePromoteLegacyPassPass(PassRegistry&); void initializePruneEHPass(PassRegistry&); void initializeRABasicPass(PassRegistry&); void initializeRAGreedyPass(PassRegistry&); void initializeReachingDefAnalysisPass(PassRegistry&); void initializeReassociateLegacyPassPass(PassRegistry&); void initializeRedundantDbgInstEliminationPass(PassRegistry&); void initializeRegAllocFastPass(PassRegistry&); void initializeRegBankSelectPass(PassRegistry&); void initializeRegToMemPass(PassRegistry&); void initializeRegUsageInfoCollectorPass(PassRegistry&); void initializeRegUsageInfoPropagationPass(PassRegistry&); void initializeRegionInfoPassPass(PassRegistry&); void initializeRegionOnlyPrinterPass(PassRegistry&); void initializeRegionOnlyViewerPass(PassRegistry&); void initializeRegionPrinterPass(PassRegistry&); void initializeRegionViewerPass(PassRegistry&); void initializeRegisterCoalescerPass(PassRegistry&); void initializeRenameIndependentSubregsPass(PassRegistry&); void initializeResetMachineFunctionPass(PassRegistry&); void initializeReversePostOrderFunctionAttrsLegacyPassPass(PassRegistry&); void initializeRewriteStatepointsForGCLegacyPassPass(PassRegistry &); void initializeRewriteSymbolsLegacyPassPass(PassRegistry&); void initializeSCCPLegacyPassPass(PassRegistry&); void initializeSCEVAAWrapperPassPass(PassRegistry&); void initializeSLPVectorizerPass(PassRegistry&); void initializeSROALegacyPassPass(PassRegistry&); void initializeSafeStackLegacyPassPass(PassRegistry&); void initializeSafepointIRVerifierPass(PassRegistry&); void initializeSampleProfileLoaderLegacyPassPass(PassRegistry&); void initializeModuleSanitizerCoverageLegacyPassPass(PassRegistry &); void initializeScalarEvolutionWrapperPassPass(PassRegistry&); void initializeScalarizeMaskedMemIntrinPass(PassRegistry&); void initializeScalarizerLegacyPassPass(PassRegistry&); void initializeScavengerTestPass(PassRegistry&); void initializeScopedNoAliasAAWrapperPassPass(PassRegistry&); void initializeSeparateConstOffsetFromGEPPass(PassRegistry&); void initializeShadowStackGCLoweringPass(PassRegistry&); void initializeShrinkWrapPass(PassRegistry&); void initializeSimpleInlinerPass(PassRegistry&); void initializeSimpleLoopUnswitchLegacyPassPass(PassRegistry&); void initializeSingleLoopExtractorPass(PassRegistry&); void initializeSinkingLegacyPassPass(PassRegistry&); void initializeSjLjEHPreparePass(PassRegistry&); void initializeSlotIndexesPass(PassRegistry&); void initializeSpeculativeExecutionLegacyPassPass(PassRegistry&); void initializeSpillPlacementPass(PassRegistry&); void initializeStackColoringPass(PassRegistry&); void initializeStackMapLivenessPass(PassRegistry&); void initializeStackProtectorPass(PassRegistry&); void initializeStackSafetyGlobalInfoWrapperPassPass(PassRegistry &); void initializeStackSafetyInfoWrapperPassPass(PassRegistry &); void initializeStackSlotColoringPass(PassRegistry&); void initializeStraightLineStrengthReducePass(PassRegistry&); void initializeStripDeadDebugInfoPass(PassRegistry&); void initializeStripDeadPrototypesLegacyPassPass(PassRegistry&); void initializeStripDebugDeclarePass(PassRegistry&); void initializeStripDebugMachineModulePass(PassRegistry &); -void initializeStripGCRelocatesPass(PassRegistry&); +void initializeStripGCRelocatesLegacyPass(PassRegistry &); void initializeStripNonDebugSymbolsPass(PassRegistry&); void initializeStripNonLineTableDebugLegacyPassPass(PassRegistry &); void initializeStripSymbolsPass(PassRegistry&); void initializeStructurizeCFGPass(PassRegistry&); void initializeTailCallElimPass(PassRegistry&); void initializeTailDuplicatePass(PassRegistry&); void initializeTargetLibraryInfoWrapperPassPass(PassRegistry&); void initializeTargetPassConfigPass(PassRegistry&); void initializeTargetTransformInfoWrapperPassPass(PassRegistry&); void initializeThreadSanitizerLegacyPassPass(PassRegistry&); void initializeTwoAddressInstructionPassPass(PassRegistry&); void initializeTypeBasedAAWrapperPassPass(PassRegistry&); void initializeTypePromotionPass(PassRegistry&); void initializeUnifyFunctionExitNodesPass(PassRegistry&); void initializeUnifyLoopExitsPass(PassRegistry &); void initializeUnpackMachineBundlesPass(PassRegistry&); void initializeUnreachableBlockElimLegacyPassPass(PassRegistry&); void initializeUnreachableMachineBlockElimPass(PassRegistry&); void initializeVectorCombineLegacyPassPass(PassRegistry&); void initializeVerifierLegacyPassPass(PassRegistry&); void initializeVirtRegMapPass(PassRegistry&); void initializeVirtRegRewriterPass(PassRegistry&); void initializeWarnMissedTransformationsLegacyPass(PassRegistry &); void initializeWasmEHPreparePass(PassRegistry&); void initializeWholeProgramDevirtPass(PassRegistry&); void initializeWinEHPreparePass(PassRegistry&); void initializeWriteBitcodePassPass(PassRegistry&); void initializeWriteThinLTOBitcodePass(PassRegistry&); void initializeXRayInstrumentationPass(PassRegistry&); } // end namespace llvm #endif // LLVM_INITIALIZEPASSES_H diff --git a/llvm/include/llvm/Transforms/Utils/StripGCRelocates.h b/llvm/include/llvm/Transforms/Utils/StripGCRelocates.h new file mode 100644 index 000000000000..13e6d8ac26a7 --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/StripGCRelocates.h @@ -0,0 +1,25 @@ +//===- StripGCRelocates.h - -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_UTILS_STRIPGCRELOCATES_H +#define LLVM_TRANSFORMS_UTILS_STRIPGCRELOCATES_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class Function; + +class StripGCRelocates : public PassInfoMixin { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_UTILS_STRIPGCRELOCATES_H diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 713c7e9af9b5..92724ed22d02 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -1,2859 +1,2860 @@ //===- Parsing, selection, and construction of pass pipelines -------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file /// /// This file provides the implementation of the PassBuilder based on our /// static pass registry as well as related functionality. It also provides /// helpers to aid in analyzing, debugging, and testing passes and pass /// pipelines. /// //===----------------------------------------------------------------------===// #include "llvm/Passes/PassBuilder.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/AliasAnalysisEvaluator.h" #include "llvm/Analysis/AliasSetTracker.h" #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/BasicAliasAnalysis.h" #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/CFGPrinter.h" #include "llvm/Analysis/CFLAndersAliasAnalysis.h" #include "llvm/Analysis/CFLSteensAliasAnalysis.h" #include "llvm/Analysis/CGSCCPassManager.h" #include "llvm/Analysis/CallGraph.h" #include "llvm/Analysis/DDG.h" #include "llvm/Analysis/Delinearization.h" #include "llvm/Analysis/DemandedBits.h" #include "llvm/Analysis/DependenceAnalysis.h" #include "llvm/Analysis/DominanceFrontier.h" #include "llvm/Analysis/FunctionPropertiesAnalysis.h" #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/IRSimilarityIdentifier.h" #include "llvm/Analysis/IVUsers.h" #include "llvm/Analysis/InlineAdvisor.h" #include "llvm/Analysis/InlineSizeEstimatorAnalysis.h" #include "llvm/Analysis/InstCount.h" #include "llvm/Analysis/LazyCallGraph.h" #include "llvm/Analysis/LazyValueInfo.h" #include "llvm/Analysis/Lint.h" #include "llvm/Analysis/LoopAccessAnalysis.h" #include "llvm/Analysis/LoopCacheAnalysis.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/LoopNestAnalysis.h" #include "llvm/Analysis/MemoryDependenceAnalysis.h" #include "llvm/Analysis/MemorySSA.h" #include "llvm/Analysis/ModuleSummaryAnalysis.h" #include "llvm/Analysis/ObjCARCAliasAnalysis.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/PhiValues.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/Analysis/RegionInfo.h" #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h" #include "llvm/Analysis/ScopedNoAliasAA.h" #include "llvm/Analysis/StackLifetime.h" #include "llvm/Analysis/StackSafetyAnalysis.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/TypeBasedAliasAnalysis.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/IRPrintingPasses.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/SafepointIRVerifier.h" #include "llvm/IR/Verifier.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/FormatVariadic.h" #include "llvm/Support/Regex.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h" #include "llvm/Transforms/Coroutines/CoroCleanup.h" #include "llvm/Transforms/Coroutines/CoroEarly.h" #include "llvm/Transforms/Coroutines/CoroElide.h" #include "llvm/Transforms/Coroutines/CoroSplit.h" #include "llvm/Transforms/HelloNew/HelloWorld.h" #include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/ArgumentPromotion.h" #include "llvm/Transforms/IPO/Attributor.h" #include "llvm/Transforms/IPO/CalledValuePropagation.h" #include "llvm/Transforms/IPO/ConstantMerge.h" #include "llvm/Transforms/IPO/CrossDSOCFI.h" #include "llvm/Transforms/IPO/DeadArgumentElimination.h" #include "llvm/Transforms/IPO/ElimAvailExtern.h" #include "llvm/Transforms/IPO/ForceFunctionAttrs.h" #include "llvm/Transforms/IPO/FunctionAttrs.h" #include "llvm/Transforms/IPO/FunctionImport.h" #include "llvm/Transforms/IPO/GlobalDCE.h" #include "llvm/Transforms/IPO/GlobalOpt.h" #include "llvm/Transforms/IPO/GlobalSplit.h" #include "llvm/Transforms/IPO/HotColdSplitting.h" #include "llvm/Transforms/IPO/InferFunctionAttrs.h" #include "llvm/Transforms/IPO/Inliner.h" #include "llvm/Transforms/IPO/Internalize.h" #include "llvm/Transforms/IPO/LowerTypeTests.h" #include "llvm/Transforms/IPO/MergeFunctions.h" #include "llvm/Transforms/IPO/OpenMPOpt.h" #include "llvm/Transforms/IPO/PartialInlining.h" #include "llvm/Transforms/IPO/SCCP.h" #include "llvm/Transforms/IPO/SampleProfile.h" #include "llvm/Transforms/IPO/StripDeadPrototypes.h" #include "llvm/Transforms/IPO/StripSymbols.h" #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h" #include "llvm/Transforms/IPO/WholeProgramDevirt.h" #include "llvm/Transforms/InstCombine/InstCombine.h" #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Instrumentation/AddressSanitizer.h" #include "llvm/Transforms/Instrumentation/BoundsChecking.h" #include "llvm/Transforms/Instrumentation/CGProfile.h" #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h" #include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h" #include "llvm/Transforms/Instrumentation/GCOVProfiler.h" #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" #include "llvm/Transforms/Instrumentation/InstrOrderFile.h" #include "llvm/Transforms/Instrumentation/InstrProfiling.h" #include "llvm/Transforms/Instrumentation/MemProfiler.h" #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" #include "llvm/Transforms/Instrumentation/PoisonChecking.h" #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h" #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" #include "llvm/Transforms/ObjCARC.h" #include "llvm/Transforms/Scalar/ADCE.h" #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h" #include "llvm/Transforms/Scalar/BDCE.h" #include "llvm/Transforms/Scalar/CallSiteSplitting.h" #include "llvm/Transforms/Scalar/ConstantHoisting.h" #include "llvm/Transforms/Scalar/ConstraintElimination.h" #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h" #include "llvm/Transforms/Scalar/DCE.h" #include "llvm/Transforms/Scalar/DeadStoreElimination.h" #include "llvm/Transforms/Scalar/DivRemPairs.h" #include "llvm/Transforms/Scalar/EarlyCSE.h" #include "llvm/Transforms/Scalar/Float2Int.h" #include "llvm/Transforms/Scalar/GVN.h" #include "llvm/Transforms/Scalar/GuardWidening.h" #include "llvm/Transforms/Scalar/IVUsersPrinter.h" #include "llvm/Transforms/Scalar/IndVarSimplify.h" #include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h" #include "llvm/Transforms/Scalar/InstSimplifyPass.h" #include "llvm/Transforms/Scalar/JumpThreading.h" #include "llvm/Transforms/Scalar/LICM.h" #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h" #include "llvm/Transforms/Scalar/LoopDataPrefetch.h" #include "llvm/Transforms/Scalar/LoopDeletion.h" #include "llvm/Transforms/Scalar/LoopDistribute.h" #include "llvm/Transforms/Scalar/LoopFlatten.h" #include "llvm/Transforms/Scalar/LoopFuse.h" #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h" #include "llvm/Transforms/Scalar/LoopInstSimplify.h" #include "llvm/Transforms/Scalar/LoopLoadElimination.h" #include "llvm/Transforms/Scalar/LoopPassManager.h" #include "llvm/Transforms/Scalar/LoopPredication.h" #include "llvm/Transforms/Scalar/LoopReroll.h" #include "llvm/Transforms/Scalar/LoopRotation.h" #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h" #include "llvm/Transforms/Scalar/LoopSink.h" #include "llvm/Transforms/Scalar/LoopStrengthReduce.h" #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h" #include "llvm/Transforms/Scalar/LoopUnrollPass.h" #include "llvm/Transforms/Scalar/LowerAtomic.h" #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h" #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h" #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h" #include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h" #include "llvm/Transforms/Scalar/LowerWidenableCondition.h" #include "llvm/Transforms/Scalar/MakeGuardsExplicit.h" #include "llvm/Transforms/Scalar/MemCpyOptimizer.h" #include "llvm/Transforms/Scalar/MergeICmps.h" #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h" #include "llvm/Transforms/Scalar/NaryReassociate.h" #include "llvm/Transforms/Scalar/NewGVN.h" #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h" #include "llvm/Transforms/Scalar/Reassociate.h" #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h" #include "llvm/Transforms/Scalar/SCCP.h" #include "llvm/Transforms/Scalar/SROA.h" #include "llvm/Transforms/Scalar/Scalarizer.h" #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h" #include "llvm/Transforms/Scalar/SimplifyCFG.h" #include "llvm/Transforms/Scalar/Sink.h" #include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h" #include "llvm/Transforms/Scalar/SpeculativeExecution.h" #include "llvm/Transforms/Scalar/TailRecursionElimination.h" #include "llvm/Transforms/Scalar/WarnMissedTransforms.h" #include "llvm/Transforms/Utils/AddDiscriminators.h" #include "llvm/Transforms/Utils/AssumeBundleBuilder.h" #include "llvm/Transforms/Utils/BreakCriticalEdges.h" #include "llvm/Transforms/Utils/CanonicalizeAliases.h" #include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h" #include "llvm/Transforms/Utils/EntryExitInstrumenter.h" #include "llvm/Transforms/Utils/InjectTLIMappings.h" #include "llvm/Transforms/Utils/LCSSA.h" #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h" #include "llvm/Transforms/Utils/LoopSimplify.h" #include "llvm/Transforms/Utils/LoopVersioning.h" #include "llvm/Transforms/Utils/LowerInvoke.h" #include "llvm/Transforms/Utils/LowerSwitch.h" #include "llvm/Transforms/Utils/Mem2Reg.h" #include "llvm/Transforms/Utils/MetaRenamer.h" #include "llvm/Transforms/Utils/NameAnonGlobals.h" +#include "llvm/Transforms/Utils/StripGCRelocates.h" #include "llvm/Transforms/Utils/StripNonLineTableDebugInfo.h" #include "llvm/Transforms/Utils/SymbolRewriter.h" #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h" #include "llvm/Transforms/Vectorize/LoopVectorize.h" #include "llvm/Transforms/Vectorize/SLPVectorizer.h" #include "llvm/Transforms/Vectorize/VectorCombine.h" using namespace llvm; static cl::opt MaxDevirtIterations("pm-max-devirt-iterations", cl::ReallyHidden, cl::init(4)); static cl::opt RunPartialInlining("enable-npm-partial-inlining", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Run Partial inlinining pass")); static cl::opt PreInlineThreshold( "npm-preinline-threshold", cl::Hidden, cl::init(75), cl::ZeroOrMore, cl::desc("Control the amount of inlining in pre-instrumentation inliner " "(default = 75)")); static cl::opt RunNewGVN("enable-npm-newgvn", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Run NewGVN instead of GVN")); static cl::opt EnableGVNHoist( "enable-npm-gvn-hoist", cl::init(false), cl::Hidden, cl::desc("Enable the GVN hoisting pass for the new PM (default = off)")); static cl::opt UseInlineAdvisor( "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden, cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"), cl::values(clEnumValN(InliningAdvisorMode::Default, "default", "Heuristics-based inliner version."), clEnumValN(InliningAdvisorMode::Development, "development", "Use development mode (runtime-loadable model)."), clEnumValN(InliningAdvisorMode::Release, "release", "Use release mode (AOT-compiled model)."))); static cl::opt EnableGVNSink( "enable-npm-gvn-sink", cl::init(false), cl::Hidden, cl::desc("Enable the GVN hoisting pass for the new PM (default = off)")); static cl::opt EnableUnrollAndJam( "enable-npm-unroll-and-jam", cl::init(false), cl::Hidden, cl::desc("Enable the Unroll and Jam pass for the new PM (default = off)")); static cl::opt EnableLoopFlatten( "enable-npm-loop-flatten", cl::init(false), cl::Hidden, cl::desc("Enable the Loop flattening pass for the new PM (default = off)")); static cl::opt EnableSyntheticCounts( "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Run synthetic function entry count generation " "pass")); static const Regex DefaultAliasRegex( "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$"); // This option is used in simplifying testing SampleFDO optimizations for // profile loading. static cl::opt EnableCHR("enable-chr-npm", cl::init(true), cl::Hidden, cl::desc("Enable control height reduction optimization (CHR)")); /// Flag to enable inline deferral during PGO. static cl::opt EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true), cl::Hidden, cl::desc("Enable inline deferral during PGO")); static cl::opt EnableMemProfiler("enable-mem-prof", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Enable memory profiler")); PipelineTuningOptions::PipelineTuningOptions() { LoopInterleaving = true; LoopVectorization = true; SLPVectorization = false; LoopUnrolling = true; ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll; Coroutines = false; LicmMssaOptCap = SetLicmMssaOptCap; LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap; CallGraphProfile = true; } extern cl::opt EnableConstraintElimination; extern cl::opt EnableHotColdSplit; extern cl::opt EnableOrderFileInstrumentation; extern cl::opt FlattenedProfileUsed; extern cl::opt AttributorRun; extern cl::opt EnableKnowledgeRetention; extern cl::opt EnableMatrix; const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O0 = { /*SpeedLevel*/ 0, /*SizeLevel*/ 0}; const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O1 = { /*SpeedLevel*/ 1, /*SizeLevel*/ 0}; const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O2 = { /*SpeedLevel*/ 2, /*SizeLevel*/ 0}; const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O3 = { /*SpeedLevel*/ 3, /*SizeLevel*/ 0}; const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::Os = { /*SpeedLevel*/ 2, /*SizeLevel*/ 1}; const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::Oz = { /*SpeedLevel*/ 2, /*SizeLevel*/ 2}; namespace { // The following passes/analyses have custom names, otherwise their name will // include `(anonymous namespace)`. These are special since they are only for // testing purposes and don't live in a header file. /// No-op module pass which does nothing. struct NoOpModulePass : PassInfoMixin { PreservedAnalyses run(Module &M, ModuleAnalysisManager &) { return PreservedAnalyses::all(); } static StringRef name() { return "NoOpModulePass"; } }; /// No-op module analysis. class NoOpModuleAnalysis : public AnalysisInfoMixin { friend AnalysisInfoMixin; static AnalysisKey Key; public: struct Result {}; Result run(Module &, ModuleAnalysisManager &) { return Result(); } static StringRef name() { return "NoOpModuleAnalysis"; } }; /// No-op CGSCC pass which does nothing. struct NoOpCGSCCPass : PassInfoMixin { PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &, LazyCallGraph &, CGSCCUpdateResult &UR) { return PreservedAnalyses::all(); } static StringRef name() { return "NoOpCGSCCPass"; } }; /// No-op CGSCC analysis. class NoOpCGSCCAnalysis : public AnalysisInfoMixin { friend AnalysisInfoMixin; static AnalysisKey Key; public: struct Result {}; Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) { return Result(); } static StringRef name() { return "NoOpCGSCCAnalysis"; } }; /// No-op function pass which does nothing. struct NoOpFunctionPass : PassInfoMixin { PreservedAnalyses run(Function &F, FunctionAnalysisManager &) { return PreservedAnalyses::all(); } static StringRef name() { return "NoOpFunctionPass"; } }; /// No-op function analysis. class NoOpFunctionAnalysis : public AnalysisInfoMixin { friend AnalysisInfoMixin; static AnalysisKey Key; public: struct Result {}; Result run(Function &, FunctionAnalysisManager &) { return Result(); } static StringRef name() { return "NoOpFunctionAnalysis"; } }; /// No-op loop pass which does nothing. struct NoOpLoopPass : PassInfoMixin { PreservedAnalyses run(Loop &L, LoopAnalysisManager &, LoopStandardAnalysisResults &, LPMUpdater &) { return PreservedAnalyses::all(); } static StringRef name() { return "NoOpLoopPass"; } }; /// No-op loop analysis. class NoOpLoopAnalysis : public AnalysisInfoMixin { friend AnalysisInfoMixin; static AnalysisKey Key; public: struct Result {}; Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) { return Result(); } static StringRef name() { return "NoOpLoopAnalysis"; } }; AnalysisKey NoOpModuleAnalysis::Key; AnalysisKey NoOpCGSCCAnalysis::Key; AnalysisKey NoOpFunctionAnalysis::Key; AnalysisKey NoOpLoopAnalysis::Key; } // namespace void PassBuilder::invokePeepholeEPCallbacks( FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { for (auto &C : PeepholeEPCallbacks) C(FPM, Level); } void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) { #define MODULE_ANALYSIS(NAME, CREATE_PASS) \ MAM.registerPass([&] { return CREATE_PASS; }); #include "PassRegistry.def" for (auto &C : ModuleAnalysisRegistrationCallbacks) C(MAM); } void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) { #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \ CGAM.registerPass([&] { return CREATE_PASS; }); #include "PassRegistry.def" for (auto &C : CGSCCAnalysisRegistrationCallbacks) C(CGAM); } void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) { #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ FAM.registerPass([&] { return CREATE_PASS; }); #include "PassRegistry.def" for (auto &C : FunctionAnalysisRegistrationCallbacks) C(FAM); } void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) { #define LOOP_ANALYSIS(NAME, CREATE_PASS) \ LAM.registerPass([&] { return CREATE_PASS; }); #include "PassRegistry.def" for (auto &C : LoopAnalysisRegistrationCallbacks) C(LAM); } // TODO: Investigate the cost/benefit of tail call elimination on debugging. FunctionPassManager PassBuilder::buildO1FunctionSimplificationPipeline( OptimizationLevel Level, ThinLTOPhase Phase, bool DebugLogging) { FunctionPassManager FPM(DebugLogging); // Form SSA out of local memory accesses after breaking apart aggregates into // scalars. FPM.addPass(SROA()); // Catch trivial redundancies FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */)); // Hoisting of scalars and load expressions. FPM.addPass(SimplifyCFGPass()); FPM.addPass(InstCombinePass()); FPM.addPass(LibCallsShrinkWrapPass()); invokePeepholeEPCallbacks(FPM, Level); FPM.addPass(SimplifyCFGPass()); // Form canonically associated expression trees, and simplify the trees using // basic mathematical properties. For example, this will form (nearly) // minimal multiplication trees. FPM.addPass(ReassociatePass()); // Add the primary loop simplification pipeline. // FIXME: Currently this is split into two loop pass pipelines because we run // some function passes in between them. These can and should be removed // and/or replaced by scheduling the loop pass equivalents in the correct // positions. But those equivalent passes aren't powerful enough yet. // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to // fully replace `SimplifyCFGPass`, and the closest to the other we have is // `LoopInstSimplify`. LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging); // Simplify the loop body. We do this initially to clean up after other loop // passes run, either when iterating on a loop or on inner loops with // implications on the outer loop. LPM1.addPass(LoopInstSimplifyPass()); LPM1.addPass(LoopSimplifyCFGPass()); LPM1.addPass(LoopRotatePass(/* Disable header duplication */ true)); // TODO: Investigate promotion cap for O1. LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap)); LPM1.addPass(SimpleLoopUnswitchPass()); LPM2.addPass(IndVarSimplifyPass()); LPM2.addPass(LoopIdiomRecognizePass()); for (auto &C : LateLoopOptimizationsEPCallbacks) C(LPM2, Level); LPM2.addPass(LoopDeletionPass()); if (EnableLoopFlatten) LPM2.addPass(LoopFlattenPass()); // Do not enable unrolling in PreLinkThinLTO phase during sample PGO // because it changes IR to makes profile annotation in back compile // inaccurate. The normal unroller doesn't pay attention to forced full unroll // attributes so we need to make sure and allow the full unroll pass to pay // attention to it. if (Phase != ThinLTOPhase::PreLink || !PGOOpt || PGOOpt->Action != PGOOptions::SampleUse) LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(), /* OnlyWhenForced= */ !PTO.LoopUnrolling, PTO.ForgetAllSCEVInLoopUnroll)); for (auto &C : LoopOptimizerEndEPCallbacks) C(LPM2, Level); // We provide the opt remark emitter pass for LICM to use. We only need to do // this once as it is immutable. FPM.addPass( RequireAnalysisPass()); FPM.addPass(createFunctionToLoopPassAdaptor( std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging)); FPM.addPass(SimplifyCFGPass()); FPM.addPass(InstCombinePass()); // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA. // *All* loop passes must preserve it, in order to be able to use it. FPM.addPass(createFunctionToLoopPassAdaptor( std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false, DebugLogging)); // Delete small array after loop unroll. FPM.addPass(SROA()); // Specially optimize memory movement as it doesn't look like dataflow in SSA. FPM.addPass(MemCpyOptPass()); // Sparse conditional constant propagation. // FIXME: It isn't clear why we do this *after* loop passes rather than // before... FPM.addPass(SCCPPass()); // Delete dead bit computations (instcombine runs after to fold away the dead // computations, and then ADCE will run later to exploit any new DCE // opportunities that creates). FPM.addPass(BDCEPass()); // Run instcombine after redundancy and dead bit elimination to exploit // opportunities opened up by them. FPM.addPass(InstCombinePass()); invokePeepholeEPCallbacks(FPM, Level); if (PTO.Coroutines) FPM.addPass(CoroElidePass()); for (auto &C : ScalarOptimizerLateEPCallbacks) C(FPM, Level); // Finally, do an expensive DCE pass to catch all the dead code exposed by // the simplifications and basic cleanup after all the simplifications. // TODO: Investigate if this is too expensive. FPM.addPass(ADCEPass()); FPM.addPass(SimplifyCFGPass()); FPM.addPass(InstCombinePass()); invokePeepholeEPCallbacks(FPM, Level); return FPM; } FunctionPassManager PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, ThinLTOPhase Phase, bool DebugLogging) { assert(Level != OptimizationLevel::O0 && "Must request optimizations!"); // The O1 pipeline has a separate pipeline creation function to simplify // construction readability. if (Level.getSpeedupLevel() == 1) return buildO1FunctionSimplificationPipeline(Level, Phase, DebugLogging); FunctionPassManager FPM(DebugLogging); // Form SSA out of local memory accesses after breaking apart aggregates into // scalars. FPM.addPass(SROA()); // Catch trivial redundancies FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */)); if (EnableKnowledgeRetention) FPM.addPass(AssumeSimplifyPass()); // Hoisting of scalars and load expressions. if (EnableGVNHoist) FPM.addPass(GVNHoistPass()); // Global value numbering based sinking. if (EnableGVNSink) { FPM.addPass(GVNSinkPass()); FPM.addPass(SimplifyCFGPass()); } if (EnableConstraintElimination) FPM.addPass(ConstraintEliminationPass()); // Speculative execution if the target has divergent branches; otherwise nop. FPM.addPass(SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true)); // Optimize based on known information about branches, and cleanup afterward. FPM.addPass(JumpThreadingPass()); FPM.addPass(CorrelatedValuePropagationPass()); FPM.addPass(SimplifyCFGPass()); if (Level == OptimizationLevel::O3) FPM.addPass(AggressiveInstCombinePass()); FPM.addPass(InstCombinePass()); if (!Level.isOptimizingForSize()) FPM.addPass(LibCallsShrinkWrapPass()); invokePeepholeEPCallbacks(FPM, Level); // For PGO use pipeline, try to optimize memory intrinsics such as memcpy // using the size value profile. Don't perform this when optimizing for size. if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse && !Level.isOptimizingForSize()) FPM.addPass(PGOMemOPSizeOpt()); FPM.addPass(TailCallElimPass()); FPM.addPass(SimplifyCFGPass()); // Form canonically associated expression trees, and simplify the trees using // basic mathematical properties. For example, this will form (nearly) // minimal multiplication trees. FPM.addPass(ReassociatePass()); // Add the primary loop simplification pipeline. // FIXME: Currently this is split into two loop pass pipelines because we run // some function passes in between them. These can and should be removed // and/or replaced by scheduling the loop pass equivalents in the correct // positions. But those equivalent passes aren't powerful enough yet. // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to // fully replace `SimplifyCFGPass`, and the closest to the other we have is // `LoopInstSimplify`. LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging); // Simplify the loop body. We do this initially to clean up after other loop // passes run, either when iterating on a loop or on inner loops with // implications on the outer loop. LPM1.addPass(LoopInstSimplifyPass()); LPM1.addPass(LoopSimplifyCFGPass()); // Rotate Loop - disable header duplication at -Oz LPM1.addPass(LoopRotatePass(Level != OptimizationLevel::Oz)); // TODO: Investigate promotion cap for O1. LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap)); LPM1.addPass(SimpleLoopUnswitchPass()); LPM2.addPass(IndVarSimplifyPass()); LPM2.addPass(LoopIdiomRecognizePass()); for (auto &C : LateLoopOptimizationsEPCallbacks) C(LPM2, Level); LPM2.addPass(LoopDeletionPass()); // Do not enable unrolling in PreLinkThinLTO phase during sample PGO // because it changes IR to makes profile annotation in back compile // inaccurate. The normal unroller doesn't pay attention to forced full unroll // attributes so we need to make sure and allow the full unroll pass to pay // attention to it. if (Phase != ThinLTOPhase::PreLink || !PGOOpt || PGOOpt->Action != PGOOptions::SampleUse) LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(), /* OnlyWhenForced= */ !PTO.LoopUnrolling, PTO.ForgetAllSCEVInLoopUnroll)); for (auto &C : LoopOptimizerEndEPCallbacks) C(LPM2, Level); // We provide the opt remark emitter pass for LICM to use. We only need to do // this once as it is immutable. FPM.addPass( RequireAnalysisPass()); FPM.addPass(createFunctionToLoopPassAdaptor( std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging)); FPM.addPass(SimplifyCFGPass()); FPM.addPass(InstCombinePass()); // The loop passes in LPM2 (IndVarSimplifyPass, LoopIdiomRecognizePass, // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA. // *All* loop passes must preserve it, in order to be able to use it. FPM.addPass(createFunctionToLoopPassAdaptor( std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false, DebugLogging)); // Delete small array after loop unroll. FPM.addPass(SROA()); // Eliminate redundancies. FPM.addPass(MergedLoadStoreMotionPass()); if (RunNewGVN) FPM.addPass(NewGVNPass()); else FPM.addPass(GVN()); // Specially optimize memory movement as it doesn't look like dataflow in SSA. FPM.addPass(MemCpyOptPass()); // Sparse conditional constant propagation. // FIXME: It isn't clear why we do this *after* loop passes rather than // before... FPM.addPass(SCCPPass()); // Delete dead bit computations (instcombine runs after to fold away the dead // computations, and then ADCE will run later to exploit any new DCE // opportunities that creates). FPM.addPass(BDCEPass()); // Run instcombine after redundancy and dead bit elimination to exploit // opportunities opened up by them. FPM.addPass(InstCombinePass()); invokePeepholeEPCallbacks(FPM, Level); // Re-consider control flow based optimizations after redundancy elimination, // redo DCE, etc. FPM.addPass(JumpThreadingPass()); FPM.addPass(CorrelatedValuePropagationPass()); FPM.addPass(DSEPass()); FPM.addPass(createFunctionToLoopPassAdaptor( LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging)); if (PTO.Coroutines) FPM.addPass(CoroElidePass()); for (auto &C : ScalarOptimizerLateEPCallbacks) C(FPM, Level); // Finally, do an expensive DCE pass to catch all the dead code exposed by // the simplifications and basic cleanup after all the simplifications. // TODO: Investigate if this is too expensive. FPM.addPass(ADCEPass()); FPM.addPass(SimplifyCFGPass()); FPM.addPass(InstCombinePass()); invokePeepholeEPCallbacks(FPM, Level); if (EnableCHR && Level == OptimizationLevel::O3 && PGOOpt && (PGOOpt->Action == PGOOptions::IRUse || PGOOpt->Action == PGOOptions::SampleUse)) FPM.addPass(ControlHeightReductionPass()); return FPM; } void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging, PassBuilder::OptimizationLevel Level, bool RunProfileGen, bool IsCS, std::string ProfileFile, std::string ProfileRemappingFile) { assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!"); // Generally running simplification passes and the inliner with an high // threshold results in smaller executables, but there may be cases where // the size grows, so let's be conservative here and skip this simplification // at -Os/Oz. We will not do this inline for context sensistive PGO (when // IsCS is true). if (!Level.isOptimizingForSize() && !IsCS) { InlineParams IP; IP.DefaultThreshold = PreInlineThreshold; // FIXME: The hint threshold has the same value used by the regular inliner. // This should probably be lowered after performance testing. // FIXME: this comment is cargo culted from the old pass manager, revisit). IP.HintThreshold = 325; ModuleInlinerWrapperPass MIWP(IP, DebugLogging); CGSCCPassManager &CGPipeline = MIWP.getPM(); FunctionPassManager FPM; FPM.addPass(SROA()); FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies. FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks. FPM.addPass(InstCombinePass()); // Combine silly sequences. invokePeepholeEPCallbacks(FPM, Level); CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM))); MPM.addPass(std::move(MIWP)); // Delete anything that is now dead to make sure that we don't instrument // dead code. Instrumentation can end up keeping dead code around and // dramatically increase code size. MPM.addPass(GlobalDCEPass()); } if (!RunProfileGen) { assert(!ProfileFile.empty() && "Profile use expecting a profile file!"); MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS)); // Cache ProfileSummaryAnalysis once to avoid the potential need to insert // RequireAnalysisPass for PSI before subsequent non-module passes. MPM.addPass(RequireAnalysisPass()); return; } // Perform PGO instrumentation. MPM.addPass(PGOInstrumentationGen(IsCS)); FunctionPassManager FPM; FPM.addPass(createFunctionToLoopPassAdaptor( LoopRotatePass(), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/false, DebugLogging)); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); // Add the profile lowering pass. InstrProfOptions Options; if (!ProfileFile.empty()) Options.InstrProfileOutput = ProfileFile; // Do counter promotion at Level greater than O0. Options.DoCounterPromotion = true; Options.UseBFIInPromotion = IsCS; MPM.addPass(InstrProfiling(Options, IsCS)); } void PassBuilder::addPGOInstrPassesForO0(ModulePassManager &MPM, bool DebugLogging, bool RunProfileGen, bool IsCS, std::string ProfileFile, std::string ProfileRemappingFile) { if (!RunProfileGen) { assert(!ProfileFile.empty() && "Profile use expecting a profile file!"); MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS)); // Cache ProfileSummaryAnalysis once to avoid the potential need to insert // RequireAnalysisPass for PSI before subsequent non-module passes. MPM.addPass(RequireAnalysisPass()); return; } // Perform PGO instrumentation. MPM.addPass(PGOInstrumentationGen(IsCS)); // Add the profile lowering pass. InstrProfOptions Options; if (!ProfileFile.empty()) Options.InstrProfileOutput = ProfileFile; // Do not do counter promotion at O0. Options.DoCounterPromotion = false; Options.UseBFIInPromotion = IsCS; MPM.addPass(InstrProfiling(Options, IsCS)); } static InlineParams getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) { return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel()); } ModuleInlinerWrapperPass PassBuilder::buildInlinerPipeline(OptimizationLevel Level, ThinLTOPhase Phase, bool DebugLogging) { InlineParams IP = getInlineParamsFromOptLevel(Level); if (Phase == PassBuilder::ThinLTOPhase::PreLink && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) IP.HotCallSiteThreshold = 0; if (PGOOpt) IP.EnableDeferral = EnablePGOInlineDeferral; ModuleInlinerWrapperPass MIWP(IP, DebugLogging, UseInlineAdvisor, MaxDevirtIterations); // Require the GlobalsAA analysis for the module so we can query it within // the CGSCC pipeline. MIWP.addRequiredModuleAnalysis(); // Require the ProfileSummaryAnalysis for the module so we can query it within // the inliner pass. MIWP.addRequiredModuleAnalysis(); // Now begin the main postorder CGSCC pipeline. // FIXME: The current CGSCC pipeline has its origins in the legacy pass // manager and trying to emulate its precise behavior. Much of this doesn't // make a lot of sense and we should revisit the core CGSCC structure. CGSCCPassManager &MainCGPipeline = MIWP.getPM(); // Note: historically, the PruneEH pass was run first to deduce nounwind and // generally clean up exception handling overhead. It isn't clear this is // valuable as the inliner doesn't currently care whether it is inlining an // invoke or a call. if (AttributorRun & AttributorRunOption::CGSCC) MainCGPipeline.addPass(AttributorCGSCCPass()); if (PTO.Coroutines) MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0)); // Now deduce any function attributes based in the current code. MainCGPipeline.addPass(PostOrderFunctionAttrsPass()); // When at O3 add argument promotion to the pass pipeline. // FIXME: It isn't at all clear why this should be limited to O3. if (Level == OptimizationLevel::O3) MainCGPipeline.addPass(ArgumentPromotionPass()); // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if // there are no OpenMP runtime calls present in the module. if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3) MainCGPipeline.addPass(OpenMPOptPass()); // Lastly, add the core function simplification pipeline nested inside the // CGSCC walk. MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor( buildFunctionSimplificationPipeline(Level, Phase, DebugLogging))); for (auto &C : CGSCCOptimizerLateEPCallbacks) C(MainCGPipeline, Level); return MIWP; } ModulePassManager PassBuilder::buildModuleSimplificationPipeline( OptimizationLevel Level, ThinLTOPhase Phase, bool DebugLogging) { ModulePassManager MPM(DebugLogging); bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse); // In ThinLTO mode, when flattened profile is used, all the available // profile information will be annotated in PreLink phase so there is // no need to load the profile again in PostLink. bool LoadSampleProfile = HasSampleProfile && !(FlattenedProfileUsed && Phase == ThinLTOPhase::PostLink); // During the ThinLTO backend phase we perform early indirect call promotion // here, before globalopt. Otherwise imported available_externally functions // look unreferenced and are removed. If we are going to load the sample // profile then defer until later. // TODO: See if we can move later and consolidate with the location where // we perform ICP when we are loading a sample profile. // TODO: We pass HasSampleProfile (whether there was a sample profile file // passed to the compile) to the SamplePGO flag of ICP. This is used to // determine whether the new direct calls are annotated with prof metadata. // Ideally this should be determined from whether the IR is annotated with // sample profile, and not whether the a sample profile was provided on the // command line. E.g. for flattened profiles where we will not be reloading // the sample profile in the ThinLTO backend, we ideally shouldn't have to // provide the sample profile file. if (Phase == ThinLTOPhase::PostLink && !LoadSampleProfile) MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile)); // Do basic inference of function attributes from known properties of system // libraries and other oracles. MPM.addPass(InferFunctionAttrsPass()); // Create an early function pass manager to cleanup the output of the // frontend. FunctionPassManager EarlyFPM(DebugLogging); EarlyFPM.addPass(SimplifyCFGPass()); EarlyFPM.addPass(SROA()); EarlyFPM.addPass(EarlyCSEPass()); EarlyFPM.addPass(LowerExpectIntrinsicPass()); if (PTO.Coroutines) EarlyFPM.addPass(CoroEarlyPass()); if (Level == OptimizationLevel::O3) EarlyFPM.addPass(CallSiteSplittingPass()); // In SamplePGO ThinLTO backend, we need instcombine before profile annotation // to convert bitcast to direct calls so that they can be inlined during the // profile annotation prepration step. // More details about SamplePGO design can be found in: // https://research.google.com/pubs/pub45290.html // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured. if (LoadSampleProfile) EarlyFPM.addPass(InstCombinePass()); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM))); if (LoadSampleProfile) { // Annotate sample profile right after early FPM to ensure freshness of // the debug info. MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile, Phase == ThinLTOPhase::PreLink)); // Cache ProfileSummaryAnalysis once to avoid the potential need to insert // RequireAnalysisPass for PSI before subsequent non-module passes. MPM.addPass(RequireAnalysisPass()); // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard // for the profile annotation to be accurate in the ThinLTO backend. if (Phase != ThinLTOPhase::PreLink) // We perform early indirect call promotion here, before globalopt. // This is important for the ThinLTO backend phase because otherwise // imported available_externally functions look unreferenced and are // removed. MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink, true /* SamplePGO */)); } if (AttributorRun & AttributorRunOption::MODULE) MPM.addPass(AttributorPass()); // Lower type metadata and the type.test intrinsic in the ThinLTO // post link pipeline after ICP. This is to enable usage of the type // tests in ICP sequences. if (Phase == ThinLTOPhase::PostLink) MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true)); // Interprocedural constant propagation now that basic cleanup has occurred // and prior to optimizing globals. // FIXME: This position in the pipeline hasn't been carefully considered in // years, it should be re-analyzed. MPM.addPass(IPSCCPPass()); // Attach metadata to indirect call sites indicating the set of functions // they may target at run-time. This should follow IPSCCP. MPM.addPass(CalledValuePropagationPass()); // Optimize globals to try and fold them into constants. MPM.addPass(GlobalOptPass()); // Promote any localized globals to SSA registers. // FIXME: Should this instead by a run of SROA? // FIXME: We should probably run instcombine and simplify-cfg afterward to // delete control flows that are dead once globals have been folded to // constants. MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass())); // Remove any dead arguments exposed by cleanups and constant folding // globals. MPM.addPass(DeadArgumentEliminationPass()); // Create a small function pass pipeline to cleanup after all the global // optimizations. FunctionPassManager GlobalCleanupPM(DebugLogging); GlobalCleanupPM.addPass(InstCombinePass()); invokePeepholeEPCallbacks(GlobalCleanupPM, Level); GlobalCleanupPM.addPass(SimplifyCFGPass()); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM))); // Add all the requested passes for instrumentation PGO, if requested. if (PGOOpt && Phase != ThinLTOPhase::PostLink && (PGOOpt->Action == PGOOptions::IRInstr || PGOOpt->Action == PGOOptions::IRUse)) { addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr, /* IsCS */ false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile); MPM.addPass(PGOIndirectCallPromotion(false, false)); } if (PGOOpt && Phase != ThinLTOPhase::PostLink && PGOOpt->CSAction == PGOOptions::CSIRInstr) MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile)); // Synthesize function entry counts for non-PGO compilation. if (EnableSyntheticCounts && !PGOOpt) MPM.addPass(SyntheticCountsPropagation()); MPM.addPass(buildInlinerPipeline(Level, Phase, DebugLogging)); if (EnableMemProfiler && Phase != ThinLTOPhase::PreLink) { MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass())); MPM.addPass(ModuleMemProfilerPass()); } return MPM; } ModulePassManager PassBuilder::buildModuleOptimizationPipeline( OptimizationLevel Level, bool DebugLogging, bool LTOPreLink) { ModulePassManager MPM(DebugLogging); // Optimize globals now that the module is fully simplified. MPM.addPass(GlobalOptPass()); MPM.addPass(GlobalDCEPass()); // Run partial inlining pass to partially inline functions that have // large bodies. if (RunPartialInlining) MPM.addPass(PartialInlinerPass()); // Remove avail extern fns and globals definitions since we aren't compiling // an object file for later LTO. For LTO we want to preserve these so they // are eligible for inlining at link-time. Note if they are unreferenced they // will be removed by GlobalDCE later, so this only impacts referenced // available externally globals. Eventually they will be suppressed during // codegen, but eliminating here enables more opportunity for GlobalDCE as it // may make globals referenced by available external functions dead and saves // running remaining passes on the eliminated functions. These should be // preserved during prelinking for link-time inlining decisions. if (!LTOPreLink) MPM.addPass(EliminateAvailableExternallyPass()); if (EnableOrderFileInstrumentation) MPM.addPass(InstrOrderFilePass()); // Do RPO function attribute inference across the module to forward-propagate // attributes where applicable. // FIXME: Is this really an optimization rather than a canonicalization? MPM.addPass(ReversePostOrderFunctionAttrsPass()); // Do a post inline PGO instrumentation and use pass. This is a context // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as // cross-module inline has not been done yet. The context sensitive // instrumentation is after all the inlines are done. if (!LTOPreLink && PGOOpt) { if (PGOOpt->CSAction == PGOOptions::CSIRInstr) addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true, /* IsCS */ true, PGOOpt->CSProfileGenFile, PGOOpt->ProfileRemappingFile); else if (PGOOpt->CSAction == PGOOptions::CSIRUse) addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false, /* IsCS */ true, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile); } // Re-require GloblasAA here prior to function passes. This is particularly // useful as the above will have inlined, DCE'ed, and function-attr // propagated everything. We should at this point have a reasonably minimal // and richly annotated call graph. By computing aliasing and mod/ref // information for all local globals here, the late loop passes and notably // the vectorizer will be able to use them to help recognize vectorizable // memory operations. MPM.addPass(RequireAnalysisPass()); FunctionPassManager OptimizePM(DebugLogging); OptimizePM.addPass(Float2IntPass()); OptimizePM.addPass(LowerConstantIntrinsicsPass()); if (EnableMatrix) { OptimizePM.addPass(LowerMatrixIntrinsicsPass()); OptimizePM.addPass(EarlyCSEPass()); } // FIXME: We need to run some loop optimizations to re-rotate loops after // simplify-cfg and others undo their rotation. // Optimize the loop execution. These passes operate on entire loop nests // rather than on each loop in an inside-out manner, and so they are actually // function passes. for (auto &C : VectorizerStartEPCallbacks) C(OptimizePM, Level); // First rotate loops that may have been un-rotated by prior passes. OptimizePM.addPass(createFunctionToLoopPassAdaptor( LoopRotatePass(), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/false, DebugLogging)); // Distribute loops to allow partial vectorization. I.e. isolate dependences // into separate loop that would otherwise inhibit vectorization. This is // currently only performed for loops marked with the metadata // llvm.loop.distribute=true or when -enable-loop-distribute is specified. OptimizePM.addPass(LoopDistributePass()); // Populates the VFABI attribute with the scalar-to-vector mappings // from the TargetLibraryInfo. OptimizePM.addPass(InjectTLIMappings()); // Now run the core loop vectorizer. OptimizePM.addPass(LoopVectorizePass( LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization))); // Eliminate loads by forwarding stores from the previous iteration to loads // of the current iteration. OptimizePM.addPass(LoopLoadEliminationPass()); // Cleanup after the loop optimization passes. OptimizePM.addPass(InstCombinePass()); // Now that we've formed fast to execute loop structures, we do further // optimizations. These are run afterward as they might block doing complex // analyses and transforms such as what are needed for loop vectorization. // Cleanup after loop vectorization, etc. Simplification passes like CVP and // GVN, loop transforms, and others have already run, so it's now better to // convert to more optimized IR using more aggressive simplify CFG options. // The extra sinking transform can create larger basic blocks, so do this // before SLP vectorization. // FIXME: study whether hoisting and/or sinking of common instructions should // be delayed until after SLP vectorizer. OptimizePM.addPass(SimplifyCFGPass(SimplifyCFGOptions() .forwardSwitchCondToPhi(true) .convertSwitchToLookupTable(true) .needCanonicalLoops(false) .hoistCommonInsts(true) .sinkCommonInsts(true))); // Optimize parallel scalar instruction chains into SIMD instructions. if (PTO.SLPVectorization) OptimizePM.addPass(SLPVectorizerPass()); // Enhance/cleanup vector code. OptimizePM.addPass(VectorCombinePass()); OptimizePM.addPass(InstCombinePass()); // Unroll small loops to hide loop backedge latency and saturate any parallel // execution resources of an out-of-order processor. We also then need to // clean up redundancies and loop invariant code. // FIXME: It would be really good to use a loop-integrated instruction // combiner for cleanup here so that the unrolling and LICM can be pipelined // across the loop nests. // We do UnrollAndJam in a separate LPM to ensure it happens before unroll if (EnableUnrollAndJam && PTO.LoopUnrolling) { OptimizePM.addPass(LoopUnrollAndJamPass(Level.getSpeedupLevel())); } OptimizePM.addPass(LoopUnrollPass(LoopUnrollOptions( Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling, PTO.ForgetAllSCEVInLoopUnroll))); OptimizePM.addPass(WarnMissedTransformationsPass()); OptimizePM.addPass(InstCombinePass()); OptimizePM.addPass(RequireAnalysisPass()); OptimizePM.addPass(createFunctionToLoopPassAdaptor( LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging)); // Now that we've vectorized and unrolled loops, we may have more refined // alignment information, try to re-derive it here. OptimizePM.addPass(AlignmentFromAssumptionsPass()); // Split out cold code. Splitting is done late to avoid hiding context from // other optimizations and inadvertently regressing performance. The tradeoff // is that this has a higher code size cost than splitting early. if (EnableHotColdSplit && !LTOPreLink) MPM.addPass(HotColdSplittingPass()); // LoopSink pass sinks instructions hoisted by LICM, which serves as a // canonicalization pass that enables other optimizations. As a result, // LoopSink pass needs to be a very late IR pass to avoid undoing LICM // result too early. OptimizePM.addPass(LoopSinkPass()); // And finally clean up LCSSA form before generating code. OptimizePM.addPass(InstSimplifyPass()); // This hoists/decomposes div/rem ops. It should run after other sink/hoist // passes to avoid re-sinking, but before SimplifyCFG because it can allow // flattening of blocks. OptimizePM.addPass(DivRemPairsPass()); // LoopSink (and other loop passes since the last simplifyCFG) might have // resulted in single-entry-single-exit or empty blocks. Clean up the CFG. OptimizePM.addPass(SimplifyCFGPass()); // Optimize PHIs by speculating around them when profitable. Note that this // pass needs to be run after any PRE or similar pass as it is essentially // inserting redundancies into the program. This even includes SimplifyCFG. OptimizePM.addPass(SpeculateAroundPHIsPass()); if (PTO.Coroutines) OptimizePM.addPass(CoroCleanupPass()); // Add the core optimizing pipeline. MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM))); for (auto &C : OptimizerLastEPCallbacks) C(MPM, Level); if (PTO.CallGraphProfile) MPM.addPass(CGProfilePass()); // Now we need to do some global optimization transforms. // FIXME: It would seem like these should come first in the optimization // pipeline and maybe be the bottom of the canonicalization pipeline? Weird // ordering here. MPM.addPass(GlobalDCEPass()); MPM.addPass(ConstantMergePass()); return MPM; } ModulePassManager PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level, bool DebugLogging, bool LTOPreLink) { assert(Level != OptimizationLevel::O0 && "Must request optimizations for the default pipeline!"); ModulePassManager MPM(DebugLogging); // Force any function attributes we want the rest of the pipeline to observe. MPM.addPass(ForceFunctionAttrsPass()); // Apply module pipeline start EP callback. for (auto &C : PipelineStartEPCallbacks) C(MPM); if (PGOOpt && PGOOpt->SamplePGOSupport) MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); // Add the core simplification pipeline. MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::None, DebugLogging)); // Now add the optimization pipeline. MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging, LTOPreLink)); return MPM; } ModulePassManager PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level, bool DebugLogging) { assert(Level != OptimizationLevel::O0 && "Must request optimizations for the default pipeline!"); ModulePassManager MPM(DebugLogging); // Force any function attributes we want the rest of the pipeline to observe. MPM.addPass(ForceFunctionAttrsPass()); if (PGOOpt && PGOOpt->SamplePGOSupport) MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); // Apply module pipeline start EP callback. for (auto &C : PipelineStartEPCallbacks) C(MPM); // If we are planning to perform ThinLTO later, we don't bloat the code with // unrolling/vectorization/... now. Just simplify the module as much as we // can. MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PreLink, DebugLogging)); // Run partial inlining pass to partially inline functions that have // large bodies. // FIXME: It isn't clear whether this is really the right place to run this // in ThinLTO. Because there is another canonicalization and simplification // phase that will run after the thin link, running this here ends up with // less information than will be available later and it may grow functions in // ways that aren't beneficial. if (RunPartialInlining) MPM.addPass(PartialInlinerPass()); // Reduce the size of the IR as much as possible. MPM.addPass(GlobalOptPass()); // Module simplification splits coroutines, but does not fully clean up // coroutine intrinsics. To ensure ThinLTO optimization passes don't trip up // on these, we schedule the cleanup here. if (PTO.Coroutines) MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass())); return MPM; } ModulePassManager PassBuilder::buildThinLTODefaultPipeline( OptimizationLevel Level, bool DebugLogging, const ModuleSummaryIndex *ImportSummary) { ModulePassManager MPM(DebugLogging); if (ImportSummary) { // These passes import type identifier resolutions for whole-program // devirtualization and CFI. They must run early because other passes may // disturb the specific instruction patterns that these passes look for, // creating dependencies on resolutions that may not appear in the summary. // // For example, GVN may transform the pattern assume(type.test) appearing in // two basic blocks into assume(phi(type.test, type.test)), which would // transform a dependency on a WPD resolution into a dependency on a type // identifier resolution for CFI. // // Also, WPD has access to more precise information than ICP and can // devirtualize more effectively, so it should operate on the IR first. // // The WPD and LowerTypeTest passes need to run at -O0 to lower type // metadata and intrinsics. MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary)); MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary)); } if (Level == OptimizationLevel::O0) return MPM; // Force any function attributes we want the rest of the pipeline to observe. MPM.addPass(ForceFunctionAttrsPass()); // Add the core simplification pipeline. MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink, DebugLogging)); // Now add the optimization pipeline. MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging)); return MPM; } ModulePassManager PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level, bool DebugLogging) { assert(Level != OptimizationLevel::O0 && "Must request optimizations for the default pipeline!"); // FIXME: We should use a customized pre-link pipeline! return buildPerModuleDefaultPipeline(Level, DebugLogging, /* LTOPreLink */ true); } ModulePassManager PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging, ModuleSummaryIndex *ExportSummary) { ModulePassManager MPM(DebugLogging); if (Level == OptimizationLevel::O0) { // The WPD and LowerTypeTest passes need to run at -O0 to lower type // metadata and intrinsics. MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr)); MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr)); // Run a second time to clean up any type tests left behind by WPD for use // in ICP. MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true)); return MPM; } if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) { // Load sample profile before running the LTO optimization pipeline. MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile, false /* ThinLTOPhase::PreLink */)); // Cache ProfileSummaryAnalysis once to avoid the potential need to insert // RequireAnalysisPass for PSI before subsequent non-module passes. MPM.addPass(RequireAnalysisPass()); } // Remove unused virtual tables to improve the quality of code generated by // whole-program devirtualization and bitset lowering. MPM.addPass(GlobalDCEPass()); // Force any function attributes we want the rest of the pipeline to observe. MPM.addPass(ForceFunctionAttrsPass()); // Do basic inference of function attributes from known properties of system // libraries and other oracles. MPM.addPass(InferFunctionAttrsPass()); if (Level.getSpeedupLevel() > 1) { FunctionPassManager EarlyFPM(DebugLogging); EarlyFPM.addPass(CallSiteSplittingPass()); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM))); // Indirect call promotion. This should promote all the targets that are // left by the earlier promotion pass that promotes intra-module targets. // This two-step promotion is to save the compile time. For LTO, it should // produce the same result as if we only do promotion here. MPM.addPass(PGOIndirectCallPromotion( true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)); // Propagate constants at call sites into the functions they call. This // opens opportunities for globalopt (and inlining) by substituting function // pointers passed as arguments to direct uses of functions. MPM.addPass(IPSCCPPass()); // Attach metadata to indirect call sites indicating the set of functions // they may target at run-time. This should follow IPSCCP. MPM.addPass(CalledValuePropagationPass()); } // Now deduce any function attributes based in the current code. MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( PostOrderFunctionAttrsPass())); // Do RPO function attribute inference across the module to forward-propagate // attributes where applicable. // FIXME: Is this really an optimization rather than a canonicalization? MPM.addPass(ReversePostOrderFunctionAttrsPass()); // Use in-range annotations on GEP indices to split globals where beneficial. MPM.addPass(GlobalSplitPass()); // Run whole program optimization of virtual call when the list of callees // is fixed. MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr)); // Stop here at -O1. if (Level == OptimizationLevel::O1) { // The LowerTypeTestsPass needs to run to lower type metadata and the // type.test intrinsics. The pass does nothing if CFI is disabled. MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr)); // Run a second time to clean up any type tests left behind by WPD for use // in ICP (which is performed earlier than this in the regular LTO // pipeline). MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true)); return MPM; } // Optimize globals to try and fold them into constants. MPM.addPass(GlobalOptPass()); // Promote any localized globals to SSA registers. MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass())); // Linking modules together can lead to duplicate global constant, only // keep one copy of each constant. MPM.addPass(ConstantMergePass()); // Remove unused arguments from functions. MPM.addPass(DeadArgumentEliminationPass()); // Reduce the code after globalopt and ipsccp. Both can open up significant // simplification opportunities, and both can propagate functions through // function pointers. When this happens, we often have to resolve varargs // calls, etc, so let instcombine do this. FunctionPassManager PeepholeFPM(DebugLogging); if (Level == OptimizationLevel::O3) PeepholeFPM.addPass(AggressiveInstCombinePass()); PeepholeFPM.addPass(InstCombinePass()); invokePeepholeEPCallbacks(PeepholeFPM, Level); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM))); // Note: historically, the PruneEH pass was run first to deduce nounwind and // generally clean up exception handling overhead. It isn't clear this is // valuable as the inliner doesn't currently care whether it is inlining an // invoke or a call. // Run the inliner now. MPM.addPass(ModuleInlinerWrapperPass(getInlineParamsFromOptLevel(Level), DebugLogging)); // Optimize globals again after we ran the inliner. MPM.addPass(GlobalOptPass()); // Garbage collect dead functions. // FIXME: Add ArgumentPromotion pass after once it's ported. MPM.addPass(GlobalDCEPass()); FunctionPassManager FPM(DebugLogging); // The IPO Passes may leave cruft around. Clean up after them. FPM.addPass(InstCombinePass()); invokePeepholeEPCallbacks(FPM, Level); FPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true)); // Do a post inline PGO instrumentation and use pass. This is a context // sensitive PGO pass. if (PGOOpt) { if (PGOOpt->CSAction == PGOOptions::CSIRInstr) addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true, /* IsCS */ true, PGOOpt->CSProfileGenFile, PGOOpt->ProfileRemappingFile); else if (PGOOpt->CSAction == PGOOptions::CSIRUse) addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false, /* IsCS */ true, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile); } // Break up allocas FPM.addPass(SROA()); // LTO provides additional opportunities for tailcall elimination due to // link-time inlining, and visibility of nocapture attribute. FPM.addPass(TailCallElimPass()); // Run a few AA driver optimizations here and now to cleanup the code. MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( PostOrderFunctionAttrsPass())); // FIXME: here we run IP alias analysis in the legacy PM. FunctionPassManager MainFPM; // FIXME: once we fix LoopPass Manager, add LICM here. // FIXME: once we provide support for enabling MLSM, add it here. if (RunNewGVN) MainFPM.addPass(NewGVNPass()); else MainFPM.addPass(GVN()); // Remove dead memcpy()'s. MainFPM.addPass(MemCpyOptPass()); // Nuke dead stores. MainFPM.addPass(DSEPass()); // FIXME: at this point, we run a bunch of loop passes: // indVarSimplify, loopDeletion, loopInterchange, loopUnroll, // loopVectorize. Enable them once the remaining issue with LPM // are sorted out. MainFPM.addPass(InstCombinePass()); MainFPM.addPass(SimplifyCFGPass()); MainFPM.addPass(SCCPPass()); MainFPM.addPass(InstCombinePass()); MainFPM.addPass(BDCEPass()); // FIXME: We may want to run SLPVectorizer here. // After vectorization, assume intrinsics may tell us more // about pointer alignments. #if 0 MainFPM.add(AlignmentFromAssumptionsPass()); #endif // FIXME: Conditionally run LoadCombine here, after it's ported // (in case we still have this pass, given its questionable usefulness). MainFPM.addPass(InstCombinePass()); invokePeepholeEPCallbacks(MainFPM, Level); MainFPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true)); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM))); // Create a function that performs CFI checks for cross-DSO calls with // targets in the current module. MPM.addPass(CrossDSOCFIPass()); // Lower type metadata and the type.test intrinsic. This pass supports // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs // to be run at link time if CFI is enabled. This pass does nothing if // CFI is disabled. MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr)); // Run a second time to clean up any type tests left behind by WPD for use // in ICP (which is performed earlier than this in the regular LTO pipeline). MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true)); // Enable splitting late in the FullLTO post-link pipeline. This is done in // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses). if (EnableHotColdSplit) MPM.addPass(HotColdSplittingPass()); // Add late LTO optimization passes. // Delete basic blocks, which optimization passes may have killed. MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass())); // Drop bodies of available eternally objects to improve GlobalDCE. MPM.addPass(EliminateAvailableExternallyPass()); // Now that we have optimized the program, discard unreachable functions. MPM.addPass(GlobalDCEPass()); // FIXME: Maybe enable MergeFuncs conditionally after it's ported. return MPM; } AAManager PassBuilder::buildDefaultAAPipeline() { AAManager AA; // The order in which these are registered determines their priority when // being queried. // First we register the basic alias analysis that provides the majority of // per-function local AA logic. This is a stateless, on-demand local set of // AA techniques. AA.registerFunctionAnalysis(); // Next we query fast, specialized alias analyses that wrap IR-embedded // information about aliasing. AA.registerFunctionAnalysis(); AA.registerFunctionAnalysis(); // Add support for querying global aliasing information when available. // Because the `AAManager` is a function analysis and `GlobalsAA` is a module // analysis, all that the `AAManager` can do is query for any *cached* // results from `GlobalsAA` through a readonly proxy. AA.registerModuleAnalysis(); return AA; } static Optional parseRepeatPassName(StringRef Name) { if (!Name.consume_front("repeat<") || !Name.consume_back(">")) return None; int Count; if (Name.getAsInteger(0, Count) || Count <= 0) return None; return Count; } static Optional parseDevirtPassName(StringRef Name) { if (!Name.consume_front("devirt<") || !Name.consume_back(">")) return None; int Count; if (Name.getAsInteger(0, Count) || Count <= 0) return None; return Count; } static bool checkParametrizedPassName(StringRef Name, StringRef PassName) { if (!Name.consume_front(PassName)) return false; // normal pass name w/o parameters == default parameters if (Name.empty()) return true; return Name.startswith("<") && Name.endswith(">"); } namespace { /// This performs customized parsing of pass name with parameters. /// /// We do not need parametrization of passes in textual pipeline very often, /// yet on a rare occasion ability to specify parameters right there can be /// useful. /// /// \p Name - parameterized specification of a pass from a textual pipeline /// is a string in a form of : /// PassName '<' parameter-list '>' /// /// Parameter list is being parsed by the parser callable argument, \p Parser, /// It takes a string-ref of parameters and returns either StringError or a /// parameter list in a form of a custom parameters type, all wrapped into /// Expected<> template class. /// template auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name, StringRef PassName) -> decltype(Parser(StringRef{})) { using ParametersT = typename decltype(Parser(StringRef{}))::value_type; StringRef Params = Name; if (!Params.consume_front(PassName)) { assert(false && "unable to strip pass name from parametrized pass specification"); } if (Params.empty()) return ParametersT{}; if (!Params.consume_front("<") || !Params.consume_back(">")) { assert(false && "invalid format for parametrized pass name"); } Expected Result = Parser(Params); assert((Result || Result.template errorIsA()) && "Pass parameter parser can only return StringErrors."); return Result; } /// Parser of parameters for LoopUnroll pass. Expected parseLoopUnrollOptions(StringRef Params) { LoopUnrollOptions UnrollOpts; while (!Params.empty()) { StringRef ParamName; std::tie(ParamName, Params) = Params.split(';'); int OptLevel = StringSwitch(ParamName) .Case("O0", 0) .Case("O1", 1) .Case("O2", 2) .Case("O3", 3) .Default(-1); if (OptLevel >= 0) { UnrollOpts.setOptLevel(OptLevel); continue; } if (ParamName.consume_front("full-unroll-max=")) { int Count; if (ParamName.getAsInteger(0, Count)) return make_error( formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(), inconvertibleErrorCode()); UnrollOpts.setFullUnrollMaxCount(Count); continue; } bool Enable = !ParamName.consume_front("no-"); if (ParamName == "partial") { UnrollOpts.setPartial(Enable); } else if (ParamName == "peeling") { UnrollOpts.setPeeling(Enable); } else if (ParamName == "profile-peeling") { UnrollOpts.setProfileBasedPeeling(Enable); } else if (ParamName == "runtime") { UnrollOpts.setRuntime(Enable); } else if (ParamName == "upperbound") { UnrollOpts.setUpperBound(Enable); } else { return make_error( formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(), inconvertibleErrorCode()); } } return UnrollOpts; } Expected parseMSanPassOptions(StringRef Params) { MemorySanitizerOptions Result; while (!Params.empty()) { StringRef ParamName; std::tie(ParamName, Params) = Params.split(';'); if (ParamName == "recover") { Result.Recover = true; } else if (ParamName == "kernel") { Result.Kernel = true; } else if (ParamName.consume_front("track-origins=")) { if (ParamName.getAsInteger(0, Result.TrackOrigins)) return make_error( formatv("invalid argument to MemorySanitizer pass track-origins " "parameter: '{0}' ", ParamName) .str(), inconvertibleErrorCode()); } else { return make_error( formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName) .str(), inconvertibleErrorCode()); } } return Result; } /// Parser of parameters for SimplifyCFG pass. Expected parseSimplifyCFGOptions(StringRef Params) { SimplifyCFGOptions Result; while (!Params.empty()) { StringRef ParamName; std::tie(ParamName, Params) = Params.split(';'); bool Enable = !ParamName.consume_front("no-"); if (ParamName == "forward-switch-cond") { Result.forwardSwitchCondToPhi(Enable); } else if (ParamName == "switch-to-lookup") { Result.convertSwitchToLookupTable(Enable); } else if (ParamName == "keep-loops") { Result.needCanonicalLoops(Enable); } else if (ParamName == "hoist-common-insts") { Result.hoistCommonInsts(Enable); } else if (ParamName == "sink-common-insts") { Result.sinkCommonInsts(Enable); } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) { APInt BonusInstThreshold; if (ParamName.getAsInteger(0, BonusInstThreshold)) return make_error( formatv("invalid argument to SimplifyCFG pass bonus-threshold " "parameter: '{0}' ", ParamName).str(), inconvertibleErrorCode()); Result.bonusInstThreshold(BonusInstThreshold.getSExtValue()); } else { return make_error( formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName).str(), inconvertibleErrorCode()); } } return Result; } /// Parser of parameters for LoopVectorize pass. Expected parseLoopVectorizeOptions(StringRef Params) { LoopVectorizeOptions Opts; while (!Params.empty()) { StringRef ParamName; std::tie(ParamName, Params) = Params.split(';'); bool Enable = !ParamName.consume_front("no-"); if (ParamName == "interleave-forced-only") { Opts.setInterleaveOnlyWhenForced(Enable); } else if (ParamName == "vectorize-forced-only") { Opts.setVectorizeOnlyWhenForced(Enable); } else { return make_error( formatv("invalid LoopVectorize parameter '{0}' ", ParamName).str(), inconvertibleErrorCode()); } } return Opts; } Expected parseLoopUnswitchOptions(StringRef Params) { bool Result = false; while (!Params.empty()) { StringRef ParamName; std::tie(ParamName, Params) = Params.split(';'); bool Enable = !ParamName.consume_front("no-"); if (ParamName == "nontrivial") { Result = Enable; } else { return make_error( formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName) .str(), inconvertibleErrorCode()); } } return Result; } Expected parseMergedLoadStoreMotionOptions(StringRef Params) { bool Result = false; while (!Params.empty()) { StringRef ParamName; std::tie(ParamName, Params) = Params.split(';'); bool Enable = !ParamName.consume_front("no-"); if (ParamName == "split-footer-bb") { Result = Enable; } else { return make_error( formatv("invalid MergedLoadStoreMotion pass parameter '{0}' ", ParamName) .str(), inconvertibleErrorCode()); } } return Result; } Expected parseGVNOptions(StringRef Params) { GVNOptions Result; while (!Params.empty()) { StringRef ParamName; std::tie(ParamName, Params) = Params.split(';'); bool Enable = !ParamName.consume_front("no-"); if (ParamName == "pre") { Result.setPRE(Enable); } else if (ParamName == "load-pre") { Result.setLoadPRE(Enable); } else if (ParamName == "memdep") { Result.setMemDep(Enable); } else { return make_error( formatv("invalid GVN pass parameter '{0}' ", ParamName).str(), inconvertibleErrorCode()); } } return Result; } Expected parseStackLifetimeOptions(StringRef Params) { StackLifetime::LivenessType Result = StackLifetime::LivenessType::May; while (!Params.empty()) { StringRef ParamName; std::tie(ParamName, Params) = Params.split(';'); if (ParamName == "may") { Result = StackLifetime::LivenessType::May; } else if (ParamName == "must") { Result = StackLifetime::LivenessType::Must; } else { return make_error( formatv("invalid StackLifetime parameter '{0}' ", ParamName).str(), inconvertibleErrorCode()); } } return Result; } } // namespace /// Tests whether a pass name starts with a valid prefix for a default pipeline /// alias. static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) { return Name.startswith("default") || Name.startswith("thinlto") || Name.startswith("lto"); } /// Tests whether registered callbacks will accept a given pass name. /// /// When parsing a pipeline text, the type of the outermost pipeline may be /// omitted, in which case the type is automatically determined from the first /// pass name in the text. This may be a name that is handled through one of the /// callbacks. We check this through the oridinary parsing callbacks by setting /// up a dummy PassManager in order to not force the client to also handle this /// type of query. template static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) { if (!Callbacks.empty()) { PassManagerT DummyPM; for (auto &CB : Callbacks) if (CB(Name, DummyPM, {})) return true; } return false; } template static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) { // Manually handle aliases for pre-configured pipeline fragments. if (startsWithDefaultPipelineAliasPrefix(Name)) return DefaultAliasRegex.match(Name); // Explicitly handle pass manager names. if (Name == "module") return true; if (Name == "cgscc") return true; if (Name == "function") return true; // Explicitly handle custom-parsed pass names. if (parseRepeatPassName(Name)) return true; #define MODULE_PASS(NAME, CREATE_PASS) \ if (Name == NAME) \ return true; #define MODULE_ANALYSIS(NAME, CREATE_PASS) \ if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ return true; #include "PassRegistry.def" return callbacksAcceptPassName(Name, Callbacks); } template static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) { // Explicitly handle pass manager names. if (Name == "cgscc") return true; if (Name == "function") return true; // Explicitly handle custom-parsed pass names. if (parseRepeatPassName(Name)) return true; if (parseDevirtPassName(Name)) return true; #define CGSCC_PASS(NAME, CREATE_PASS) \ if (Name == NAME) \ return true; #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \ if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ return true; #include "PassRegistry.def" return callbacksAcceptPassName(Name, Callbacks); } template static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) { // Explicitly handle pass manager names. if (Name == "function") return true; if (Name == "loop" || Name == "loop-mssa") return true; // Explicitly handle custom-parsed pass names. if (parseRepeatPassName(Name)) return true; #define FUNCTION_PASS(NAME, CREATE_PASS) \ if (Name == NAME) \ return true; #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ if (checkParametrizedPassName(Name, NAME)) \ return true; #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ return true; #include "PassRegistry.def" return callbacksAcceptPassName(Name, Callbacks); } template static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) { // Explicitly handle pass manager names. if (Name == "loop" || Name == "loop-mssa") return true; // Explicitly handle custom-parsed pass names. if (parseRepeatPassName(Name)) return true; #define LOOP_PASS(NAME, CREATE_PASS) \ if (Name == NAME) \ return true; #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ if (checkParametrizedPassName(Name, NAME)) \ return true; #define LOOP_ANALYSIS(NAME, CREATE_PASS) \ if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ return true; #include "PassRegistry.def" return callbacksAcceptPassName(Name, Callbacks); } Optional> PassBuilder::parsePipelineText(StringRef Text) { std::vector ResultPipeline; SmallVector *, 4> PipelineStack = { &ResultPipeline}; for (;;) { std::vector &Pipeline = *PipelineStack.back(); size_t Pos = Text.find_first_of(",()"); Pipeline.push_back({Text.substr(0, Pos), {}}); // If we have a single terminating name, we're done. if (Pos == Text.npos) break; char Sep = Text[Pos]; Text = Text.substr(Pos + 1); if (Sep == ',') // Just a name ending in a comma, continue. continue; if (Sep == '(') { // Push the inner pipeline onto the stack to continue processing. PipelineStack.push_back(&Pipeline.back().InnerPipeline); continue; } assert(Sep == ')' && "Bogus separator!"); // When handling the close parenthesis, we greedily consume them to avoid // empty strings in the pipeline. do { // If we try to pop the outer pipeline we have unbalanced parentheses. if (PipelineStack.size() == 1) return None; PipelineStack.pop_back(); } while (Text.consume_front(")")); // Check if we've finished parsing. if (Text.empty()) break; // Otherwise, the end of an inner pipeline always has to be followed by // a comma, and then we can continue. if (!Text.consume_front(",")) return None; } if (PipelineStack.size() > 1) // Unbalanced paretheses. return None; assert(PipelineStack.back() == &ResultPipeline && "Wrong pipeline at the bottom of the stack!"); return {std::move(ResultPipeline)}; } Error PassBuilder::parseModulePass(ModulePassManager &MPM, const PipelineElement &E, bool VerifyEachPass, bool DebugLogging) { auto &Name = E.Name; auto &InnerPipeline = E.InnerPipeline; // First handle complex passes like the pass managers which carry pipelines. if (!InnerPipeline.empty()) { if (Name == "module") { ModulePassManager NestedMPM(DebugLogging); if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; MPM.addPass(std::move(NestedMPM)); return Error::success(); } if (Name == "cgscc") { CGSCCPassManager CGPM(DebugLogging); if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM))); return Error::success(); } if (Name == "function") { FunctionPassManager FPM(DebugLogging); if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); return Error::success(); } if (auto Count = parseRepeatPassName(Name)) { ModulePassManager NestedMPM(DebugLogging); if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM))); return Error::success(); } for (auto &C : ModulePipelineParsingCallbacks) if (C(Name, MPM, InnerPipeline)) return Error::success(); // Normal passes can't have pipelines. return make_error( formatv("invalid use of '{0}' pass as module pipeline", Name).str(), inconvertibleErrorCode()); ; } // Manually handle aliases for pre-configured pipeline fragments. if (startsWithDefaultPipelineAliasPrefix(Name)) { SmallVector Matches; if (!DefaultAliasRegex.match(Name, &Matches)) return make_error( formatv("unknown default pipeline alias '{0}'", Name).str(), inconvertibleErrorCode()); assert(Matches.size() == 3 && "Must capture two matched strings!"); OptimizationLevel L = StringSwitch(Matches[2]) .Case("O0", OptimizationLevel::O0) .Case("O1", OptimizationLevel::O1) .Case("O2", OptimizationLevel::O2) .Case("O3", OptimizationLevel::O3) .Case("Os", OptimizationLevel::Os) .Case("Oz", OptimizationLevel::Oz); if (L == OptimizationLevel::O0) { // Add instrumentation PGO passes -- at O0 we can still do PGO. if (PGOOpt && Matches[1] != "thinlto" && (PGOOpt->Action == PGOOptions::IRInstr || PGOOpt->Action == PGOOptions::IRUse)) addPGOInstrPassesForO0( MPM, DebugLogging, /* RunProfileGen */ (PGOOpt->Action == PGOOptions::IRInstr), /* IsCS */ false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile); // For IR that makes use of coroutines intrinsics, coroutine passes must // be run, even at -O0. if (PTO.Coroutines) { MPM.addPass(createModuleToFunctionPassAdaptor(CoroEarlyPass())); CGSCCPassManager CGPM(DebugLogging); CGPM.addPass(CoroSplitPass()); CGPM.addPass(createCGSCCToFunctionPassAdaptor(CoroElidePass())); MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM))); MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass())); } // Do nothing else at all! return Error::success(); } // This is consistent with old pass manager invoked via opt, but // inconsistent with clang. Clang doesn't enable loop vectorization // but does enable slp vectorization at Oz. PTO.LoopVectorization = L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz; PTO.SLPVectorization = L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz; if (Matches[1] == "default") { MPM.addPass(buildPerModuleDefaultPipeline(L, DebugLogging)); } else if (Matches[1] == "thinlto-pre-link") { MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L, DebugLogging)); } else if (Matches[1] == "thinlto") { MPM.addPass(buildThinLTODefaultPipeline(L, DebugLogging, nullptr)); } else if (Matches[1] == "lto-pre-link") { MPM.addPass(buildLTOPreLinkDefaultPipeline(L, DebugLogging)); } else { assert(Matches[1] == "lto" && "Not one of the matched options!"); MPM.addPass(buildLTODefaultPipeline(L, DebugLogging, nullptr)); } return Error::success(); } // Finally expand the basic registered passes from the .inc file. #define MODULE_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ MPM.addPass(CREATE_PASS); \ return Error::success(); \ } #define MODULE_ANALYSIS(NAME, CREATE_PASS) \ if (Name == "require<" NAME ">") { \ MPM.addPass( \ RequireAnalysisPass< \ std::remove_reference::type, Module>()); \ return Error::success(); \ } \ if (Name == "invalidate<" NAME ">") { \ MPM.addPass(InvalidateAnalysisPass< \ std::remove_reference::type>()); \ return Error::success(); \ } #define CGSCC_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(CREATE_PASS)); \ return Error::success(); \ } #define FUNCTION_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS)); \ return Error::success(); \ } #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ if (checkParametrizedPassName(Name, NAME)) { \ auto Params = parsePassParameters(PARSER, Name, NAME); \ if (!Params) \ return Params.takeError(); \ MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \ return Error::success(); \ } #define LOOP_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ MPM.addPass( \ createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \ CREATE_PASS, false, false, DebugLogging))); \ return Error::success(); \ } #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ if (checkParametrizedPassName(Name, NAME)) { \ auto Params = parsePassParameters(PARSER, Name, NAME); \ if (!Params) \ return Params.takeError(); \ MPM.addPass( \ createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \ CREATE_PASS(Params.get()), false, false, DebugLogging))); \ return Error::success(); \ } #include "PassRegistry.def" for (auto &C : ModulePipelineParsingCallbacks) if (C(Name, MPM, InnerPipeline)) return Error::success(); return make_error( formatv("unknown module pass '{0}'", Name).str(), inconvertibleErrorCode()); } Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM, const PipelineElement &E, bool VerifyEachPass, bool DebugLogging) { auto &Name = E.Name; auto &InnerPipeline = E.InnerPipeline; // First handle complex passes like the pass managers which carry pipelines. if (!InnerPipeline.empty()) { if (Name == "cgscc") { CGSCCPassManager NestedCGPM(DebugLogging); if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; // Add the nested pass manager with the appropriate adaptor. CGPM.addPass(std::move(NestedCGPM)); return Error::success(); } if (Name == "function") { FunctionPassManager FPM(DebugLogging); if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; // Add the nested pass manager with the appropriate adaptor. CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM))); return Error::success(); } if (auto Count = parseRepeatPassName(Name)) { CGSCCPassManager NestedCGPM(DebugLogging); if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM))); return Error::success(); } if (auto MaxRepetitions = parseDevirtPassName(Name)) { CGSCCPassManager NestedCGPM(DebugLogging); if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; CGPM.addPass( createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions)); return Error::success(); } for (auto &C : CGSCCPipelineParsingCallbacks) if (C(Name, CGPM, InnerPipeline)) return Error::success(); // Normal passes can't have pipelines. return make_error( formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(), inconvertibleErrorCode()); } // Now expand the basic registered passes from the .inc file. #define CGSCC_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ CGPM.addPass(CREATE_PASS); \ return Error::success(); \ } #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \ if (Name == "require<" NAME ">") { \ CGPM.addPass(RequireAnalysisPass< \ std::remove_reference::type, \ LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, \ CGSCCUpdateResult &>()); \ return Error::success(); \ } \ if (Name == "invalidate<" NAME ">") { \ CGPM.addPass(InvalidateAnalysisPass< \ std::remove_reference::type>()); \ return Error::success(); \ } #define FUNCTION_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS)); \ return Error::success(); \ } #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ if (checkParametrizedPassName(Name, NAME)) { \ auto Params = parsePassParameters(PARSER, Name, NAME); \ if (!Params) \ return Params.takeError(); \ CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \ return Error::success(); \ } #define LOOP_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ CGPM.addPass( \ createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \ CREATE_PASS, false, false, DebugLogging))); \ return Error::success(); \ } #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ if (checkParametrizedPassName(Name, NAME)) { \ auto Params = parsePassParameters(PARSER, Name, NAME); \ if (!Params) \ return Params.takeError(); \ CGPM.addPass( \ createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \ CREATE_PASS(Params.get()), false, false, DebugLogging))); \ return Error::success(); \ } #include "PassRegistry.def" for (auto &C : CGSCCPipelineParsingCallbacks) if (C(Name, CGPM, InnerPipeline)) return Error::success(); return make_error( formatv("unknown cgscc pass '{0}'", Name).str(), inconvertibleErrorCode()); } Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM, const PipelineElement &E, bool VerifyEachPass, bool DebugLogging) { auto &Name = E.Name; auto &InnerPipeline = E.InnerPipeline; // First handle complex passes like the pass managers which carry pipelines. if (!InnerPipeline.empty()) { if (Name == "function") { FunctionPassManager NestedFPM(DebugLogging); if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; // Add the nested pass manager with the appropriate adaptor. FPM.addPass(std::move(NestedFPM)); return Error::success(); } if (Name == "loop" || Name == "loop-mssa") { LoopPassManager LPM(DebugLogging); if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; // Add the nested pass manager with the appropriate adaptor. bool UseMemorySSA = (Name == "loop-mssa"); bool UseBFI = std::any_of(InnerPipeline.begin(), InnerPipeline.end(), [](auto Pipeline) { return Pipeline.Name == "licm"; }); FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), UseMemorySSA, UseBFI, DebugLogging)); return Error::success(); } if (auto Count = parseRepeatPassName(Name)) { FunctionPassManager NestedFPM(DebugLogging); if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM))); return Error::success(); } for (auto &C : FunctionPipelineParsingCallbacks) if (C(Name, FPM, InnerPipeline)) return Error::success(); // Normal passes can't have pipelines. return make_error( formatv("invalid use of '{0}' pass as function pipeline", Name).str(), inconvertibleErrorCode()); } // Now expand the basic registered passes from the .inc file. #define FUNCTION_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ FPM.addPass(CREATE_PASS); \ return Error::success(); \ } #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ if (checkParametrizedPassName(Name, NAME)) { \ auto Params = parsePassParameters(PARSER, Name, NAME); \ if (!Params) \ return Params.takeError(); \ FPM.addPass(CREATE_PASS(Params.get())); \ return Error::success(); \ } #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ if (Name == "require<" NAME ">") { \ FPM.addPass( \ RequireAnalysisPass< \ std::remove_reference::type, Function>()); \ return Error::success(); \ } \ if (Name == "invalidate<" NAME ">") { \ FPM.addPass(InvalidateAnalysisPass< \ std::remove_reference::type>()); \ return Error::success(); \ } // FIXME: UseMemorySSA is set to false. Maybe we could do things like: // bool UseMemorySSA = !("canon-freeze" || "loop-predication" || // "guard-widening"); // The risk is that it may become obsolete if we're not careful. #define LOOP_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS, false, false, \ DebugLogging)); \ return Error::success(); \ } #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ if (checkParametrizedPassName(Name, NAME)) { \ auto Params = parsePassParameters(PARSER, Name, NAME); \ if (!Params) \ return Params.takeError(); \ FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS(Params.get()), \ false, false, DebugLogging)); \ return Error::success(); \ } #include "PassRegistry.def" for (auto &C : FunctionPipelineParsingCallbacks) if (C(Name, FPM, InnerPipeline)) return Error::success(); return make_error( formatv("unknown function pass '{0}'", Name).str(), inconvertibleErrorCode()); } Error PassBuilder::parseLoopPass(LoopPassManager &LPM, const PipelineElement &E, bool VerifyEachPass, bool DebugLogging) { StringRef Name = E.Name; auto &InnerPipeline = E.InnerPipeline; // First handle complex passes like the pass managers which carry pipelines. if (!InnerPipeline.empty()) { if (Name == "loop") { LoopPassManager NestedLPM(DebugLogging); if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; // Add the nested pass manager with the appropriate adaptor. LPM.addPass(std::move(NestedLPM)); return Error::success(); } if (auto Count = parseRepeatPassName(Name)) { LoopPassManager NestedLPM(DebugLogging); if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM))); return Error::success(); } for (auto &C : LoopPipelineParsingCallbacks) if (C(Name, LPM, InnerPipeline)) return Error::success(); // Normal passes can't have pipelines. return make_error( formatv("invalid use of '{0}' pass as loop pipeline", Name).str(), inconvertibleErrorCode()); } // Now expand the basic registered passes from the .inc file. #define LOOP_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ LPM.addPass(CREATE_PASS); \ return Error::success(); \ } #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ if (checkParametrizedPassName(Name, NAME)) { \ auto Params = parsePassParameters(PARSER, Name, NAME); \ if (!Params) \ return Params.takeError(); \ LPM.addPass(CREATE_PASS(Params.get())); \ return Error::success(); \ } #define LOOP_ANALYSIS(NAME, CREATE_PASS) \ if (Name == "require<" NAME ">") { \ LPM.addPass(RequireAnalysisPass< \ std::remove_reference::type, Loop, \ LoopAnalysisManager, LoopStandardAnalysisResults &, \ LPMUpdater &>()); \ return Error::success(); \ } \ if (Name == "invalidate<" NAME ">") { \ LPM.addPass(InvalidateAnalysisPass< \ std::remove_reference::type>()); \ return Error::success(); \ } #include "PassRegistry.def" for (auto &C : LoopPipelineParsingCallbacks) if (C(Name, LPM, InnerPipeline)) return Error::success(); return make_error(formatv("unknown loop pass '{0}'", Name).str(), inconvertibleErrorCode()); } bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) { #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ if (Name == NAME) { \ AA.registerModuleAnalysis< \ std::remove_reference::type>(); \ return true; \ } #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ if (Name == NAME) { \ AA.registerFunctionAnalysis< \ std::remove_reference::type>(); \ return true; \ } #include "PassRegistry.def" for (auto &C : AAParsingCallbacks) if (C(Name, AA)) return true; return false; } Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM, ArrayRef Pipeline, bool VerifyEachPass, bool DebugLogging) { for (const auto &Element : Pipeline) { if (auto Err = parseLoopPass(LPM, Element, VerifyEachPass, DebugLogging)) return Err; // FIXME: No verifier support for Loop passes! } return Error::success(); } Error PassBuilder::parseFunctionPassPipeline(FunctionPassManager &FPM, ArrayRef Pipeline, bool VerifyEachPass, bool DebugLogging) { for (const auto &Element : Pipeline) { if (auto Err = parseFunctionPass(FPM, Element, VerifyEachPass, DebugLogging)) return Err; if (VerifyEachPass) FPM.addPass(VerifierPass()); } return Error::success(); } Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM, ArrayRef Pipeline, bool VerifyEachPass, bool DebugLogging) { for (const auto &Element : Pipeline) { if (auto Err = parseCGSCCPass(CGPM, Element, VerifyEachPass, DebugLogging)) return Err; // FIXME: No verifier support for CGSCC passes! } return Error::success(); } void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM, FunctionAnalysisManager &FAM, CGSCCAnalysisManager &CGAM, ModuleAnalysisManager &MAM) { MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); }); MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); }); CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); }); FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); }); FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); }); FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); }); LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); }); } Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM, ArrayRef Pipeline, bool VerifyEachPass, bool DebugLogging) { for (const auto &Element : Pipeline) { if (auto Err = parseModulePass(MPM, Element, VerifyEachPass, DebugLogging)) return Err; if (VerifyEachPass) MPM.addPass(VerifierPass()); } return Error::success(); } // Primary pass pipeline description parsing routine for a \c ModulePassManager // FIXME: Should this routine accept a TargetMachine or require the caller to // pre-populate the analysis managers with target-specific stuff? Error PassBuilder::parsePassPipeline(ModulePassManager &MPM, StringRef PipelineText, bool VerifyEachPass, bool DebugLogging) { auto Pipeline = parsePipelineText(PipelineText); if (!Pipeline || Pipeline->empty()) return make_error( formatv("invalid pipeline '{0}'", PipelineText).str(), inconvertibleErrorCode()); // If the first name isn't at the module layer, wrap the pipeline up // automatically. StringRef FirstName = Pipeline->front().Name; if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) { if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) { Pipeline = {{"cgscc", std::move(*Pipeline)}}; } else if (isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks)) { Pipeline = {{"function", std::move(*Pipeline)}}; } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) { Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}}; } else { for (auto &C : TopLevelPipelineParsingCallbacks) if (C(MPM, *Pipeline, VerifyEachPass, DebugLogging)) return Error::success(); // Unknown pass or pipeline name! auto &InnerPipeline = Pipeline->front().InnerPipeline; return make_error( formatv("unknown {0} name '{1}'", (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName) .str(), inconvertibleErrorCode()); } } if (auto Err = parseModulePassPipeline(MPM, *Pipeline, VerifyEachPass, DebugLogging)) return Err; return Error::success(); } // Primary pass pipeline description parsing routine for a \c CGSCCPassManager Error PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM, StringRef PipelineText, bool VerifyEachPass, bool DebugLogging) { auto Pipeline = parsePipelineText(PipelineText); if (!Pipeline || Pipeline->empty()) return make_error( formatv("invalid pipeline '{0}'", PipelineText).str(), inconvertibleErrorCode()); StringRef FirstName = Pipeline->front().Name; if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) return make_error( formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName, PipelineText) .str(), inconvertibleErrorCode()); if (auto Err = parseCGSCCPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging)) return Err; return Error::success(); } // Primary pass pipeline description parsing routine for a \c // FunctionPassManager Error PassBuilder::parsePassPipeline(FunctionPassManager &FPM, StringRef PipelineText, bool VerifyEachPass, bool DebugLogging) { auto Pipeline = parsePipelineText(PipelineText); if (!Pipeline || Pipeline->empty()) return make_error( formatv("invalid pipeline '{0}'", PipelineText).str(), inconvertibleErrorCode()); StringRef FirstName = Pipeline->front().Name; if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks)) return make_error( formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName, PipelineText) .str(), inconvertibleErrorCode()); if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline, VerifyEachPass, DebugLogging)) return Err; return Error::success(); } // Primary pass pipeline description parsing routine for a \c LoopPassManager Error PassBuilder::parsePassPipeline(LoopPassManager &CGPM, StringRef PipelineText, bool VerifyEachPass, bool DebugLogging) { auto Pipeline = parsePipelineText(PipelineText); if (!Pipeline || Pipeline->empty()) return make_error( formatv("invalid pipeline '{0}'", PipelineText).str(), inconvertibleErrorCode()); if (auto Err = parseLoopPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging)) return Err; return Error::success(); } Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) { // If the pipeline just consists of the word 'default' just replace the AA // manager with our default one. if (PipelineText == "default") { AA = buildDefaultAAPipeline(); return Error::success(); } while (!PipelineText.empty()) { StringRef Name; std::tie(Name, PipelineText) = PipelineText.split(','); if (!parseAAPassName(AA, Name)) return make_error( formatv("unknown alias analysis name '{0}'", Name).str(), inconvertibleErrorCode()); } return Error::success(); } bool PassBuilder::isAAPassName(StringRef PassName) { #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ if (PassName == NAME) \ return true; #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ if (PassName == NAME) \ return true; #include "PassRegistry.def" return false; } bool PassBuilder::isAnalysisPassName(StringRef PassName) { #define MODULE_ANALYSIS(NAME, CREATE_PASS) \ if (PassName == NAME) \ return true; #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ if (PassName == NAME) \ return true; #define LOOP_ANALYSIS(NAME, CREATE_PASS) \ if (PassName == NAME) \ return true; #define CGSSC_ANALYSIS(NAME, CREATE_PASS) \ if (PassName == NAME) \ return true; #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ if (PassName == NAME) \ return true; #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ if (PassName == NAME) \ return true; #include "PassRegistry.def" return false; } void PassBuilder::registerParseTopLevelPipelineCallback( const std::function, bool VerifyEachPass, bool DebugLogging)> &C) { TopLevelPipelineParsingCallbacks.push_back(C); } diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index f9a208b92b19..79b942a21b07 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -1,388 +1,389 @@ //===- PassRegistry.def - Registry of passes --------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file is used as the registry of passes that are part of the core LLVM // libraries. This file describes both transformation passes and analyses // Analyses are registered while transformation passes have names registered // that can be used when providing a textual pass pipeline. // //===----------------------------------------------------------------------===// // NOTE: NO INCLUDE GUARD DESIRED! #ifndef MODULE_ANALYSIS #define MODULE_ANALYSIS(NAME, CREATE_PASS) #endif MODULE_ANALYSIS("callgraph", CallGraphAnalysis()) MODULE_ANALYSIS("lcg", LazyCallGraphAnalysis()) MODULE_ANALYSIS("module-summary", ModuleSummaryIndexAnalysis()) MODULE_ANALYSIS("no-op-module", NoOpModuleAnalysis()) MODULE_ANALYSIS("profile-summary", ProfileSummaryAnalysis()) MODULE_ANALYSIS("stack-safety", StackSafetyGlobalAnalysis()) MODULE_ANALYSIS("verify", VerifierAnalysis()) MODULE_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC)) MODULE_ANALYSIS("asan-globals-md", ASanGlobalsMetadataAnalysis()) MODULE_ANALYSIS("inline-advisor", InlineAdvisorAnalysis()) MODULE_ANALYSIS("ir-similarity", IRSimilarityAnalysis()) #ifndef MODULE_ALIAS_ANALYSIS #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ MODULE_ANALYSIS(NAME, CREATE_PASS) #endif MODULE_ALIAS_ANALYSIS("globals-aa", GlobalsAA()) #undef MODULE_ALIAS_ANALYSIS #undef MODULE_ANALYSIS #ifndef MODULE_PASS #define MODULE_PASS(NAME, CREATE_PASS) #endif MODULE_PASS("always-inline", AlwaysInlinerPass()) MODULE_PASS("attributor", AttributorPass()) MODULE_PASS("called-value-propagation", CalledValuePropagationPass()) MODULE_PASS("canonicalize-aliases", CanonicalizeAliasesPass()) MODULE_PASS("cg-profile", CGProfilePass()) MODULE_PASS("constmerge", ConstantMergePass()) MODULE_PASS("cross-dso-cfi", CrossDSOCFIPass()) MODULE_PASS("deadargelim", DeadArgumentEliminationPass()) MODULE_PASS("elim-avail-extern", EliminateAvailableExternallyPass()) MODULE_PASS("forceattrs", ForceFunctionAttrsPass()) MODULE_PASS("function-import", FunctionImportPass()) MODULE_PASS("globaldce", GlobalDCEPass()) MODULE_PASS("globalopt", GlobalOptPass()) MODULE_PASS("globalsplit", GlobalSplitPass()) MODULE_PASS("hotcoldsplit", HotColdSplittingPass()) MODULE_PASS("hwasan", HWAddressSanitizerPass(false, false)) MODULE_PASS("khwasan", HWAddressSanitizerPass(true, true)) MODULE_PASS("inferattrs", InferFunctionAttrsPass()) MODULE_PASS("inliner-wrapper", ModuleInlinerWrapperPass()) MODULE_PASS("insert-gcov-profiling", GCOVProfilerPass()) MODULE_PASS("instrorderfile", InstrOrderFilePass()) MODULE_PASS("instrprof", InstrProfiling()) MODULE_PASS("internalize", InternalizePass()) MODULE_PASS("invalidate", InvalidateAllAnalysesPass()) MODULE_PASS("ipsccp", IPSCCPPass()) MODULE_PASS("print-ir-similarity", IRSimilarityAnalysisPrinterPass(dbgs())) MODULE_PASS("lowertypetests", LowerTypeTestsPass()) MODULE_PASS("metarenamer", MetaRenamerPass()) MODULE_PASS("mergefunc", MergeFunctionsPass()) MODULE_PASS("name-anon-globals", NameAnonGlobalPass()) MODULE_PASS("no-op-module", NoOpModulePass()) MODULE_PASS("objc-arc", ObjCARCOptPass()) MODULE_PASS("objc-arc-contract", ObjCARCContractPass()) MODULE_PASS("partial-inliner", PartialInlinerPass()) MODULE_PASS("pgo-icall-prom", PGOIndirectCallPromotion()) MODULE_PASS("pgo-instr-gen", PGOInstrumentationGen()) MODULE_PASS("pgo-instr-use", PGOInstrumentationUse()) MODULE_PASS("print-profile-summary", ProfileSummaryPrinterPass(dbgs())) MODULE_PASS("print-callgraph", CallGraphPrinterPass(dbgs())) MODULE_PASS("print", PrintModulePass(dbgs())) MODULE_PASS("print-lcg", LazyCallGraphPrinterPass(dbgs())) MODULE_PASS("print-lcg-dot", LazyCallGraphDOTPrinterPass(dbgs())) MODULE_PASS("print-stack-safety", StackSafetyGlobalPrinterPass(dbgs())) MODULE_PASS("rewrite-statepoints-for-gc", RewriteStatepointsForGC()) MODULE_PASS("rewrite-symbols", RewriteSymbolPass()) MODULE_PASS("rpo-function-attrs", ReversePostOrderFunctionAttrsPass()) MODULE_PASS("sample-profile", SampleProfileLoaderPass()) MODULE_PASS("scc-oz-module-inliner", buildInlinerPipeline(OptimizationLevel::Oz, ThinLTOPhase::None, DebugLogging)) MODULE_PASS("oz-module-optimizer", buildModuleOptimizationPipeline(OptimizationLevel::Oz, DebugLogging, /*LTOPreLink*/false)) MODULE_PASS("strip", StripSymbolsPass()) MODULE_PASS("strip-dead-debug-info", StripDeadDebugInfoPass()) MODULE_PASS("strip-dead-prototypes", StripDeadPrototypesPass()) MODULE_PASS("strip-debug-declare", StripDebugDeclarePass()) MODULE_PASS("strip-nondebug", StripNonDebugSymbolsPass()) MODULE_PASS("strip-nonlinetable-debuginfo", StripNonLineTableDebugInfoPass()) MODULE_PASS("synthetic-counts-propagation", SyntheticCountsPropagation()) MODULE_PASS("wholeprogramdevirt", WholeProgramDevirtPass()) MODULE_PASS("verify", VerifierPass()) MODULE_PASS("dfsan", DataFlowSanitizerPass()) MODULE_PASS("asan-module", ModuleAddressSanitizerPass(/*CompileKernel=*/false, false, true, false)) MODULE_PASS("msan-module", MemorySanitizerPass({})) MODULE_PASS("tsan-module", ThreadSanitizerPass()) MODULE_PASS("kasan-module", ModuleAddressSanitizerPass(/*CompileKernel=*/true, false, true, false)) MODULE_PASS("sancov-module", ModuleSanitizerCoveragePass()) MODULE_PASS("memprof-module", ModuleMemProfilerPass()) MODULE_PASS("poison-checking", PoisonCheckingPass()) #undef MODULE_PASS #ifndef CGSCC_ANALYSIS #define CGSCC_ANALYSIS(NAME, CREATE_PASS) #endif CGSCC_ANALYSIS("no-op-cgscc", NoOpCGSCCAnalysis()) CGSCC_ANALYSIS("fam-proxy", FunctionAnalysisManagerCGSCCProxy()) CGSCC_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC)) #undef CGSCC_ANALYSIS #ifndef CGSCC_PASS #define CGSCC_PASS(NAME, CREATE_PASS) #endif CGSCC_PASS("argpromotion", ArgumentPromotionPass()) CGSCC_PASS("invalidate", InvalidateAllAnalysesPass()) CGSCC_PASS("function-attrs", PostOrderFunctionAttrsPass()) CGSCC_PASS("attributor-cgscc", AttributorCGSCCPass()) CGSCC_PASS("inline", InlinerPass()) CGSCC_PASS("openmpopt", OpenMPOptPass()) CGSCC_PASS("coro-split", CoroSplitPass()) CGSCC_PASS("no-op-cgscc", NoOpCGSCCPass()) #undef CGSCC_PASS #ifndef FUNCTION_ANALYSIS #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) #endif FUNCTION_ANALYSIS("aa", AAManager()) FUNCTION_ANALYSIS("assumptions", AssumptionAnalysis()) FUNCTION_ANALYSIS("block-freq", BlockFrequencyAnalysis()) FUNCTION_ANALYSIS("branch-prob", BranchProbabilityAnalysis()) FUNCTION_ANALYSIS("domtree", DominatorTreeAnalysis()) FUNCTION_ANALYSIS("postdomtree", PostDominatorTreeAnalysis()) FUNCTION_ANALYSIS("demanded-bits", DemandedBitsAnalysis()) FUNCTION_ANALYSIS("domfrontier", DominanceFrontierAnalysis()) FUNCTION_ANALYSIS("func-properties", FunctionPropertiesAnalysis()) FUNCTION_ANALYSIS("loops", LoopAnalysis()) FUNCTION_ANALYSIS("lazy-value-info", LazyValueAnalysis()) FUNCTION_ANALYSIS("da", DependenceAnalysis()) FUNCTION_ANALYSIS("inliner-size-estimator", InlineSizeEstimatorAnalysis()) FUNCTION_ANALYSIS("memdep", MemoryDependenceAnalysis()) FUNCTION_ANALYSIS("memoryssa", MemorySSAAnalysis()) FUNCTION_ANALYSIS("phi-values", PhiValuesAnalysis()) FUNCTION_ANALYSIS("regions", RegionInfoAnalysis()) FUNCTION_ANALYSIS("no-op-function", NoOpFunctionAnalysis()) FUNCTION_ANALYSIS("opt-remark-emit", OptimizationRemarkEmitterAnalysis()) FUNCTION_ANALYSIS("scalar-evolution", ScalarEvolutionAnalysis()) FUNCTION_ANALYSIS("stack-safety-local", StackSafetyAnalysis()) FUNCTION_ANALYSIS("targetlibinfo", TargetLibraryAnalysis()) FUNCTION_ANALYSIS("targetir", TM ? TM->getTargetIRAnalysis() : TargetIRAnalysis()) FUNCTION_ANALYSIS("verify", VerifierAnalysis()) FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC)) #ifndef FUNCTION_ALIAS_ANALYSIS #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ FUNCTION_ANALYSIS(NAME, CREATE_PASS) #endif FUNCTION_ALIAS_ANALYSIS("basic-aa", BasicAA()) FUNCTION_ALIAS_ANALYSIS("cfl-anders-aa", CFLAndersAA()) FUNCTION_ALIAS_ANALYSIS("cfl-steens-aa", CFLSteensAA()) FUNCTION_ALIAS_ANALYSIS("objc-arc-aa", objcarc::ObjCARCAA()) FUNCTION_ALIAS_ANALYSIS("scev-aa", SCEVAA()) FUNCTION_ALIAS_ANALYSIS("scoped-noalias-aa", ScopedNoAliasAA()) FUNCTION_ALIAS_ANALYSIS("tbaa", TypeBasedAA()) #undef FUNCTION_ALIAS_ANALYSIS #undef FUNCTION_ANALYSIS #ifndef FUNCTION_PASS #define FUNCTION_PASS(NAME, CREATE_PASS) #endif FUNCTION_PASS("aa-eval", AAEvaluator()) FUNCTION_PASS("adce", ADCEPass()) FUNCTION_PASS("add-discriminators", AddDiscriminatorsPass()) FUNCTION_PASS("aggressive-instcombine", AggressiveInstCombinePass()) FUNCTION_PASS("assume-builder", AssumeBuilderPass()) FUNCTION_PASS("assume-simplify", AssumeSimplifyPass()) FUNCTION_PASS("alignment-from-assumptions", AlignmentFromAssumptionsPass()) FUNCTION_PASS("bdce", BDCEPass()) FUNCTION_PASS("bounds-checking", BoundsCheckingPass()) FUNCTION_PASS("break-crit-edges", BreakCriticalEdgesPass()) FUNCTION_PASS("callsite-splitting", CallSiteSplittingPass()) FUNCTION_PASS("consthoist", ConstantHoistingPass()) FUNCTION_PASS("constraint-elimination", ConstraintEliminationPass()) FUNCTION_PASS("chr", ControlHeightReductionPass()) FUNCTION_PASS("coro-early", CoroEarlyPass()) FUNCTION_PASS("coro-elide", CoroElidePass()) FUNCTION_PASS("coro-cleanup", CoroCleanupPass()) FUNCTION_PASS("correlated-propagation", CorrelatedValuePropagationPass()) FUNCTION_PASS("dce", DCEPass()) FUNCTION_PASS("div-rem-pairs", DivRemPairsPass()) FUNCTION_PASS("dse", DSEPass()) FUNCTION_PASS("dot-cfg", CFGPrinterPass()) FUNCTION_PASS("dot-cfg-only", CFGOnlyPrinterPass()) FUNCTION_PASS("early-cse", EarlyCSEPass(/*UseMemorySSA=*/false)) FUNCTION_PASS("early-cse-memssa", EarlyCSEPass(/*UseMemorySSA=*/true)) FUNCTION_PASS("ee-instrument", EntryExitInstrumenterPass(/*PostInlining=*/false)) FUNCTION_PASS("make-guards-explicit", MakeGuardsExplicitPass()) FUNCTION_PASS("post-inline-ee-instrument", EntryExitInstrumenterPass(/*PostInlining=*/true)) FUNCTION_PASS("gvn-hoist", GVNHoistPass()) FUNCTION_PASS("gvn-sink", GVNSinkPass()) FUNCTION_PASS("helloworld", HelloWorldPass()) FUNCTION_PASS("instcombine", InstCombinePass()) FUNCTION_PASS("instcount", InstCountPass()) FUNCTION_PASS("instsimplify", InstSimplifyPass()) FUNCTION_PASS("invalidate", InvalidateAllAnalysesPass()) FUNCTION_PASS("irce", IRCEPass()) FUNCTION_PASS("float2int", Float2IntPass()) FUNCTION_PASS("no-op-function", NoOpFunctionPass()) FUNCTION_PASS("libcalls-shrinkwrap", LibCallsShrinkWrapPass()) FUNCTION_PASS("lint", LintPass()) FUNCTION_PASS("inject-tli-mappings", InjectTLIMappings()) FUNCTION_PASS("loweratomic", LowerAtomicPass()) FUNCTION_PASS("lower-expect", LowerExpectIntrinsicPass()) FUNCTION_PASS("lower-guard-intrinsic", LowerGuardIntrinsicPass()) FUNCTION_PASS("lower-constant-intrinsics", LowerConstantIntrinsicsPass()) FUNCTION_PASS("lower-matrix-intrinsics", LowerMatrixIntrinsicsPass()) FUNCTION_PASS("lower-widenable-condition", LowerWidenableConditionPass()) FUNCTION_PASS("guard-widening", GuardWideningPass()) FUNCTION_PASS("load-store-vectorizer", LoadStoreVectorizerPass()) FUNCTION_PASS("loop-simplify", LoopSimplifyPass()) FUNCTION_PASS("loop-sink", LoopSinkPass()) FUNCTION_PASS("loop-unroll-and-jam", LoopUnrollAndJamPass()) FUNCTION_PASS("lowerinvoke", LowerInvokePass()) FUNCTION_PASS("lowerswitch", LowerSwitchPass()) FUNCTION_PASS("mem2reg", PromotePass()) FUNCTION_PASS("memcpyopt", MemCpyOptPass()) FUNCTION_PASS("mergeicmps", MergeICmpsPass()) FUNCTION_PASS("nary-reassociate", NaryReassociatePass()) FUNCTION_PASS("newgvn", NewGVNPass()) FUNCTION_PASS("jump-threading", JumpThreadingPass()) FUNCTION_PASS("partially-inline-libcalls", PartiallyInlineLibCallsPass()) FUNCTION_PASS("lcssa", LCSSAPass()) FUNCTION_PASS("loop-data-prefetch", LoopDataPrefetchPass()) FUNCTION_PASS("loop-load-elim", LoopLoadEliminationPass()) FUNCTION_PASS("loop-fusion", LoopFusePass()) FUNCTION_PASS("loop-distribute", LoopDistributePass()) FUNCTION_PASS("loop-versioning", LoopVersioningPass()) FUNCTION_PASS("pgo-memop-opt", PGOMemOPSizeOpt()) FUNCTION_PASS("print", PrintFunctionPass(dbgs())) FUNCTION_PASS("print", AssumptionPrinterPass(dbgs())) FUNCTION_PASS("print", BlockFrequencyPrinterPass(dbgs())) FUNCTION_PASS("print", BranchProbabilityPrinterPass(dbgs())) FUNCTION_PASS("print", DependenceAnalysisPrinterPass(dbgs())) FUNCTION_PASS("print", DominatorTreePrinterPass(dbgs())) FUNCTION_PASS("print", PostDominatorTreePrinterPass(dbgs())) FUNCTION_PASS("print", DelinearizationPrinterPass(dbgs())) FUNCTION_PASS("print", DemandedBitsPrinterPass(dbgs())) FUNCTION_PASS("print", DominanceFrontierPrinterPass(dbgs())) FUNCTION_PASS("print", FunctionPropertiesPrinterPass(dbgs())) FUNCTION_PASS("print", InlineCostAnnotationPrinterPass(dbgs())) FUNCTION_PASS("print", InlineSizeEstimatorAnalysisPrinterPass(dbgs())) FUNCTION_PASS("print", LoopPrinterPass(dbgs())) FUNCTION_PASS("print", MemorySSAPrinterPass(dbgs())) FUNCTION_PASS("print", PhiValuesPrinterPass(dbgs())) FUNCTION_PASS("print", RegionInfoPrinterPass(dbgs())) FUNCTION_PASS("print", ScalarEvolutionPrinterPass(dbgs())) FUNCTION_PASS("print", StackSafetyPrinterPass(dbgs())) // TODO: rename to print after NPM switch FUNCTION_PASS("print-alias-sets", AliasSetsPrinterPass(dbgs())) FUNCTION_PASS("print-predicateinfo", PredicateInfoPrinterPass(dbgs())) FUNCTION_PASS("reassociate", ReassociatePass()) FUNCTION_PASS("scalarizer", ScalarizerPass()) FUNCTION_PASS("sccp", SCCPPass()) FUNCTION_PASS("simplifycfg", SimplifyCFGPass()) FUNCTION_PASS("sink", SinkingPass()) FUNCTION_PASS("slp-vectorizer", SLPVectorizerPass()) FUNCTION_PASS("speculative-execution", SpeculativeExecutionPass()) FUNCTION_PASS("spec-phis", SpeculateAroundPHIsPass()) FUNCTION_PASS("sroa", SROA()) +FUNCTION_PASS("strip-gc-relocates", StripGCRelocates()) FUNCTION_PASS("tailcallelim", TailCallElimPass()) FUNCTION_PASS("vector-combine", VectorCombinePass()) FUNCTION_PASS("verify", VerifierPass()) FUNCTION_PASS("verify", DominatorTreeVerifierPass()) FUNCTION_PASS("verify", LoopVerifierPass()) FUNCTION_PASS("verify", MemorySSAVerifierPass()) FUNCTION_PASS("verify", RegionInfoVerifierPass()) FUNCTION_PASS("verify", SafepointIRVerifierPass()) FUNCTION_PASS("verify", ScalarEvolutionVerifierPass()) FUNCTION_PASS("view-cfg", CFGViewerPass()) FUNCTION_PASS("view-cfg-only", CFGOnlyViewerPass()) FUNCTION_PASS("transform-warning", WarnMissedTransformationsPass()) FUNCTION_PASS("asan", AddressSanitizerPass(false, false, false)) FUNCTION_PASS("kasan", AddressSanitizerPass(true, false, false)) FUNCTION_PASS("msan", MemorySanitizerPass({})) FUNCTION_PASS("kmsan", MemorySanitizerPass({0, false, /*Kernel=*/true})) FUNCTION_PASS("tsan", ThreadSanitizerPass()) FUNCTION_PASS("memprof", MemProfilerPass()) #undef FUNCTION_PASS #ifndef FUNCTION_PASS_WITH_PARAMS #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) #endif FUNCTION_PASS_WITH_PARAMS("loop-unroll", [](LoopUnrollOptions Opts) { return LoopUnrollPass(Opts); }, parseLoopUnrollOptions) FUNCTION_PASS_WITH_PARAMS("msan", [](MemorySanitizerOptions Opts) { return MemorySanitizerPass(Opts); }, parseMSanPassOptions) FUNCTION_PASS_WITH_PARAMS("simplify-cfg", [](SimplifyCFGOptions Opts) { return SimplifyCFGPass(Opts); }, parseSimplifyCFGOptions) FUNCTION_PASS_WITH_PARAMS("loop-vectorize", [](LoopVectorizeOptions Opts) { return LoopVectorizePass(Opts); }, parseLoopVectorizeOptions) FUNCTION_PASS_WITH_PARAMS("mldst-motion", [](MergedLoadStoreMotionOptions Opts) { return MergedLoadStoreMotionPass(Opts); }, parseMergedLoadStoreMotionOptions) FUNCTION_PASS_WITH_PARAMS("gvn", [](GVNOptions Opts) { return GVN(Opts); }, parseGVNOptions) FUNCTION_PASS_WITH_PARAMS("print", [](StackLifetime::LivenessType Type) { return StackLifetimePrinterPass(dbgs(), Type); }, parseStackLifetimeOptions) #undef FUNCTION_PASS_WITH_PARAMS #ifndef LOOP_ANALYSIS #define LOOP_ANALYSIS(NAME, CREATE_PASS) #endif LOOP_ANALYSIS("no-op-loop", NoOpLoopAnalysis()) LOOP_ANALYSIS("access-info", LoopAccessAnalysis()) LOOP_ANALYSIS("ddg", DDGAnalysis()) LOOP_ANALYSIS("iv-users", IVUsersAnalysis()) LOOP_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC)) #undef LOOP_ANALYSIS #ifndef LOOP_PASS #define LOOP_PASS(NAME, CREATE_PASS) #endif LOOP_PASS("canon-freeze", CanonicalizeFreezeInLoopsPass()) LOOP_PASS("invalidate", InvalidateAllAnalysesPass()) LOOP_PASS("licm", LICMPass()) LOOP_PASS("loop-idiom", LoopIdiomRecognizePass()) LOOP_PASS("loop-instsimplify", LoopInstSimplifyPass()) LOOP_PASS("loop-rotate", LoopRotatePass()) LOOP_PASS("no-op-loop", NoOpLoopPass()) LOOP_PASS("print", PrintLoopPass(dbgs())) LOOP_PASS("loop-deletion", LoopDeletionPass()) LOOP_PASS("loop-flatten", LoopFlattenPass()) LOOP_PASS("loop-simplifycfg", LoopSimplifyCFGPass()) LOOP_PASS("loop-reduce", LoopStrengthReducePass()) LOOP_PASS("indvars", IndVarSimplifyPass()) LOOP_PASS("loop-unroll-full", LoopFullUnrollPass()) LOOP_PASS("print-access-info", LoopAccessInfoPrinterPass(dbgs())) LOOP_PASS("print", DDGAnalysisPrinterPass(dbgs())) LOOP_PASS("print", IVUsersPrinterPass(dbgs())) LOOP_PASS("print", LoopNestPrinterPass(dbgs())) LOOP_PASS("print", LoopCachePrinterPass(dbgs())) LOOP_PASS("loop-predication", LoopPredicationPass()) LOOP_PASS("guard-widening", GuardWideningPass()) LOOP_PASS("simple-loop-unswitch", SimpleLoopUnswitchPass()) LOOP_PASS("loop-reroll", LoopRerollPass()) #undef LOOP_PASS #ifndef LOOP_PASS_WITH_PARAMS #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) #endif LOOP_PASS_WITH_PARAMS("unswitch", [](bool NonTrivial) { return SimpleLoopUnswitchPass(NonTrivial); }, parseLoopUnswitchOptions) #undef LOOP_PASS_WITH_PARAMS diff --git a/llvm/lib/Transforms/Utils/StripGCRelocates.cpp b/llvm/lib/Transforms/Utils/StripGCRelocates.cpp index b559811d120b..1fa574f04c37 100644 --- a/llvm/lib/Transforms/Utils/StripGCRelocates.cpp +++ b/llvm/lib/Transforms/Utils/StripGCRelocates.cpp @@ -1,76 +1,88 @@ //===- StripGCRelocates.cpp - Remove gc.relocates inserted by RewriteStatePoints===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This is a little utility pass that removes the gc.relocates inserted by // RewriteStatepointsForGC. Note that the generated IR is incorrect, // but this is useful as a single pass in itself, for analysis of IR, without // the GC.relocates. The statepoint and gc.result instrinsics would still be // present. //===----------------------------------------------------------------------===// +#include "llvm/Transforms/Utils/StripGCRelocates.h" #include "llvm/IR/Function.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Statepoint.h" #include "llvm/IR/Type.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; -namespace { -struct StripGCRelocates : public FunctionPass { - static char ID; // Pass identification, replacement for typeid - StripGCRelocates() : FunctionPass(ID) { - initializeStripGCRelocatesPass(*PassRegistry::getPassRegistry()); - } - - void getAnalysisUsage(AnalysisUsage &Info) const override {} - - bool runOnFunction(Function &F) override; - -}; -char StripGCRelocates::ID = 0; -} - -bool StripGCRelocates::runOnFunction(Function &F) { +static bool stripGCRelocates(Function &F) { // Nothing to do for declarations. if (F.isDeclaration()) return false; SmallVector GCRelocates; // TODO: We currently do not handle gc.relocates that are in landing pads, // i.e. not bound to a single statepoint token. for (Instruction &I : instructions(F)) { if (auto *GCR = dyn_cast(&I)) if (isa(GCR->getOperand(0))) GCRelocates.push_back(GCR); } // All gc.relocates are bound to a single statepoint token. The order of // visiting gc.relocates for deletion does not matter. for (GCRelocateInst *GCRel : GCRelocates) { Value *OrigPtr = GCRel->getDerivedPtr(); Value *ReplaceGCRel = OrigPtr; // All gc_relocates are i8 addrspace(1)* typed, we need a bitcast from i8 // addrspace(1)* to the type of the OrigPtr, if the are not the same. if (GCRel->getType() != OrigPtr->getType()) ReplaceGCRel = new BitCastInst(OrigPtr, GCRel->getType(), "cast", GCRel); // Replace all uses of gc.relocate and delete the gc.relocate // There maybe unncessary bitcasts back to the OrigPtr type, an instcombine // pass would clear this up. GCRel->replaceAllUsesWith(ReplaceGCRel); GCRel->eraseFromParent(); } return !GCRelocates.empty(); } -INITIALIZE_PASS(StripGCRelocates, "strip-gc-relocates", +PreservedAnalyses StripGCRelocates::run(Function &F, + FunctionAnalysisManager &AM) { + if (!stripGCRelocates(F)) + return PreservedAnalyses::all(); + + // Removing gc.relocate preserves the CFG, but most other analysis probably + // need to re-run. + PreservedAnalyses PA; + PA.preserveSet(); + return PA; +} + +namespace { +struct StripGCRelocatesLegacy : public FunctionPass { + static char ID; // Pass identification, replacement for typeid + StripGCRelocatesLegacy() : FunctionPass(ID) { + initializeStripGCRelocatesLegacyPass(*PassRegistry::getPassRegistry()); + } + + void getAnalysisUsage(AnalysisUsage &Info) const override {} + + bool runOnFunction(Function &F) override { return ::stripGCRelocates(F); } +}; +char StripGCRelocatesLegacy::ID = 0; +} // namespace + +INITIALIZE_PASS(StripGCRelocatesLegacy, "strip-gc-relocates", "Strip gc.relocates inserted through RewriteStatepointsForGC", true, false) diff --git a/llvm/lib/Transforms/Utils/Utils.cpp b/llvm/lib/Transforms/Utils/Utils.cpp index a3bed38bafd0..5f975965f1a5 100644 --- a/llvm/lib/Transforms/Utils/Utils.cpp +++ b/llvm/lib/Transforms/Utils/Utils.cpp @@ -1,66 +1,66 @@ //===-- Utils.cpp - TransformUtils Infrastructure -------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file defines the common initialization infrastructure for the // TransformUtils library. // //===----------------------------------------------------------------------===// #include "llvm/Transforms/Utils.h" #include "llvm-c/Initialization.h" #include "llvm-c/Transforms/Utils.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/InitializePasses.h" #include "llvm/PassRegistry.h" using namespace llvm; /// initializeTransformUtils - Initialize all passes in the TransformUtils /// library. void llvm::initializeTransformUtils(PassRegistry &Registry) { initializeAddDiscriminatorsLegacyPassPass(Registry); initializeAssumeSimplifyPassLegacyPassPass(Registry); initializeAssumeBuilderPassLegacyPassPass(Registry); initializeBreakCriticalEdgesPass(Registry); initializeCanonicalizeAliasesLegacyPassPass(Registry); initializeCanonicalizeFreezeInLoopsPass(Registry); initializeInstNamerPass(Registry); initializeLCSSAWrapperPassPass(Registry); initializeLibCallsShrinkWrapLegacyPassPass(Registry); initializeLoopSimplifyPass(Registry); initializeLowerInvokeLegacyPassPass(Registry); initializeLowerSwitchLegacyPassPass(Registry); initializeNameAnonGlobalLegacyPassPass(Registry); initializePromoteLegacyPassPass(Registry); initializeStripNonLineTableDebugLegacyPassPass(Registry); initializeUnifyFunctionExitNodesPass(Registry); initializeMetaRenamerPass(Registry); - initializeStripGCRelocatesPass(Registry); + initializeStripGCRelocatesLegacyPass(Registry); initializePredicateInfoPrinterLegacyPassPass(Registry); initializeInjectTLIMappingsLegacyPass(Registry); initializeFixIrreduciblePass(Registry); initializeUnifyLoopExitsPass(Registry); initializeUniqueInternalLinkageNamesLegacyPassPass(Registry); } /// LLVMInitializeTransformUtils - C binding for initializeTransformUtilsPasses. void LLVMInitializeTransformUtils(LLVMPassRegistryRef R) { initializeTransformUtils(*unwrap(R)); } void LLVMAddLowerSwitchPass(LLVMPassManagerRef PM) { unwrap(PM)->add(createLowerSwitchPass()); } void LLVMAddPromoteMemoryToRegisterPass(LLVMPassManagerRef PM) { unwrap(PM)->add(createPromoteMemoryToRegisterPass()); } void LLVMAddAddDiscriminatorsPass(LLVMPassManagerRef PM) { unwrap(PM)->add(createAddDiscriminatorsPass()); } diff --git a/llvm/test/Transforms/Util/strip-gc-relocates.ll b/llvm/test/Transforms/Util/strip-gc-relocates.ll index 77b8ffd079e9..9aa18ff7bf87 100644 --- a/llvm/test/Transforms/Util/strip-gc-relocates.ll +++ b/llvm/test/Transforms/Util/strip-gc-relocates.ll @@ -1,120 +1,121 @@ ; RUN: opt -S -strip-gc-relocates -instcombine < %s | FileCheck %s +; RUN: opt -S -passes=strip-gc-relocates,instcombine < %s | FileCheck %s ; test utility/debugging pass which removes gc.relocates, inserted by -rewrite-statepoints-for-gc declare void @use_obj32(i32 addrspace(1)*) "gc-leaf-function" declare void @g() declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...) declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32, i32) #0 declare void @do_safepoint() declare i32 addrspace(1)* @new_instance() #1 ; Simple case: remove gc.relocate define i32 addrspace(1)* @test1(i32 addrspace(1)* %arg) gc "statepoint-example" { entry: ; CHECK-LABEL: test1 ; CHECK: gc.statepoint ; CHECK-NOT: gc.relocate ; CHECK: ret i32 addrspace(1)* %arg %statepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @g, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i32 addrspace(1)* %arg), "deopt" (i32 100)] %arg.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 0, i32 0) ; (%arg, %arg) %arg.relocated.casted = bitcast i8 addrspace(1)* %arg.relocated to i32 addrspace(1)* ret i32 addrspace(1)* %arg.relocated.casted } ; Remove gc.relocates in presence of nested relocates. define void @test2(i32 addrspace(1)* %base) gc "statepoint-example" { entry: ; CHECK-LABEL: test2 ; CHECK: statepoint ; CHECK-NOT: gc.relocate ; CHECK: call void @use_obj32(i32 addrspace(1)* %ptr.gep1) ; CHECK: call void @use_obj32(i32 addrspace(1)* %ptr.gep1) %ptr.gep = getelementptr i32, i32 addrspace(1)* %base, i32 15 %ptr.gep1 = getelementptr i32, i32 addrspace(1)* %ptr.gep, i32 15 %statepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i32 addrspace(1)* %ptr.gep1, i32 addrspace(1)* %base)] %ptr.gep1.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 1, i32 0) ; (%base, %ptr.gep1) %ptr.gep1.relocated.casted = bitcast i8 addrspace(1)* %ptr.gep1.relocated to i32 addrspace(1)* %base.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 1, i32 1) ; (%base, %base) %base.relocated.casted = bitcast i8 addrspace(1)* %base.relocated to i32 addrspace(1)* call void @use_obj32(i32 addrspace(1)* %ptr.gep1.relocated.casted) %statepoint_token1 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i32 addrspace(1)* %ptr.gep1.relocated.casted, i32 addrspace(1)* %base.relocated.casted)] %ptr.gep1.relocated2 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token1, i32 1, i32 0) ; (%base.relocated.casted, %ptr.gep1.relocated.casted) %ptr.gep1.relocated2.casted = bitcast i8 addrspace(1)* %ptr.gep1.relocated2 to i32 addrspace(1)* %base.relocated3 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token1, i32 1, i32 1) ; (%base.relocated.casted, %base.relocated.casted) %base.relocated3.casted = bitcast i8 addrspace(1)* %base.relocated3 to i32 addrspace(1)* call void @use_obj32(i32 addrspace(1)* %ptr.gep1.relocated2.casted) ret void } ; landing pad gc.relocates removed by instcombine since it has no uses. define i32 addrspace(1)* @test3(i32 addrspace(1)* %arg) gc "statepoint-example" personality i32 8 { ; CHECK-LABEL: test3( ; CHECK: gc.statepoint ; CHECK-LABEL: normal_dest: ; CHECK-NOT: gc.relocate ; CHECK: ret i32 addrspace(1)* %arg ; CHECK-LABEL: unwind_dest: ; CHECK-NOT: gc.relocate entry: %statepoint_token = invoke token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @g, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i32 addrspace(1)* %arg), "deopt" (i32 100)] to label %normal_dest unwind label %unwind_dest normal_dest: ; preds = %entry %arg.relocated1 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 0, i32 0) ; (%arg, %arg) %arg.relocated1.casted = bitcast i8 addrspace(1)* %arg.relocated1 to i32 addrspace(1)* ret i32 addrspace(1)* %arg.relocated1.casted unwind_dest: ; preds = %entry %lpad = landingpad token cleanup %arg.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %lpad, i32 0, i32 0) ; (%arg, %arg) %arg.relocated.casted = bitcast i8 addrspace(1)* %arg.relocated to i32 addrspace(1)* resume token undef } ; in presence of phi define void @test4(i1 %cond) gc "statepoint-example" { ; CHECK-LABEL: test4 entry: %base1 = call i32 addrspace(1)* @new_instance() %base2 = call i32 addrspace(1)* @new_instance() br i1 %cond, label %here, label %there here: ; preds = %entry br label %merge there: ; preds = %entry br label %merge merge: ; preds = %there, %here ; CHECK-LABEL: merge: ; CHECK-NOT: gc.relocate ; CHECK: %ptr.gep.remat = getelementptr i32, i32 addrspace(1)* %basephi.base %basephi.base = phi i32 addrspace(1)* [ %base1, %here ], [ %base2, %there ], !is_base_value !0 %basephi = phi i32 addrspace(1)* [ %base1, %here ], [ %base2, %there ] %ptr.gep = getelementptr i32, i32 addrspace(1)* %basephi, i32 15 %statepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i32 addrspace(1)* %basephi.base)] %basephi.base.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 0, i32 0) ; (%basephi.base, %basephi.base) %basephi.base.relocated.casted = bitcast i8 addrspace(1)* %basephi.base.relocated to i32 addrspace(1)* %ptr.gep.remat = getelementptr i32, i32 addrspace(1)* %basephi.base.relocated.casted, i32 15 call void @use_obj32(i32 addrspace(1)* %ptr.gep.remat) ret void } ; The gc.relocate type is different from %arg, but removing the gc.relocate, ; needs a bitcast to be added from i32 addrspace(1)* to i8 addrspace(1)* define i8 addrspace(1)* @test5(i32 addrspace(1)* %arg) gc "statepoint-example" { entry: ; CHECK-LABEL: test5 ; CHECK: gc.statepoint ; CHECK-NOT: gc.relocate %statepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @g, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i32 addrspace(1)* %arg), "deopt" (i32 100)] %arg.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 0, i32 0) ; (%arg, %arg) ret i8 addrspace(1)* %arg.relocated } attributes #0 = { nounwind readonly } attributes #1 = { nounwind "gc-leaf-function" } !0 = !{}