[AIX] Support init priority attribute

[aaron.ballman]

clang/lib/CodeGen/CGDeclCXX.cpp

574

This isn't quite true -- it's enforced by sema for user code, but not for code that identifies itself as a system header. I see that this is existing code that's moved around a bit, but we may want to add an assertion here just in case.

[Xiangling_L]

Add assertion to make sure we have correct priority value in range;

[jasonliu]

LGTM.

clang/lib/CodeGen/CGDeclCXX.cpp

663

Just noting that this trunk of code have very similar logic counter part in EmitCXXGlobalInitFunc. It seems there is no easy way to common them up given the current way of implementing it. It may require a non-trivial amount of refactoring for us to common up the similar logic between init and cleanup.

[Xiangling_L]

clang/lib/CodeGen/CGDeclCXX.cpp

663

Yes, I thought about this as well but it looks if we'd like to common it up, we may need to create some template function.

clang/lib/CodeGen/CGDeclCXX.cpp

if (D->getTLSKind()) {
     // FIXME: Should we support init_priority for thread_local?
     // FIXME: We only need to register one __cxa_thread_atexit function for the
     // entire TU.
     CXXThreadLocalInits.push_back(Fn);
     CXXThreadLocalInitVars.push_back(D);
   } else if (PerformInit && ISA) {
     EmitPointerToInitFunc(D, Addr, Fn, ISA);
   } else if (auto *IPA = D->getAttr<InitPriorityAttr>()) {
-    OrderGlobalInits Key(IPA->getPriority(), PrioritizedCXXGlobalInits.size());
+    OrderGlobalInitsOrStermFinalizers Key(IPA->getPriority(),
+                                          PrioritizedCXXGlobalInits.size());
     PrioritizedCXXGlobalInits.push_back(std::make_pair(Key, Fn));
   } else if (isTemplateInstantiation(D->getTemplateSpecializationKind()) ||
              getContext().GetGVALinkageForVariable(D) == GVA_DiscardableODR) {
     // C++ [basic.start.init]p2:
     //   Definitions of explicitly specialized class template static data
     //   members have ordered initialization. Other class template static data
     //   members (i.e., implicitly or explicitly instantiated specializations)
     //   have unordered initialization.
for (size_t i = 0; i < FileName.size(); ++i) {
     // to be the set of C preprocessing numbers.
     if (!isPreprocessingNumberBody(FileName[i]))
       FileName[i] = '_';
   }
 
   return FileName;
 }
 
+static std::string getPrioritySuffix(unsigned int Priority) {
+  assert(Priority <= 65535 && "Priority should always be <= 65535.");
+
+  // Compute the function suffix from priority. Prepend with zeroes to make
+  // sure the function names are also ordered as priorities.
+  std::string PrioritySuffix = llvm::utostr(Priority);
+  PrioritySuffix = std::string(6 - PrioritySuffix.size(), '0') + PrioritySuffix;
+
+  return PrioritySuffix;
+}
+
 void
 CodeGenModule::EmitCXXGlobalInitFunc() {
   while (!CXXGlobalInits.empty() && !CXXGlobalInits.back())
     CXXGlobalInits.pop_back();
 
   if (CXXGlobalInits.empty() && PrioritizedCXXGlobalInits.empty())
     return;
 
   llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
   const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
 
-  const bool UseSinitAndSterm = getCXXABI().useSinitAndSterm();
   // Create our global prioritized initialization function.
   if (!PrioritizedCXXGlobalInits.empty()) {
-    assert(!UseSinitAndSterm && "Prioritized sinit and sterm functions are not"
-                                " supported yet.");
-
     SmallVector<llvm::Function *, 8> LocalCXXGlobalInits;
     llvm::array_pod_sort(PrioritizedCXXGlobalInits.begin(),
                          PrioritizedCXXGlobalInits.end());
     // Iterate over "chunks" of ctors with same priority and emit each chunk
     // into separate function. Note - everything is sorted first by priority,
     // second - by lex order, so we emit ctor functions in proper order.
     for (SmallVectorImpl<GlobalInitData >::iterator
            I = PrioritizedCXXGlobalInits.begin(),
            E = PrioritizedCXXGlobalInits.end(); I != E; ) {
       SmallVectorImpl<GlobalInitData >::iterator
         PrioE = std::upper_bound(I + 1, E, *I, GlobalInitPriorityCmp());
 
       LocalCXXGlobalInits.clear();
-      unsigned Priority = I->first.priority;
-      // Compute the function suffix from priority. Prepend with zeroes to make
-      // sure the function names are also ordered as priorities.
-      std::string PrioritySuffix = llvm::utostr(Priority);
-      // Priority is always <= 65535 (enforced by sema).
-      PrioritySuffix = std::string(6-PrioritySuffix.size(), '0')+PrioritySuffix;
+
+      unsigned int Priority = I->first.priority;
       llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
-          FTy, "_GLOBAL__I_" + PrioritySuffix, FI);
+          FTy, "_GLOBAL__I_" + getPrioritySuffix(Priority), FI);
 
       for (; I < PrioE; ++I)
         LocalCXXGlobalInits.push_back(I->second);
 
       CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, LocalCXXGlobalInits);
       AddGlobalCtor(Fn, Priority);
     }
     PrioritizedCXXGlobalInits.clear();
   }
 
-  if (UseSinitAndSterm && CXXGlobalInits.empty())
+  if (getCXXABI().useSinitAndSterm() && CXXGlobalInits.empty())
     return;
 
   // Include the filename in the symbol name. Including "sub_" matches gcc
   // and makes sure these symbols appear lexicographically behind the symbols
   // with priority emitted above.
   llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
       FTy, llvm::Twine("_GLOBAL__sub_I_", getTransformedFileName(getModule())),
       FI);
if (getLangOpts().HIP) {
     Fn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
     Fn->addFnAttr("device-init");
   }
 
   CXXGlobalInits.clear();
 }
 
 void CodeGenModule::EmitCXXGlobalCleanUpFunc() {
-  if (CXXGlobalDtorsOrStermFinalizers.empty())
+  if (CXXGlobalDtorsOrStermFinalizers.empty() &&
+      PrioritizedCXXStermFinalizers.empty())
     return;
 
   llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
   const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
 
+  // Create our global prioritized cleanup function.
+  if (!PrioritizedCXXStermFinalizers.empty()) {
+    SmallVector<CXXGlobalDtorsOrStermFinalizer_t, 8> LocalCXXStermFinalizers;
+    llvm::array_pod_sort(PrioritizedCXXStermFinalizers.begin(),
+                         PrioritizedCXXStermFinalizers.end());
+    // Iterate over "chunks" of dtors with same priority and emit each chunk
+    // into separate function. Note - everything is sorted first by priority,
+    // second - by lex order, so we emit dtor functions in proper order.
+    for (SmallVectorImpl<StermFinalizerData>::iterator
+             I = PrioritizedCXXStermFinalizers.begin(),
+             E = PrioritizedCXXStermFinalizers.end();
+         I != E;) {
+      SmallVectorImpl<StermFinalizerData>::iterator PrioE =
+          std::upper_bound(I + 1, E, *I, StermFinalizerPriorityCmp());
+
+      LocalCXXStermFinalizers.clear();
+
+      unsigned int Priority = I->first.priority;
+      llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
+          FTy, "_GLOBAL__a_" + getPrioritySuffix(Priority), FI);
+
+      for (; I < PrioE; ++I) {
+        llvm::FunctionCallee DtorFn = I->second;
+        LocalCXXStermFinalizers.emplace_back(DtorFn.getFunctionType(),
+                                             DtorFn.getCallee(), nullptr);
+      }
+
+      CodeGenFunction(*this).GenerateCXXGlobalCleanUpFunc(
+          Fn, LocalCXXStermFinalizers);
+      AddGlobalDtor(Fn, Priority);
+    }
+    PrioritizedCXXStermFinalizers.clear();
+  }
+
+  if (CXXGlobalDtorsOrStermFinalizers.empty())
+    return;
+
   // Create our global cleanup function.
   llvm::Function *Fn =
       CreateGlobalInitOrCleanUpFunction(FTy, "_GLOBAL__D_a", FI);
 
   CodeGenFunction(*this).GenerateCXXGlobalCleanUpFunc(
       Fn, CXXGlobalDtorsOrStermFinalizers);
   AddGlobalDtor(Fn);
   CXXGlobalDtorsOrStermFinalizers.clear();
CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn,
     }
   }
 
   FinishFunction();
 }
 
 void CodeGenFunction::GenerateCXXGlobalCleanUpFunc(
     llvm::Function *Fn,
-    const std::vector<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
-                                 llvm::Constant *>> &DtorsOrStermFinalizers) {
+    ArrayRef<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
+                        llvm::Constant *>>
+        DtorsOrStermFinalizers) {
   {
     auto NL = ApplyDebugLocation::CreateEmpty(*this);
     StartFunction(GlobalDecl(), getContext().VoidTy, Fn,
                   getTypes().arrangeNullaryFunction(), FunctionArgList());
     // Emit an artificial location for this function.
     auto AL = ApplyDebugLocation::CreateArtificial(*this);
 
     // Emit the cleanups, in reverse order from construction.

clang/lib/CodeGen/CodeGenFunction.h

public:
   GenerateCXXGlobalInitFunc(llvm::Function *Fn,
                             ArrayRef<llvm::Function *> CXXThreadLocals,
                             ConstantAddress Guard = ConstantAddress::invalid());
 
   /// GenerateCXXGlobalCleanUpFunc - Generates code for cleaning up global
   /// variables.
   void GenerateCXXGlobalCleanUpFunc(
       llvm::Function *Fn,
-      const std::vector<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
-                                   llvm::Constant *>> &DtorsOrStermFinalizers);
+      ArrayRef<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
+                          llvm::Constant *>>
+          DtorsOrStermFinalizers);
 
   void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn,
                                         const VarDecl *D,
                                         llvm::GlobalVariable *Addr,
                                         bool PerformInit);
 
   void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest);
 

clang/lib/CodeGen/CodeGenModule.h

 class CoverageMappingModuleGen;
 class TargetCodeGenInfo;
 
 enum ForDefinition_t : bool {
   NotForDefinition = false,
   ForDefinition = true
 };
 
-struct OrderGlobalInits {
+struct OrderGlobalInitsOrStermFinalizers {
   unsigned int priority;
   unsigned int lex_order;
-  OrderGlobalInits(unsigned int p, unsigned int l)
+  OrderGlobalInitsOrStermFinalizers(unsigned int p, unsigned int l)
       : priority(p), lex_order(l) {}
 
-  bool operator==(const OrderGlobalInits &RHS) const {
+  bool operator==(const OrderGlobalInitsOrStermFinalizers &RHS) const {
     return priority == RHS.priority && lex_order == RHS.lex_order;
   }
 
-  bool operator<(const OrderGlobalInits &RHS) const {
+  bool operator<(const OrderGlobalInitsOrStermFinalizers &RHS) const {
     return std::tie(priority, lex_order) <
            std::tie(RHS.priority, RHS.lex_order);
   }
 };
 
 struct ObjCEntrypoints {
   ObjCEntrypoints() { memset(this, 0, sizeof(*this)); }
 
private:
 
   /// When a C++ decl with an initializer is deferred, null is
   /// appended to CXXGlobalInits, and the index of that null is placed
   /// here so that the initializer will be performed in the correct
   /// order. Once the decl is emitted, the index is replaced with ~0U to ensure
   /// that we don't re-emit the initializer.
   llvm::DenseMap<const Decl*, unsigned> DelayedCXXInitPosition;
 
-  typedef std::pair<OrderGlobalInits, llvm::Function*> GlobalInitData;
+  typedef std::pair<OrderGlobalInitsOrStermFinalizers, llvm::Function *>
+      GlobalInitData;
 
   struct GlobalInitPriorityCmp {
     bool operator()(const GlobalInitData &LHS,
                     const GlobalInitData &RHS) const {
       return LHS.first.priority < RHS.first.priority;
     }
   };
 
   /// Global variables with initializers whose order of initialization is set by
   /// init_priority attribute.
   SmallVector<GlobalInitData, 8> PrioritizedCXXGlobalInits;
 
   /// Global destructor functions and arguments that need to run on termination.
   /// When UseSinitAndSterm is set, it instead contains sterm finalizer
   /// functions, which also run on unloading a shared library.
-  std::vector<
-      std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH, llvm::Constant *>>
+  typedef std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
+                     llvm::Constant *>
+      CXXGlobalDtorsOrStermFinalizer_t;
+  SmallVector<CXXGlobalDtorsOrStermFinalizer_t, 8>
       CXXGlobalDtorsOrStermFinalizers;
 
+  typedef std::pair<OrderGlobalInitsOrStermFinalizers, llvm::Function *>
+      StermFinalizerData;
+
+  struct StermFinalizerPriorityCmp {
+    bool operator()(const StermFinalizerData &LHS,
+                    const StermFinalizerData &RHS) const {
+      return LHS.first.priority < RHS.first.priority;
+    }
+  };
+
+  /// Global variables with sterm finalizers whose order of initialization is
+  /// set by init_priority attribute.
+  SmallVector<StermFinalizerData, 8> PrioritizedCXXStermFinalizers;
+
   /// The complete set of modules that has been imported.
   llvm::SetVector<clang::Module *> ImportedModules;
 
   /// The set of modules for which the module initializers
   /// have been emitted.
   llvm::SmallPtrSet<clang::Module *, 16> EmittedModuleInitializers;
 
   /// A vector of metadata strings for linker options.
public:
   }
 
   /// Add an sterm finalizer to its own llvm.global_dtors entry.
   void AddCXXStermFinalizerToGlobalDtor(llvm::Function *StermFinalizer,
                                         int Priority) {
     AddGlobalDtor(StermFinalizer, Priority);
   }
 
+  void AddCXXPrioritizedStermFinalizerEntry(llvm::Function *StermFinalizer,
+                                            int Priority) {
+    OrderGlobalInitsOrStermFinalizers Key(Priority,
+                                          PrioritizedCXXStermFinalizers.size());
+    PrioritizedCXXStermFinalizers.push_back(
+        std::make_pair(Key, StermFinalizer));
+  }
+
   /// Create or return a runtime function declaration with the specified type
   /// and name. If \p AssumeConvergent is true, the call will have the
   /// convergent attribute added.
   llvm::FunctionCallee
   CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name,
                         llvm::AttributeList ExtraAttrs = llvm::AttributeList(),
                         bool Local = false, bool AssumeConvergent = false);
 

clang/lib/CodeGen/ItaniumCXXABI.cpp

void XLCXXABI::emitCXXStermFinalizer(const VarDecl &D, llvm::Function *dtorStub,
 
   // Make sure the call and the callee agree on calling convention.
   CI->setCallingConv(dtorStub->getCallingConv());
 
   CGF.EmitBlock(EndBlock);
 
   CGF.FinishFunction();
 
-  assert(!D.getAttr<InitPriorityAttr>() &&
-         "Prioritized sinit and sterm functions are not yet supported.");
-
-  if (isTemplateInstantiation(D.getTemplateSpecializationKind()) ||
-      getContext().GetGVALinkageForVariable(&D) == GVA_DiscardableODR)
+  if (auto *IPA = D.getAttr<InitPriorityAttr>()) {
+    CGM.AddCXXPrioritizedStermFinalizerEntry(StermFinalizer,
+                                             IPA->getPriority());
+  } else if (isTemplateInstantiation(D.getTemplateSpecializationKind()) ||
+             getContext().GetGVALinkageForVariable(&D) == GVA_DiscardableODR) {
     // According to C++ [basic.start.init]p2, class template static data
     // members (i.e., implicitly or explicitly instantiated specializations)
     // have unordered initialization. As a consequence, we can put them into
     // their own llvm.global_dtors entry.
     CGM.AddCXXStermFinalizerToGlobalDtor(StermFinalizer, 65535);
-  else
+  } else {
     CGM.AddCXXStermFinalizerEntry(StermFinalizer);
   }
+}

clang/lib/Sema/SemaDeclAttr.cpp

case ParsedAttr::AT_ReqdWorkGroupSize:
     break;
   case ParsedAttr::AT_OpenCLIntelReqdSubGroupSize:
     handleSubGroupSize(S, D, AL);
     break;
   case ParsedAttr::AT_VecTypeHint:
     handleVecTypeHint(S, D, AL);
     break;
   case ParsedAttr::AT_InitPriority:
-    if (S.Context.getTargetInfo().getTriple().isOSAIX())
-      llvm::report_fatal_error(
-          "'init_priority' attribute is not yet supported on AIX");
-    else
       handleInitPriorityAttr(S, D, AL);
     break;
   case ParsedAttr::AT_Packed:
     handlePackedAttr(S, D, AL);
     break;
   case ParsedAttr::AT_PreferredName:
     handlePreferredName(S, D, AL);
     break;

clang/test/CodeGen/aix-init-priority-attribute.cpp

-// RUN: not %clang_cc1 -triple powerpc-ibm-aix-xcoff -x c++ -emit-llvm < %s \
-// RUN:     2>&1 | \
+// RUN: %clang_cc1 -triple powerpc-ibm-aix-xcoff -x c++ -emit-llvm < %s | \
 // RUN:   FileCheck %s
-// RUN: not %clang_cc1 -triple powerpc64-ibm-aix-xcoff -x c++ -emit-llvm < %s \
-// RUN:     2>&1 | \
+// RUN: %clang_cc1 -triple powerpc64-ibm-aix-xcoff -x c++ -emit-llvm < %s | \
 // RUN:   FileCheck %s
 
-class test {
-  int a;
-
-public:
-  test(int c) { a = c; }
-  ~test() { a = 0; }
+struct test {
+  test() {}
+  ~test() {}
 };
 
-__attribute__((init_priority(2000)))
-test t(1);
+__attribute__((init_priority(200)))
+test t1;
+__attribute__((init_priority(200)))
+test t2;
+__attribute__((init_priority(300)))
+test t3;
+__attribute__((init_priority(150)))
+test t4;
+test t5;
+
+// CHECK: @llvm.global_ctors = appending global [4 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 150, void ()* @_GLOBAL__I_000150, i8* null }, { i32, void ()*, i8* } { i32 200, void ()* @_GLOBAL__I_000200, i8* null }, { i32, void ()*, i8* } { i32 300, void ()* @_GLOBAL__I_000300, i8* null }, { i32, void ()*, i8* } { i32 65535, void ()* @_GLOBAL__sub_I__, i8* null }]
+// CHECK: @llvm.global_dtors = appending global [4 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 150, void ()* @_GLOBAL__a_000150, i8* null }, { i32, void ()*, i8* } { i32 200, void ()* @_GLOBAL__a_000200, i8* null }, { i32, void ()*, i8* } { i32 300, void ()* @_GLOBAL__a_000300, i8* null }, { i32, void ()*, i8* } { i32 65535, void ()* @_GLOBAL__D_a, i8* null }]
+
+// CHECK: define internal void @_GLOBAL__I_000150() [[ATTR:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   call void @__cxx_global_var_init.3()
+// CHECK:   ret void
+// CHECK: }
+
+// CHECK: define internal void @_GLOBAL__I_000200() [[ATTR:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   call void @__cxx_global_var_init()
+// CHECK:   call void @__cxx_global_var_init.1()
+// CHECK:   ret void
+// CHECK: }
+
+// CHECK: define internal void @_GLOBAL__I_000300() [[ATTR:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   call void @__cxx_global_var_init.2()
+// CHECK:   ret void
+// CHECK: }
+
+// CHECK: define internal void @_GLOBAL__sub_I__() [[ATTR:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   call void @__cxx_global_var_init.4()
+// CHECK:   ret void
+// CHECK: }
+
+// CHECK: define internal void @_GLOBAL__a_000150() [[ATTR:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   call void @__finalize_t4()
+// CHECK:   ret void
+// CHECK: }
+
+// CHECK: define internal void @_GLOBAL__a_000200() [[ATTR:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   call void @__finalize_t2()
+// CHECK:   call void @__finalize_t1()
+// CHECK:   ret void
+// CHECK: }
+
+// CHECK: define internal void @_GLOBAL__a_000300() [[ATTR:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   call void @__finalize_t3()
+// CHECK:   ret void
+// CHECK: }
 
-// CHECK: fatal error: error in backend: 'init_priority' attribute is not yet supported on AIX
+// CHECK: define internal void @_GLOBAL__D_a() [[ATTR:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   call void @__finalize_t5()
+// CHECK:   ret void
+// CHECK: }