[Attributor] AACallEdges, Add a way to ask nonasm unknown callees

[kuter]

Summary

This patch adds a feature to AACallEdges AbstractAttribute that allows
users to ask if there is a unknown callee that isn't a inline assembly.
This feature is needed by some of it's users.

[jdoerfert]

LG

llvm/include/llvm/Transforms/IPO/Attributor.h

struct AACallEdges : public StateWrapper<BooleanState, AbstractAttribute>,
   using Base = StateWrapper<BooleanState, AbstractAttribute>;
 
   AACallEdges(const IRPosition &IRP, Attributor &A)
       : Base(IRP), AACallGraphNode(A) {}
 
   /// Get the optimistic edges.
   virtual const SetVector<Function *> &getOptimisticEdges() const = 0;
 
-  /// Is there in this function call with a unknown Callee.
+  /// Is there any call with a unknown callee.
   virtual bool hasUnknownCallee() const = 0;
 
+  /// Is there any call with a unknown callee, excluding any inline asm.
+  virtual bool hasNonAsmUnknownCallee() const = 0;
+
   /// Iterator for exploring the call graph.
   AACallEdgeIterator optimisticEdgesBegin() const override {
     return AACallEdgeIterator(A, getOptimisticEdges().begin());
   }
 
   /// Iterator for exploring the call graph.
   AACallEdgeIterator optimisticEdgesEnd() const override {
     return AACallEdgeIterator(A, getOptimisticEdges().end());

llvm/lib/Transforms/IPO/AttributorAttributes.cpp

 struct AACallEdgesFunction : public AACallEdges {
   AACallEdgesFunction(const IRPosition &IRP, Attributor &A)
       : AACallEdges(IRP, A) {}
 
   /// See AbstractAttribute::updateImpl(...).
   ChangeStatus updateImpl(Attributor &A) override {
     ChangeStatus Change = ChangeStatus::UNCHANGED;
     bool OldHasUnknownCallee = HasUnknownCallee;
+    bool OldHasUnknownCalleeNonAsm = HasUnknownCalleeNonAsm;
 
     auto AddCalledFunction = [&](Function *Fn) {
       if (CalledFunctions.insert(Fn)) {
         Change = ChangeStatus::CHANGED;
         LLVM_DEBUG(dbgs() << "[AACallEdges] New call edge: " << Fn->getName()
                           << "\n");
       }
     };
 
     auto VisitValue = [&](Value &V, const Instruction *CtxI, bool &HasUnknown,
                           bool Stripped) -> bool {
       if (Function *Fn = dyn_cast<Function>(&V)) {
         AddCalledFunction(Fn);
       } else {
         LLVM_DEBUG(dbgs() << "[AACallEdges] Unrecognized value: " << V << "\n");
         HasUnknown = true;
+        HasUnknownCalleeNonAsm = true;
       }
 
       // Explore all values.
       return true;
     };
 
     // Process any value that we might call.
     auto ProcessCalledOperand = [&](Value *V, Instruction *Ctx) {
-      if (!genericValueTraversal<bool>(A, IRPosition::value(*V), *this,
-                                       HasUnknownCallee, VisitValue, nullptr,
-                                       false))
+      if (!genericValueTraversal<bool>(
+              A, IRPosition::value(*V), *this, HasUnknownCallee, VisitValue,
+              nullptr, false)) {
         // If we haven't gone through all values, assume that there are unknown
         // callees.
         HasUnknownCallee = true;
+        HasUnknownCalleeNonAsm = true;
+      }
     };
 
     auto ProcessCallInst = [&](Instruction &Inst) {
       CallBase &CB = static_cast<CallBase &>(Inst);
+      if (CB.isInlineAsm()) {
+        HasUnknownCallee = true;
+        return true;
+      }
 
       // Process callee metadata if available.
       if (auto *MD = Inst.getMetadata(LLVMContext::MD_callees)) {
         for (auto &Op : MD->operands()) {
           Function *Callee = mdconst::extract_or_null<Function>(Op);
           if (Callee)
             AddCalledFunction(Callee);
         }
auto ProcessCallInst = [&](Instruction &Inst) {
         ProcessCalledOperand(U->get(), &Inst);
 
       return true;
     };
 
     // Visit all callable instructions.
     bool UsedAssumedInformation = false;
     if (!A.checkForAllCallLikeInstructions(ProcessCallInst, *this,
-                                           UsedAssumedInformation))
+                                           UsedAssumedInformation)) {
       // If we haven't looked at all call like instructions, assume that there
       // are unknown callees.
       HasUnknownCallee = true;
+      HasUnknownCalleeNonAsm = true;
+    }
+
     // Track changes.
-    if (OldHasUnknownCallee != HasUnknownCallee)
+    if (OldHasUnknownCallee != HasUnknownCallee ||
+        OldHasUnknownCalleeNonAsm != HasUnknownCalleeNonAsm)
       Change = ChangeStatus::CHANGED;
 
     return Change;
   }
 
   virtual const SetVector<Function *> &getOptimisticEdges() const override {
     return CalledFunctions;
   };
 
   virtual bool hasUnknownCallee() const override { return HasUnknownCallee; }
 
+  virtual bool hasNonAsmUnknownCallee() const override {
+    return HasUnknownCalleeNonAsm;
+  }
+
   const std::string getAsStr() const override {
     return "CallEdges[" + std::to_string(HasUnknownCallee) + "," +
            std::to_string(CalledFunctions.size()) + "]";
   }
 
   void trackStatistics() const override {}
 
   /// Optimistic set of functions that might be called by this function.
   SetVector<Function *> CalledFunctions;
 
-  /// Does this function have a call to a function that we don't know about.
+  /// Is there any call with a unknown callee.
   bool HasUnknownCallee = false;
+
+  /// Is there any call with a unknown callee, excluding any inline asm.
+  bool HasUnknownCalleeNonAsm = false;
 };
 
 struct AAFunctionReachabilityFunction : public AAFunctionReachability {
   AAFunctionReachabilityFunction(const IRPosition &IRP, Attributor &A)
       : AAFunctionReachability(IRP, A) {}
 
   bool canReach(Attributor &A, Function *Fn) const override {
     // Assume that we can reach any function if we can reach a call with