[DataCollection] Allow choosing between collection categories

[johannes]

Summary

This adds a list of collection categories to
ClangDataCollectorsEmitter.cpp, in an aim to eventually unify the
various AST hashing mechanisms (each one will get its own category).

In the various *DataCollectors.td files we can specify for each AST node
a way to collect data specific to a collection category. Clients can
then define COLLECT_FROM_* to collect data as defined in this category.

include/clang/AST/StmtDataCollectors.td

-class Stmt {
-  code Code = [{
+class Collector {
+  code TypeIIClone = [{}];
+}
+
+def Stmt : Collector {
+  let TypeIIClone = [{
     addData(S->getStmtClass());
     // This ensures that non-macro-generated code isn't identical to
     // macro-generated code.
     addData(data_collection::getMacroStack(S->getLocStart(), Context));
     addData(data_collection::getMacroStack(S->getLocEnd(), Context));
   }];
 }
 
-class Expr {
-  code Code = [{
+def Expr : Collector {
+  let TypeIIClone = [{
     addData(S->getType());
   }];
 }
 
 //--- Builtin functionality ----------------------------------------------//
-class ArrayTypeTraitExpr {
-  code Code = [{
+def ArrayTypeTraitExpr : Collector {
+  let TypeIIClone = [{
     addData(S->getTrait());
   }];
 }
-class ExpressionTraitExpr {
-  code Code = [{
+def ExpressionTraitExpr : Collector {
+  let TypeIIClone = [{
     addData(S->getTrait());
   }];
 }
-class PredefinedExpr {
-  code Code = [{
+def PredefinedExpr : Collector {
+  let TypeIIClone = [{
     addData(S->getIdentType());
   }];
 }
-class TypeTraitExpr {
-  code Code = [{
+def TypeTraitExpr : Collector {
+  let TypeIIClone = [{
     addData(S->getTrait());
     for (unsigned i = 0; i < S->getNumArgs(); ++i)
       addData(S->getArg(i)->getType());
   }];
 }
 
 //--- Calls --------------------------------------------------------------//
-class CallExpr {
-  code Code = [{
+def CallExpr : Collector {
+  let TypeIIClone = [{
     // Function pointers don't have a callee and we just skip hashing it.
     if (const FunctionDecl *D = S->getDirectCallee()) {
       // If the function is a template specialization, we also need to handle
       // the template arguments as they are not included in the qualified name.
       if (auto Args = D->getTemplateSpecializationArgs()) {
         std::string ArgString;
 
         // Print all template arguments into ArgString
         llvm::raw_string_ostream OS(ArgString);
         for (unsigned i = 0; i < Args->size(); ++i) {
           Args->get(i).print(Context.getLangOpts(), OS);
           // Add a padding character so that 'foo<X, XX>()' != 'foo<XX, X>()'.
           OS << '\n';
         }
         OS.flush();
 
         addData(ArgString);
       }
       addData(D->getQualifiedNameAsString());
     }
   }];
 }
 
 //--- Value references ---------------------------------------------------//
-class DeclRefExpr {
-  code Code = [{
+def DeclRefExpr : Collector {
+  let TypeIIClone = [{
     addData(S->getDecl()->getQualifiedNameAsString());
   }];
 }
-class MemberExpr {
-  code Code = [{
+def MemberExpr : Collector {
+  let TypeIIClone = [{
     addData(S->getMemberDecl()->getName());
   }];
 }
 
 //--- Literals -----------------------------------------------------------//
-class IntegerLiteral {
-  code Code = [{
+def IntegerLiteral : Collector {
+  let TypeIIClone = [{
     addData(llvm::hash_value(S->getValue()));
   }];
 }
-class FloatingLiteral {
-  code Code = [{
+def FloatingLiteral : Collector {
+  let TypeIIClone = [{
     addData(llvm::hash_value(S->getValue()));
   }];
 }
-class StringLiteral {
-  code Code = [{
+def StringLiteral : Collector {
+  let TypeIIClone = [{
     addData(S->getString());
 }];
 }
-class CXXBoolLiteralExpr {
-  code Code = [{
+def CXXBoolLiteralExpr : Collector {
+  let TypeIIClone = [{
     addData(S->getValue());
   }];
 }
-class CharacterLiteral {
-  code Code = [{
+def CharacterLiteral : Collector {
+  let TypeIIClone = [{
     addData(S->getValue());
   }];
 }
 
 //--- Exceptions ---------------------------------------------------------//
-class CXXCatchStmt {
-  code Code = [{
+def CXXCatchStmt : Collector {
+  let TypeIIClone = [{
     addData(S->getCaughtType());
   }];
 }
 
 //--- C++ OOP Stmts ------------------------------------------------------//
-class CXXDeleteExpr {
-  code Code = [{
+def CXXDeleteExpr : Collector {
+  let TypeIIClone = [{
     addData(S->isArrayFormAsWritten()); addData(S->isGlobalDelete());
   }];
 }
 
 //--- Casts --------------------------------------------------------------//
-class ObjCBridgedCastExpr {
-  code Code = [{
+def ObjCBridgedCastExpr : Collector {
+  let TypeIIClone = [{
     addData(S->getBridgeKind());
   }];
 }
 
 //--- Miscellaneous Exprs ------------------------------------------------//
-class BinaryOperator {
-  code Code = [{
+def BinaryOperator : Collector {
+  let TypeIIClone = [{
     addData(S->getOpcode());
   }];
 }
-class UnaryOperator {
-  code Code = [{
+def UnaryOperator : Collector {
+  let TypeIIClone = [{
     addData(S->getOpcode());
   }];
 }
 
 //--- Control flow -------------------------------------------------------//
-class GotoStmt {
-  code Code = [{
+def GotoStmt : Collector {
+  let TypeIIClone = [{
     addData(S->getLabel()->getName());
   }];
 }
-class IndirectGotoStmt {
-  code Code = [{
+def IndirectGotoStmt : Collector {
+  let TypeIIClone = [{
     if (S->getConstantTarget())
       addData(S->getConstantTarget()->getName());
   }];
 }
-class LabelStmt {
-  code Code = [{
+def LabelStmt : Collector {
+  let TypeIIClone = [{
     addData(S->getDecl()->getName());
   }];
 }
-class MSDependentExistsStmt {
-  code Code = [{
+def MSDependentExistsStmt : Collector {
+  let TypeIIClone = [{
     addData(S->isIfExists());
   }];
 }
-class AddrLabelExpr {
-  code Code = [{
+def AddrLabelExpr : Collector {
+  let TypeIIClone = [{
     addData(S->getLabel()->getName());
   }];
 }
 
 //--- Objective-C --------------------------------------------------------//
-class ObjCIndirectCopyRestoreExpr {
-  code Code = [{
+def ObjCIndirectCopyRestoreExpr : Collector {
+  let TypeIIClone = [{
     addData(S->shouldCopy());
   }];
 }
-class ObjCPropertyRefExpr {
-  code Code = [{
+def ObjCPropertyRefExpr : Collector {
+  let TypeIIClone = [{
     addData(S->isSuperReceiver()); addData(S->isImplicitProperty());
   }];
 }
-class ObjCAtCatchStmt {
-  code Code = [{
+def ObjCAtCatchStmt : Collector {
+  let TypeIIClone = [{
     addData(S->hasEllipsis());
   }];
 }
 
 //--- Miscellaneous Stmts ------------------------------------------------//
-class CXXFoldExpr {
-  code Code = [{
+def CXXFoldExpr : Collector {
+  let TypeIIClone = [{
     addData(S->isRightFold()); addData(S->getOperator());
   }];
 }
-class GenericSelectionExpr {
-  code Code = [{
+def GenericSelectionExpr : Collector {
+  let TypeIIClone = [{
     for (unsigned i = 0; i < S->getNumAssocs(); ++i) {
       addData(S->getAssocType(i));
     }
   }];
 }
-class LambdaExpr {
-  code Code = [{
+def LambdaExpr : Collector {
+  let TypeIIClone = [{
     for (const LambdaCapture &C : S->captures()) {
       addData(C.isPackExpansion());
       addData(C.getCaptureKind());
       if (C.capturesVariable())
         addData(C.getCapturedVar()->getType());
     }
     addData(S->isGenericLambda());
     addData(S->isMutable());
   }];
 }
-class DeclStmt {
-  code Code = [{
+def DeclStmt : Collector {
+  let TypeIIClone = [{
     auto numDecls = std::distance(S->decl_begin(), S->decl_end());
     addData(static_cast<unsigned>(numDecls));
     for (const Decl *D : S->decls()) {
       if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
         addData(VD->getType());
       }
     }
   }];
 }
-class AsmStmt {
-  code Code = [{
+def AsmStmt : Collector {
+  let TypeIIClone = [{
     addData(S->isSimple());
     addData(S->isVolatile());
     addData(S->generateAsmString(Context));
     for (unsigned i = 0; i < S->getNumInputs(); ++i) {
       addData(S->getInputConstraint(i));
     }
     for (unsigned i = 0; i < S->getNumOutputs(); ++i) {
       addData(S->getOutputConstraint(i));
     }
     for (unsigned i = 0; i < S->getNumClobbers(); ++i) {
       addData(S->getClobber(i));
     }
   }];
 }
-class AttributedStmt {
-  code Code = [{
+def AttributedStmt : Collector {
+  let TypeIIClone = [{
     for (const Attr *A : S->getAttrs()) {
       addData(std::string(A->getSpelling()));
     }
   }];
 }

lib/Analysis/CloneDetection.cpp

CloneTypeIIStmtDataCollector(const Stmt *S, ASTContext &Context,
                                T &DataConsumer)
       : Context(Context), DataConsumer(DataConsumer) {
     this->Visit(S);
   }
 
 // Define a visit method for each class to collect data and subsequently visit
 // all parent classes. This uses a template so that custom visit methods by us
 // take precedence.
-#define DEF_ADD_DATA(CLASS, CODE)                                              \
+#define COLLECT_FOR_TYPEIICLONE(CLASS, CODE)                                   \
   template <class = void> void Visit##CLASS(const CLASS *S) {                  \
     CODE;                                                                      \
     ConstStmtVisitor<CloneTypeIIStmtDataCollector<T>>::Visit##CLASS(S);        \
   }
 
 #include "clang/AST/StmtDataCollectors.inc"
 
 // Type II clones ignore variable names and literals, so let's skip them.

unittests/AST/DataCollectionTest.cpp

class StmtDataCollector : public ConstStmtVisitor<StmtDataCollector> {
   }
 
 public:
   StmtDataCollector(const Stmt *S, ASTContext &Context, llvm::MD5 &DataConsumer)
       : Context(Context), DataConsumer(DataConsumer) {
     this->Visit(S);
   }
 
-#define DEF_ADD_DATA(CLASS, CODE)                                              \
+#define COLLECT_FOR_TYPEIICLONE(CLASS, CODE)                                   \
   template <class Dummy = void> Dummy Visit##CLASS(const CLASS *S) {           \
     CODE;                                                                      \
     ConstStmtVisitor<StmtDataCollector>::Visit##CLASS(S);                      \
   }
 
 #include "clang/AST/StmtDataCollectors.inc"
 };
 } // end anonymous namespace

utils/TableGen/ClangDataCollectorsEmitter.cpp

 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
 
 using namespace llvm;
 
 namespace clang {
+
+static struct {
+  const char *Name;
+  const char *MacroName;
+} CollectionCategories[] = {
+    {"TypeIIClone", "COLLECT_FOR_TYPEIICLONE"},
+};
+
 void EmitClangDataCollectors(RecordKeeper &RK, raw_ostream &OS) {
-  const auto &Defs = RK.getClasses();
+  for (const auto &Category : CollectionCategories) {
+    OS << "#ifndef " << Category.MacroName << "\n";
+    OS << "#define " << Category.MacroName << "(A, B)\n";
+    OS << "#endif\n";
+  }
+  const auto &Defs = RK.getDefs();
   for (const auto &Entry : Defs) {
     Record &R = *Entry.second;
-    OS << "DEF_ADD_DATA(" << R.getName() << ", {\n";
-    auto Code = R.getValue("Code")->getValue();
+    for (auto Category : CollectionCategories) {
+      OS << Category.MacroName << "(" << R.getName() << ", {\n";
+      auto Code = R.getValue(Category.Name)->getValue();
       OS << Code->getAsUnquotedString() << "}\n)";
       OS << "\n";
     }
-  OS << "#undef DEF_ADD_DATA\n";
+  }
+  for (const auto &Category : CollectionCategories)
+    OS << "#undef " << Category.MacroName << "\n";
 }
 } // end namespace clang