[gicombiner] Add a CodeExpander to handle C++ fragments with variable expansion

[dsanders]

Summary

This will handle expansion of C++ fragments in the declarative combiner
including custom predicates, and escapes into C++ to aid the migration
effort.

[volkan]

LGTM.

[volkan]

llvm/utils/TableGen/GlobalISel/CMakeLists.txt

6

Nit: extra whitespace

[thakis]

llvm/trunk/utils/TableGen/GlobalISel/CMakeLists.txt

5 ↗

(On Diff #222946)

Out of interest, why is this an object library instead of a normal STATIC library?

[dsanders]

llvm/trunk/utils/TableGen/GlobalISel/CMakeLists.txt

5 ↗

(On Diff #222946)

I assume you're asking because some of the bots are failing. AFAICT it's the ones with older cmakes.

I went with an object library because it doesn't need to be an actual library. It's just a means to include them in the llbm-tblgen link. Switching to a STATIC library might be how I end up fixing it.

[thakis]

llvm/trunk/utils/TableGen/GlobalISel/CMakeLists.txt

5 ↗

(On Diff #222946)

No, I asked out of interest, like I said :)

Most libraries in LLVM don't need to be an actual library. It's just the usual unit of abstraction for a bunch of files. The index therein and their load semantics tend to be good for binary size.

[dsanders]

llvm/trunk/utils/TableGen/GlobalISel/CMakeLists.txt

5 ↗

(On Diff #222946)

Ah it was just coincidental timing then :-). Your question arrived in the middle of twenty or so bot failure emails about older cmake's not liking the object library.

The index therein and their load semantics tend to be good for binary size.

I'm a bit curious about this as a static library (on Linux and macOS at least) is just the object files inside an 'ar' archive. I'm not aware of any functional difference (other than the existence of the static library) between linking a static library and linking the object files directly. Are you on a platform where that's not the case? or is there something about static libraries that I'm not aware of?

llvm/unittests/CMakeLists.txt

 add_subdirectory(MI)
 add_subdirectory(Object)
 add_subdirectory(ObjectYAML)
 add_subdirectory(Option)
 add_subdirectory(Remarks)
 add_subdirectory(Passes)
 add_subdirectory(ProfileData)
 add_subdirectory(Support)
-add_subdirectory(TextAPI)
+add_subdirectory(TableGen)
 add_subdirectory(Target)
+add_subdirectory(TextAPI)
 add_subdirectory(Transforms)
 add_subdirectory(XRay)
 add_subdirectory(tools)

llvm/unittests/TableGen/CMakeLists.txt

+set(LLVM_LINK_COMPONENTS
+  TableGen
+  Support
+  )
+
+add_llvm_unittest(TableGenTests
+  CodeExpanderTest.cpp
+  $<TARGET_OBJECTS:obj.LLVMTableGenGlobalISel>
+  )
+
+include_directories(${CMAKE_SOURCE_DIR}/utils/TableGen)

llvm/unittests/TableGen/CodeExpanderTest.cpp

+//===- llvm/unittest/TableGen/CodeExpanderTest.cpp - Tests ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "GlobalISel/CodeExpander.h"
+#include "GlobalISel/CodeExpansions.h"
+
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/TableGen/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+static StringRef bufferize(StringRef Str) {
+  std::unique_ptr<MemoryBuffer> Buffer =
+      MemoryBuffer::getMemBufferCopy(Str, "TestBuffer");
+  StringRef StrBufferRef = Buffer->getBuffer();
+  SrcMgr.AddNewSourceBuffer(std::move(Buffer), SMLoc());
+  return StrBufferRef;
+}
+
+class RAIIDiagnosticChecker {
+  std::string EmittedDiags;
+  raw_string_ostream OS;
+  std::vector<SMDiagnostic> Expected;
+  std::vector<SMDiagnostic> Received;
+
+public:
+  RAIIDiagnosticChecker() : OS(EmittedDiags) {
+    SrcMgr.setDiagHandler(handler, this);
+  }
+  ~RAIIDiagnosticChecker() {
+    SrcMgr.setDiagHandler(nullptr);
+    EXPECT_EQ(Received.size(), Expected.size());
+    for (unsigned i = 0; i < Received.size() && i < Expected.size(); ++i) {
+      EXPECT_EQ(Received[i].getLoc(), Expected[i].getLoc());
+      EXPECT_EQ(Received[i].getFilename(), Expected[i].getFilename());
+      EXPECT_EQ(Received[i].getKind(), Expected[i].getKind());
+      EXPECT_EQ(Received[i].getLineNo(), Expected[i].getLineNo());
+      EXPECT_EQ(Received[i].getColumnNo(), Expected[i].getColumnNo());
+      EXPECT_EQ(Received[i].getMessage(), Expected[i].getMessage());
+      EXPECT_EQ(Received[i].getLineContents(), Expected[i].getLineContents());
+      EXPECT_EQ(Received[i].getRanges(), Expected[i].getRanges());
+    }
+
+    if (testing::Test::HasFailure())
+      errs() << "Emitted diagnostic:\n" << OS.str();
+  }
+
+  void expect(SMDiagnostic D) { Expected.push_back(D); }
+
+  void diag(const SMDiagnostic &D) {
+    Received.push_back(D);
+  }
+
+  static void handler(const SMDiagnostic &D, void *Context) {
+    RAIIDiagnosticChecker *Self = static_cast<RAIIDiagnosticChecker *>(Context);
+    Self->diag(D);
+    SrcMgr.setDiagHandler(nullptr);
+    SrcMgr.PrintMessage(Self->OS, D);
+    SrcMgr.setDiagHandler(handler, Context);
+  };
+};
+
+TEST(CodeExpander, NoExpansions) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("No expansions", Expansions, SMLoc()).emit(OS);
+  EXPECT_EQ(OS.str(), "No expansions");
+}
+
+// Indentation is applied to all lines except the first
+TEST(CodeExpander, Indentation) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("No expansions\nsecond line\nthird line", Expansions, SMLoc(),
+               "  ")
+      .emit(OS);
+  EXPECT_EQ(OS.str(), "No expansions\n  second line\n  third line");
+}
+
+// \ is an escape character that removes special meanings from the next
+// character.
+TEST(CodeExpander, Escape) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("\\\\\\a\\$", Expansions, SMLoc()).emit(OS);
+  EXPECT_EQ(OS.str(), "\\a$");
+}
+
+// $foo is not an expansion. It should warn though.
+TEST(CodeExpander, NotAnExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  StringRef In = bufferize(" $foo");
+  CodeExpander(" $foo", Expansions, SMLoc::getFromPointer(In.data())).emit(OS);
+  EXPECT_EQ(OS.str(), " $foo");
+  DiagChecker.expect(SMDiagnostic(
+      SrcMgr, SMLoc::getFromPointer(In.data() + 1), "TestBuffer", 1, 1,
+      SourceMgr::DK_Warning, "Assuming missing escape character", " $foo", {}));
+}
+
+// \$foo is not an expansion but shouldn't warn as it's using the escape.
+TEST(CodeExpander, EscapedNotAnExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("\\$foo", Expansions, SMLoc()).emit(OS);
+  EXPECT_EQ(OS.str(), "$foo");
+}
+
+// \${foo is not an expansion but shouldn't warn as it's using the escape.
+TEST(CodeExpander, EscapedUnterminatedExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("\\${foo", Expansions, SMLoc()).emit(OS);
+  EXPECT_EQ(OS.str(), "${foo");
+}
+
+// \${foo is not an expansion but shouldn't warn as it's using the escape.
+TEST(CodeExpander, EscapedExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("\\${foo}", Expansions, SMLoc()).emit(OS);
+  EXPECT_EQ(OS.str(), "${foo}");
+}
+
+// ${foo} is an undefined expansion and should error.
+TEST(CodeExpander, UndefinedExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+  Expansions.declare("bar", "expansion");
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("${foo}${bar}", Expansions, SMLoc()).emit(OS);
+  EXPECT_EQ(OS.str(), "expansion");
+  DiagChecker.expect(
+      SMDiagnostic(SrcMgr, SMLoc(), "<unknown>", 0, -1, SourceMgr::DK_Error,
+                   "Attempting to expand an undeclared variable foo", "", {}));
+}
+
+// ${foo} is an undefined expansion and should error. When given a valid
+// location for the start of the buffer it should correctly point at the
+// expansion being performed.
+TEST(CodeExpander, UndefinedExpansionWithLoc) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+  Expansions.declare("bar", "expansion");
+
+  RAIIDiagnosticChecker DiagChecker;
+  StringRef In = bufferize("Padding ${foo}${bar}");
+  CodeExpander(In, Expansions, SMLoc::getFromPointer(In.data())).emit(OS);
+  EXPECT_EQ(OS.str(), "Padding expansion");
+  DiagChecker.expect(SMDiagnostic(
+      SrcMgr, SMLoc::getFromPointer(In.data() + 8), "TestBuffer", 1, 8,
+      SourceMgr::DK_Error, "Attempting to expand an undeclared variable foo",
+      "Padding ${foo}${bar}", {}));
+}
+
+// ${bar is an unterminated expansion. Warn and implicitly terminate it.
+TEST(CodeExpander, UnterminatedExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+  Expansions.declare("bar", "expansion");
+
+  RAIIDiagnosticChecker DiagChecker;
+  StringRef In = bufferize(" ${bar");
+  CodeExpander(In, Expansions, SMLoc::getFromPointer(In.data())).emit(OS);
+  EXPECT_EQ(OS.str(), " expansion");
+  DiagChecker.expect(SMDiagnostic(SrcMgr, SMLoc::getFromPointer(In.data() + 1),
+                                  "TestBuffer", 1, 1, SourceMgr::DK_Warning,
+                                  "Unterminated expansion", " ${bar", {}));
+}

llvm/utils/TableGen/GlobalISel/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   Support
   )
 
 llvm_add_library(LLVMTableGenGlobalISel OBJECT
+                 CodeExpander.cpp
      GICombinerEmitter.cpp
 )

llvm/utils/TableGen/GlobalISel/CodeExpander.h

+//===- CodeExpander.h - Expand variables in a string ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file Expand the variables in a string.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_UTILS_TABLEGEN_CODEEXPANDER_H
+#define LLVM_UTILS_TABLEGEN_CODEEXPANDER_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/SMLoc.h"
+
+namespace llvm {
+class CodeExpansions;
+class raw_ostream;
+
+/// Emit the given code with all '${foo}' placeholders expanded to their
+/// replacements.
+///
+/// It's an error to use an undefined expansion and expansion-like output that
+/// needs to be emitted verbatim can be escaped as '\${foo}'
+///
+/// The emitted code can be given a custom indent to enable both indentation by
+/// an arbitrary amount of whitespace and emission of the code as a comment.
+class CodeExpander {
+  StringRef Code;
+  const CodeExpansions &Expansions;
+  const ArrayRef<SMLoc> &Loc;
+  StringRef Indent;
+
+public:
+  CodeExpander(StringRef Code, const CodeExpansions &Expansions,
+               const ArrayRef<SMLoc> &Loc)
+      : CodeExpander(Code, Expansions, Loc, "    ") {}
+  CodeExpander(StringRef Code, const CodeExpansions &Expansions,
+               const ArrayRef<SMLoc> &Loc, StringRef Indent)
+      : Code(Code), Expansions(Expansions), Loc(Loc), Indent(Indent) {}
+
+  void emit(raw_ostream &OS) const;
+};
+
+inline raw_ostream &operator<<(raw_ostream &OS, const CodeExpander &Expander) {
+  Expander.emit(OS);
+  return OS;
+}
+} // end namespace llvm
+
+#endif // ifndef LLVM_UTILS_TABLEGEN_CODEEXPANDER_H

llvm/utils/TableGen/GlobalISel/CodeExpander.cpp

+//===- CodeExpander.cpp - Expand variables in a string --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file Expand the variables in a string.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CodeExpander.h"
+#include "CodeExpansions.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/TableGen/Error.h"
+
+using namespace llvm;
+
+// TODO: Move category declaration to GICombinerEmitter once that exists.
+cl::OptionCategory
+    GICombinerEmitterCat("Options for -gen-global-isel-combiner");
+static cl::opt<bool> ShowExpansions(
+    "gicombiner-show-expansions",
+    cl::desc("Use C++ comments to indicate occurence of code expansion"),
+    cl::cat(GICombinerEmitterCat));
+
+void CodeExpander::emit(raw_ostream &OS) const {
+  StringRef Current = Code;
+
+  while (!Current.empty()) {
+    size_t Pos = Current.find_first_of("$\n\\");
+    if (Pos == StringRef::npos) {
+      OS << Current;
+      Current = "";
+      continue;
+    }
+
+    OS << Current.substr(0, Pos);
+    Current = Current.substr(Pos);
+
+    if (Current.startswith("\n")) {
+      OS << "\n" << Indent;
+      Current = Current.drop_front(1);
+      continue;
+    }
+
+    if (Current.startswith("\\$") || Current.startswith("\\\\")) {
+      OS << Current[1];
+      Current = Current.drop_front(2);
+      continue;
+    }
+
+    if (Current.startswith("\\")) {
+      Current = Current.drop_front(1);
+      continue;
+    }
+
+    if (Current.startswith("${")) {
+      StringRef StartVar = Current;
+      Current = Current.drop_front(2);
+      StringRef Var;
+      std::tie(Var, Current) = Current.split("}");
+
+      // Warn if we split because no terminator was found.
+      StringRef EndVar = StartVar.drop_front(2 /* ${ */ + Var.size());
+      if (EndVar.empty()) {
+        size_t LocOffset = StartVar.data() - Code.data();
+        PrintWarning(
+            Loc.size() > 0 && Loc[0].isValid()
+                ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset)
+                : SMLoc(),
+            "Unterminated expansion");
+      }
+
+      auto ValueI = Expansions.find(Var);
+      if (ValueI == Expansions.end()) {
+        size_t LocOffset = StartVar.data() - Code.data();
+        PrintError(Loc.size() > 0 && Loc[0].isValid()
+                       ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset)
+                       : SMLoc(),
+                   "Attempting to expand an undeclared variable " + Var);
+      }
+      if (ShowExpansions)
+        OS << "/*$" << Var << "{*/";
+      OS << Expansions.lookup(Var);
+      if (ShowExpansions)
+        OS << "/*}*/";
+      continue;
+    }
+
+    size_t LocOffset = Current.data() - Code.data();
+    PrintWarning(Loc.size() > 0 && Loc[0].isValid()
+                     ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset)
+                     : SMLoc(),
+                 "Assuming missing escape character");
+    OS << "$";
+    Current = Current.drop_front(1);
+  }
+}

llvm/utils/TableGen/GlobalISel/CodeExpansions.h

+//===- CodeExpansions.h - Record expansions for CodeExpander --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file Record the expansions to use in a CodeExpander.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/StringMap.h"
+
+#ifndef LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H
+#define LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H
+namespace llvm {
+class CodeExpansions {
+public:
+  using const_iterator = StringMap<std::string>::const_iterator;
+
+protected:
+  StringMap<std::string> Expansions;
+
+public:
+  void declare(StringRef Name, StringRef Expansion) {
+    bool Inserted = Expansions.try_emplace(Name, Expansion).second;
+    assert(Inserted && "Declared variable twice");
+    (void)Inserted;
+  }
+
+  std::string lookup(StringRef Variable) const {
+    return Expansions.lookup(Variable);
+  }
+
+  const_iterator begin() const { return Expansions.begin(); }
+  const_iterator end() const { return Expansions.end(); }
+  const_iterator find(StringRef Variable) const {
+    return Expansions.find(Variable);
+  }
+};
+} // end namespace llvm
+#endif // ifndef LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H