Page MenuHomePhabricator
Differential D109636 Diff 372282 mlir/lib/Support/ToolUtilities.cpp

Changeset View

Standalone View

View Options

mlir/lib/Support/ToolUtilities.cpp

//===- ToolUtilities.cpp - MLIR Tool Utilities ----------------------------===// //===- ToolUtilities.cpp - MLIR Tool Utilities ----------------------------===//
// //
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information. // See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// //
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
// //
// This file defines common utilities for implementing MLIR tools. // This file defines common utilities for implementing MLIR tools.
// //
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
#include "mlir/Support/ToolUtilities.h" #include "mlir/Support/ToolUtilities.h"
#include "mlir/Support/LLVM.h" #include "mlir/Support/LLVM.h"
#include "mlir/Support/LogicalResult.h" #include "mlir/Support/LogicalResult.h"
#include "llvm/Support/SourceMgr.h" #include "llvm/Support/SourceMgr.h"
#include "llvm/Support/raw_ostream.h"
using namespace mlir; using namespace mlir;
LogicalResult LogicalResult
mlir::splitAndProcessBuffer(std::unique_ptr<llvm::MemoryBuffer> originalBuffer, mlir::splitAndProcessBuffer(std::unique_ptr<llvm::MemoryBuffer> originalBuffer,
ChunkBufferHandler processChunkBuffer, ChunkBufferHandler processChunkBuffer,
raw_ostream &os) { raw_ostream &os) {
const char splitMarker[] = "// -----"; const char splitMarkerConst[] = "// -----";
StringRef splitMarker(splitMarkerConst);
const int splitMarkerLen = splitMarker.size();
auto *origMemBuffer = originalBuffer.get(); auto *origMemBuffer = originalBuffer.get();
SmallVector<StringRef, 8> sourceBuffers; SmallVector<StringRef, 8> rawSourceBuffers;
origMemBuffer->getBuffer().split(sourceBuffers, splitMarker); const int checkLen = 2;
// Split dropping the last checkLen chars to enable flagging near misses.
origMemBuffer->getBuffer().split(rawSourceBuffers,
splitMarker.drop_back(checkLen));
if (rawSourceBuffers.empty())
return success();
// Add the original buffer to the source manager. // Add the original buffer to the source manager.
llvm::SourceMgr fileSourceMgr; llvm::SourceMgr fileSourceMgr;
fileSourceMgr.AddNewSourceBuffer(std::move(originalBuffer), llvm::SMLoc()); fileSourceMgr.AddNewSourceBuffer(std::move(originalBuffer), llvm::SMLoc());
// Flag near misses by iterating over all the sub-buffers found when splitting
// with the prefix of the splitMarker. Use a sliding window where we only
// add a buffer as a sourceBuffer if terminated by a full match of the
// splitMarker (followed by newline), else flag a warning and extend the
// size of the buffer under consideration assuming there is no near miss.
SmallVector<StringRef, 8> sourceBuffers;
StringRef prev;
for (auto buffer : rawSourceBuffers) {
if (prev.empty()) {
prev = buffer;
mehdi_aminiUnsubmitted
Done
ReplyInline Actions

This patch made me look at our current behavior: we don't require a new line after the pattern!
I think we should?

I mean that it is happy to parse this:

func private @foo()
// -----func private @foo()

->

$ ./bin/mlir-opt  -split-input-file split.mlir 
module  {
  func private @foo()
}

module  {
  func private @foo()
}

So I would check exactly --\n here.

mehdi_amini: This patch made me look at our current behavior: we don't require a new line after the pattern!
rriddleUnsubmitted
Not Done
ReplyInline Actions

We actually semi rely on this when we generate "print-ir-*" output. We generate output such that each output block can be treated as a separate IR block.

rriddle: We actually semi rely on this when we generate "print-ir-*" output. We generate output such…
mehdi_aminiUnsubmitted
Not Done
ReplyInline Actions

Oh wow!

// -----// IR Dump Before CSE //----- //
func @foo() {
  return
}

// -----// IR Dump Before CSE //----- //
func @baz() {
  return
}

// -----// IR Dump Before Canonicalizer //----- //
func @foo() {
  return
}

// -----// IR Dump Before Canonicalizer //----- //
func @baz() {
  return
}

I didn't notice this trick about the extra // before, nifty!

mehdi_amini: Oh wow! ``` // -----// IR Dump Before CSE //----- // func @foo() { return } // -----// IR…
jpienaarAuthorUnsubmitted
Done
ReplyInline Actions

I'll revert the newline check then (or I could additionally allow for "" or " " suffices, but not sure that valuable).

Although what is the use case? E.g., if you dump all the sections, would you feed this output file into mlir-opt to run a pass over all stages of a pipeline? (I'd have thought you'd just copy & past the part of interest out into a new file post print-ir-* rather than feed in the whole file).

jpienaar: I'll revert the newline check then (or I could additionally allow for "//" or " //" suffices…
rriddleUnsubmitted
Not Done
ReplyInline Actions

The main concrete use case is that you can open the dump in vscode(or another IDE) and get language server capabilities (without having the different dumps negatively interact with each other).

rriddle: The main concrete use case is that you can open the dump in vscode(or another IDE) and get…
continue;
}
mehdi_aminiUnsubmitted
Done
ReplyInline Actions

Can you make splitMarker a StringRef from the beginning?

mehdi_amini: Can you make splitMarker a StringRef from the beginning?
// Check that suffix is as expected and ends with newline.
bool expectedSuffix = buffer.startswith(splitMarker.take_back(checkLen)) &&
buffer[checkLen] == '\n';
rriddleUnsubmitted
Not Done
ReplyInline Actions

Will this crash if the split is the last line?

rriddle: Will this crash if the split is the last line?
jpienaarAuthorUnsubmitted
Done
ReplyInline Actions

Good question, it didn't but I think that was as mostly null-terminated main buffer here, but checking explicitly better.

jpienaar: Good question, it didn't but I think that was as mostly null-terminated main buffer here, but…
if (expectedSuffix) {
sourceBuffers.push_back(prev);
prev = buffer.drop_front(checkLen);
} else {
// TODO: Consider making this a failure.
auto splitLoc = llvm::SMLoc::getFromPointer(buffer.data());
fileSourceMgr.PrintMessage(llvm::errs(), splitLoc,
llvm::SourceMgr::DK_Warning,
"near miss with file split marker");
prev = StringRef(prev.data(),
mehdi_aminiUnsubmitted
Done
ReplyInline Actions

I think this loop deserve a bit more doc

mehdi_amini: I think this loop deserve a bit more doc
prev.size() + splitMarkerLen - checkLen + buffer.size());
}
}
if (!prev.empty())
sourceBuffers.push_back(prev);
// Process each chunk in turn. // Process each chunk in turn.
bool hadFailure = false; bool hadFailure = false;
for (auto &subBuffer : sourceBuffers) { for (auto &subBuffer : sourceBuffers) {
auto splitLoc = llvm::SMLoc::getFromPointer(subBuffer.data()); auto splitLoc = llvm::SMLoc::getFromPointer(subBuffer.data());
unsigned splitLine = fileSourceMgr.getLineAndColumn(splitLoc).first; unsigned splitLine = fileSourceMgr.getLineAndColumn(splitLoc).first;
auto subMemBuffer = llvm::MemoryBuffer::getMemBufferCopy( auto subMemBuffer = llvm::MemoryBuffer::getMemBufferCopy(
subBuffer, Twine("within split at ") + subBuffer, Twine("within split at ") +
origMemBuffer->getBufferIdentifier() + ":" + origMemBuffer->getBufferIdentifier() + ":" +
Twine(splitLine) + " offset "); Twine(splitLine) + " offset ");
if (failed(processChunkBuffer(std::move(subMemBuffer), os))) if (failed(processChunkBuffer(std::move(subMemBuffer), os)))
hadFailure = true; hadFailure = true;
} }
// If any fails, then return a failure of the tool. // If any fails, then return a failure of the tool.
return failure(hadFailure); return failure(hadFailure);
} }