Index: llvm/tools/llvm-exegesis/lib/Assembler.cpp =================================================================== --- llvm/tools/llvm-exegesis/lib/Assembler.cpp +++ llvm/tools/llvm-exegesis/lib/Assembler.cpp @@ -8,6 +8,7 @@ #include "Assembler.h" +#include "InlineAsmMCExpr.h" #include "Target.h" #include "llvm/CodeGen/GlobalISel/CallLowering.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" @@ -110,6 +111,9 @@ Builder.addReg(Op.getReg(), Flags); } else if (Op.isImm()) { Builder.addImm(Op.getImm()); + } else if (Op.isExpr() && Opcode == TargetOpcode::INLINEASM) { + Builder.addExternalSymbol( + cast(Op.getExpr())->getAsmString()); } else if (!Op.isValid()) { llvm_unreachable("Operand is not set"); } else { Index: llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h =================================================================== --- llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h +++ llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h @@ -36,6 +36,9 @@ BenchmarkFailure(const llvm::Twine &S); }; +// Function to dump an MCInst to string. +std::string DumpToString(const llvm::MCInst &inst); + // Common code for all benchmark modes. class BenchmarkRunner { public: @@ -78,6 +81,10 @@ const LLVMState &State; const InstructionBenchmark::ModeE Mode; + static std::vector + GenerateInstructions(const BenchmarkCode &BC, const size_t MinInstructions, + bool IncludeIacaMakers); + private: virtual llvm::Expected> runMeasurements(const FunctionExecutor &Executor) const = 0; Index: llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp =================================================================== --- llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -7,22 +7,55 @@ //===----------------------------------------------------------------------===// #include +#include #include #include "Assembler.h" #include "BenchmarkRunner.h" +#include "InlineAsmMCExpr.h" #include "MCInstrDescView.h" +#include "MCTargetDesc/X86MCTargetDesc.h" #include "PerfHelper.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" +#include "llvm/MC/MCInst.h" +#include "llvm/MC/MCInstBuilder.h" #include "llvm/Support/CrashRecoveryContext.h" +#include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Program.h" namespace llvm { namespace exegesis { +namespace { + +// Inserts IACA-start marker at the end of the given vector. +void InsertStartMarker(std::vector &Instructions) { + // mov ebx, 111 ; IACA Start marker bytes + // db 0x64, 0x67, 0x90 ; IACA Start marker bytes + Instructions.push_back( + MCInstBuilder(llvm::X86::MOV32ri).addReg(llvm::X86::EBX).addImm(111)); + Instructions.push_back(MCInstBuilder(TargetOpcode::INLINEASM) + .addExpr(new iaca::IacaMarkerBytesMCExpr()) + // Because our IACA markers trap: + .addImm(InlineAsm::Extra_HasSideEffects)); +} + +// Inserts IACA-end marker at the end of the given vector. +void InsertEndMarker(std::vector &Instructions) { + // mov ebx, 222 ; IACA End marker bytes + // db 0x64, 0x67, 0x90 ; IACA End marker bytes + Instructions.push_back( + MCInstBuilder(llvm::X86::MOV32ri).addReg(llvm::X86::EBX).addImm(222)); + Instructions.push_back(MCInstBuilder(TargetOpcode::INLINEASM) + .addExpr(new iaca::IacaMarkerBytesMCExpr()) + // Because our IACA markers trap: + .addImm(InlineAsm::Extra_HasSideEffects)); +} + +} // end of anonymous namespace BenchmarkFailure::BenchmarkFailure(const llvm::Twine &S) : llvm::StringError(S, llvm::inconvertibleErrorCode()) {} @@ -35,13 +68,32 @@ // Repeat the snippet until there are at least MinInstructions in the resulting // code. -static std::vector -GenerateInstructions(const BenchmarkCode &BC, const size_t MinInstructions) { - if (BC.Instructions.empty()) +std::vector +BenchmarkRunner::GenerateInstructions(const BenchmarkCode &BC, + const size_t MinInstructions, + const bool IncludeIacaMakers) { + if (BC.Instructions.empty()) { return {}; - std::vector Code = BC.Instructions; - for (int I = 0; Code.size() < MinInstructions; ++I) + } + + std::vector Code; + Code.reserve(MinInstructions + (IncludeIacaMakers ? 4 : 0)); + + if (IncludeIacaMakers) { + InsertStartMarker(Code); + } + + // Do not count the IACA markers when checking the size because these + // instructions are not executable. + const int AdjustedSize = MinInstructions + (IncludeIacaMakers ? 2 : 0); + for (int I = 0; Code.size() < AdjustedSize; ++I) { Code.push_back(BC.Instructions[I % BC.Instructions.size()]); + } + + if (IncludeIacaMakers) { + InsertEndMarker(Code); + } + return Code; } @@ -95,6 +147,14 @@ }; } // namespace +std::string DumpToString(const llvm::MCInst &inst) { + std::string buffer; + llvm::raw_string_ostream stream(buffer); + inst.print(stream); + stream.flush(); + return buffer; +} + InstructionBenchmark BenchmarkRunner::runConfiguration(const BenchmarkCode &BC, unsigned NumRepetitions, @@ -117,8 +177,11 @@ // that the inside instructions are repeated. constexpr const int kMinInstructionsForSnippet = 16; { - auto ObjectFilePath = writeObjectFile( - BC, GenerateInstructions(BC, kMinInstructionsForSnippet)); + // NOTE(vyng): "IncludeIacaMarkers" must be false, regardless of the mode + // because we don't want to insert the markers around single snippet. + auto ObjectFilePath = + writeObjectFile(BC, GenerateInstructions(BC, kMinInstructionsForSnippet, + /*IncludeIacaMakers=*/false)); if (llvm::Error E = ObjectFilePath.takeError()) { InstrBenchmark.Error = llvm::toString(std::move(E)); return InstrBenchmark; @@ -131,9 +194,21 @@ // Assemble NumRepetitions instructions repetitions of the snippet for // measurements. - const auto Code = GenerateInstructions(BC, InstrBenchmark.NumRepetitions); + // TODO(vyng): Change the IncludeIacaMarkers to the right value based on mode. + const auto Code = GenerateInstructions(BC, InstrBenchmark.NumRepetitions, + /*IncludeIacaMakers=*/false); llvm::object::OwningBinary ObjectFile; + // For debugging. + { + int i = 0; + llvm::outs() << "*** START genereated instructions: \n"; + for (const auto &line : Code) { + llvm::outs() << i++ << ": " << DumpToString(line) << "\n"; + } + llvm::outs() << "*** END genereated instructions \n"; + } + if (DumpObjectToDisk) { auto ObjectFilePath = writeObjectFile(BC, Code); if (llvm::Error E = ObjectFilePath.takeError()) { Index: llvm/tools/llvm-exegesis/lib/InlineAsmMCExpr.h =================================================================== --- /dev/null +++ llvm/tools/llvm-exegesis/lib/InlineAsmMCExpr.h @@ -0,0 +1,56 @@ +#ifndef THIRD_PARTY_LLVM_LLVM_TOOLS_LLVM_EXEGESIS_INLINEASMMCEXPR_H_ +#define THIRD_PARTY_LLVM_LLVM_TOOLS_LLVM_EXEGESIS_INLINEASMMCEXPR_H_ + +#include "llvm/MC/MCExpr.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +namespace exegesis { + +// Class for emitting arbitrary inline-asm. +class InlineAsmMCExpr : public MCTargetExpr { +public: + virtual const char *getAsmString() const = 0; + + void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override { + OS << getAsmString(); + } + + bool evaluateAsRelocatableImpl(MCValue &Res, const MCAsmLayout *Layout, + const MCFixup *Fixup) const override { + return false; + } + + bool inlineAssignedExpr() const override { return true; } + + MCFragment *findAssociatedFragment() const override { return nullptr; } +}; + +namespace iaca { + +// Class for emitting IACA magic-bytes in start/end markers. +class IacaMarkerBytesMCExpr : public InlineAsmMCExpr { +public: + const char *getAsmString() const override { return ".byte 0x64, 0x67, 0x90"; } + + bool isEqualTo(const MCExpr *X) const override { + if (dyn_cast(X)) { + return true; + } + return false; + } + + void visitUsedExpr(MCStreamer &Streamer) const override { + /* Does nothing. */ + } + + void fixELFSymbolsInTLSFixups(MCAssembler &Asm) const override { + /* Does nothing. */ + } +}; + +} // namespace iaca +} // namespace exegesis +} // namespace llvm +#endif // THIRD_PARTY_LLVM_LLVM_TOOLS_LLVM_EXEGESIS_INLINEASMMCEXPR_H_ Index: llvm/unittests/tools/llvm-exegesis/BenchmarkRunnerTest.cpp =================================================================== --- llvm/unittests/tools/llvm-exegesis/BenchmarkRunnerTest.cpp +++ llvm/unittests/tools/llvm-exegesis/BenchmarkRunnerTest.cpp @@ -10,11 +10,94 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "third_party/absl/flags/flag.h" +#include "third_party/llvm/llvm/include/llvm/MC/MCExpr.h" +#include "third_party/llvm/llvm/include/llvm/MC/MCInst.h" +#include "third_party/llvm/llvm/include/llvm/MC/MCInstBuilder.h" +#include "third_party/llvm/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h" +#include "third_party/llvm/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h" +#include "third_party/llvm/llvm/tools/llvm-exegesis/lib/InlineAsmMCExpr.h" +#include "third_party/llvm/llvm/tools/llvm-exegesis/lib/RegisterValue.h" + namespace llvm { namespace exegesis { - namespace { +class IacaBenchmarkRunnerTester : public BenchmarkRunner { +public: + IacaBenchmarkRunnerTester() + : BenchmarkRunner(LLVMState(""), InstructionBenchmark::ModeE::Latency) {} + ~IacaBenchmarkRunnerTester() = default; + +protected: + // Exposed for testing. + using BenchmarkRunner::GenerateInstructions; + + llvm::Expected> + runMeasurements(const FunctionExecutor &Executor) const override { + // This does nothing because we're only testing the generation of IACA + // start/end markers. + // TODO: Change this once we've plugged in the iaca backend. + return std::vector(); + } + + friend class IacaBenchmarkRunnerTest; +}; + +class IacaBenchmarkRunnerTest : public ::testing::Test { +public: + IacaBenchmarkRunnerTest() : iaca_runner_(IacaBenchmarkRunnerTester()) {} + + static void SetUpTestCase() {} + static void TearDownTestCase() {} + + // Creates a piece of BenchmarkCode. + static BenchmarkCode MakeBenchmarkCode() { + BenchmarkCode Code; + Code.Info = "Random code."; + + Code.LiveIns.push_back(llvm::X86::EAX); + Code.LiveIns.push_back(llvm::X86::EBX); + Code.LiveIns.push_back(llvm::X86::ECX); + + Code.RegisterInitialValues.push_back(RegisterValue::zero(llvm::X86::EAX)); + Code.RegisterInitialValues.push_back(RegisterValue::zero(llvm::X86::EBX)); + Code.RegisterInitialValues.push_back(RegisterValue::zero(llvm::X86::ECX)); + + Code.Instructions.push_back( + MCInstBuilder(llvm::X86::MOV32ri).addReg(llvm::X86::EAX).addImm(10)); + Code.Instructions.push_back( + MCInstBuilder(llvm::X86::ADD32ri).addReg(llvm::X86::EAX).addImm(220)); + + return Code; + } + + // Checks that the argument is the IACA magic bytes. + // .byte 0x64, 0x67, 0x90 + static void AssertIacaMagicBytesInstruction(const llvm::MCInst &Inst) { + const std::string InstructionDump = DumpToString(Inst); + EXPECT_EQ(TargetOpcode::INLINEASM, Inst.getOpcode()) + << "Instruction dump: " << InstructionDump; + EXPECT_EQ(2, Inst.getNumOperands()) + << "Instruction dump: " << InstructionDump; + EXPECT_EQ( + " >", + InstructionDump); + } + + // Wrapper function that simply passes through the arguments to the real + // function. + std::vector GenerateInstructions(const BenchmarkCode &BC, + const size_t MinInstructions, + bool IncludeIacaMakers) { + return iaca_runner_.GenerateInstructions(BC, MinInstructions, + IncludeIacaMakers); + } + +protected: + IacaBenchmarkRunnerTester iaca_runner_; +}; + TEST(ScratchSpaceTest, Works) { BenchmarkRunner::ScratchSpace Space; EXPECT_EQ(reinterpret_cast(Space.ptr()) % @@ -27,6 +110,51 @@ EXPECT_EQ(Space.ptr()[BenchmarkRunner::ScratchSpace::kSize - 1], 0); } +TEST_F(IacaBenchmarkRunnerTest, IacaMarkers) { + const BenchmarkCode Code = MakeBenchmarkCode(); + const std::vector GeneratedInstructions = + GenerateInstructions(Code, Code.Instructions.size(), + /*IncludeIacaMakers=*/true); + + EXPECT_EQ(/*two 2-instruction markers + original size*/ + Code.Instructions.size() + 4, GeneratedInstructions.size()); + + // Start-maker + { + const llvm::MCInst &FirstInst = GeneratedInstructions.at(0); + const std::string FirstInstDump = DumpToString(FirstInst); + + EXPECT_EQ(llvm::X86::MOV32ri, FirstInst.getOpcode()) + << "Instruction dump: " << FirstInstDump; + EXPECT_EQ(2, FirstInst.getNumOperands()) + << "Instruction dump: " << FirstInstDump; + EXPECT_EQ(llvm::X86::EBX, FirstInst.getOperand(0).getReg()) + << "Instruction dump: " << FirstInstDump; + EXPECT_EQ(111, FirstInst.getOperand(1).getImm()) + << "Instruction dump: " << FirstInstDump; + + AssertIacaMagicBytesInstruction(GeneratedInstructions.at(1)); + } + + // End-maker + { + const llvm::MCInst &FirstInst = + GeneratedInstructions.at(GeneratedInstructions.size() - 2); + std::string FirstInstDump = DumpToString(FirstInst); + EXPECT_EQ(llvm::X86::MOV32ri, FirstInst.getOpcode()) + << "Instruction dump: " << FirstInstDump; + EXPECT_EQ(2, FirstInst.getNumOperands()) + << "Instruction dump: " << FirstInstDump; + EXPECT_EQ(llvm::X86::EBX, FirstInst.getOperand(0).getReg()) + << "Instruction dump: " << FirstInstDump; + EXPECT_EQ(222, FirstInst.getOperand(1).getImm()) + << "Instruction dump: " << FirstInstDump; + + AssertIacaMagicBytesInstruction( + GeneratedInstructions.at(GeneratedInstructions.size() - 1)); + } +} + } // namespace } // namespace exegesis } // namespace llvm