Index: tools/llvm-exegesis/lib/Latency.cpp =================================================================== --- tools/llvm-exegesis/lib/Latency.cpp +++ tools/llvm-exegesis/lib/Latency.cpp @@ -79,10 +79,27 @@ return EM; } +// Instanciates memory operands for instructions that happen to touch memory but +// are not serial through memory. We just need to ensure that all instruction in +// the serial sequence use non-overlapping memory spans. +static void instantiateMemoryOperands(const LLVMState &State, + CodeTemplate &CT) { + if (CT.ScratchSpacePointerInReg == 0) + return; // no memory operands. + const auto &ET = State.getExegesisTarget(); + const unsigned MemStep = ET.getMaxMemoryAccessSize(); + size_t I = 0; + for (InstructionTemplate &IT : CT.Instructions) { + ET.fillMemoryOperands(IT, CT.ScratchSpacePointerInReg, I * MemStep); + ++I; + } +} + static void appendCodeTemplates(const LLVMState &State, const Instruction &Instr, ExecutionMode ExecutionModeBit, llvm::StringRef ExecutionClassDescription, + const unsigned ScratchSpacePointerInReg, std::vector &CodeTemplates) { assert(isEnumValue(ExecutionModeBit) && "Bit must be a power of two"); switch (ExecutionModeBit) { @@ -93,9 +110,11 @@ // Picking whatever value for the tied variable will make the instruction // serial. CodeTemplate CT; + CT.ScratchSpacePointerInReg = ScratchSpacePointerInReg; CT.Execution = ExecutionModeBit; CT.Info = ExecutionClassDescription; CT.Instructions.push_back(Instr); + instantiateMemoryOperands(State, CT); CodeTemplates.push_back(std::move(CT)); return; } @@ -115,9 +134,11 @@ // instance, hence twice IT in the following call. setRandomAliasing(SelfAliasing, IT, IT); CodeTemplate CT; + CT.ScratchSpacePointerInReg = ScratchSpacePointerInReg; CT.Execution = ExecutionModeBit; CT.Info = ExecutionClassDescription; CT.Instructions.push_back(std::move(IT)); + instantiateMemoryOperands(State, CT); CodeTemplates.push_back(std::move(CT)); return; } @@ -134,10 +155,12 @@ if (!Back.hasImplicitAliasing()) setRandomAliasing(Back, OtherIT, ThisIT); CodeTemplate CT; + CT.ScratchSpacePointerInReg = ScratchSpacePointerInReg; CT.Execution = ExecutionModeBit; CT.Info = ExecutionClassDescription; CT.Instructions.push_back(std::move(ThisIT)); CT.Instructions.push_back(std::move(OtherIT)); + instantiateMemoryOperands(State, CT); CodeTemplates.push_back(std::move(CT)); } return; @@ -152,11 +175,17 @@ llvm::Expected> LatencySnippetGenerator::generateCodeTemplates(const Instruction &Instr) const { std::vector Results; + const llvm::BitVector *ScratchSpaceAliasedRegs; + unsigned ScratchSpacePointerInReg; + if (auto E = handleInstrWithMemoryOperand(Instr, ScratchSpacePointerInReg, ScratchSpaceAliasedRegs)) { + return std::move(E); + } + const ExecutionMode EM = getExecutionModes(Instr); for (const auto EC : kExecutionClasses) { for (const auto ExecutionModeBit : getExecutionModeBits(EM & EC.Mask)) appendCodeTemplates(State, Instr, ExecutionModeBit, EC.Description, - Results); + ScratchSpacePointerInReg, Results); if (!Results.empty()) break; } Index: tools/llvm-exegesis/lib/SnippetGenerator.h =================================================================== --- tools/llvm-exegesis/lib/SnippetGenerator.h +++ tools/llvm-exegesis/lib/SnippetGenerator.h @@ -67,6 +67,13 @@ protected: const LLVMState &State; + // A helper that checks whether Instr is supported (or has no memory operand). + // On success, it sets `ScratchSpacePointerInReg` and + // `ScratchSpaceAliasedRegs`. + Error handleInstrWithMemoryOperand( + const Instruction &Instr, unsigned &ScratchSpacePointerInReg, + const llvm::BitVector *&ScratchSpaceAliasedRegs) const; + private: // API to be implemented by subclasses. virtual llvm::Expected> Index: tools/llvm-exegesis/lib/SnippetGenerator.cpp =================================================================== --- tools/llvm-exegesis/lib/SnippetGenerator.cpp +++ tools/llvm-exegesis/lib/SnippetGenerator.cpp @@ -13,6 +13,7 @@ #include "Assembler.h" #include "MCInstrDescView.h" #include "SnippetGenerator.h" +#include "Target.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" @@ -108,6 +109,32 @@ return RIV; } +Error SnippetGenerator::handleInstrWithMemoryOperand( + const Instruction &Instr, unsigned &ScratchSpacePointerInReg, + const llvm::BitVector *&ScratchSpaceAliasedRegs) const { + ScratchSpacePointerInReg = 0; + ScratchSpaceAliasedRegs = nullptr; + if (Instr.hasMemoryOperands()) { + const auto &ET = State.getExegesisTarget(); + ScratchSpacePointerInReg = + ET.getScratchMemoryRegister(State.getTargetMachine().getTargetTriple()); + if (ScratchSpacePointerInReg == 0) + return llvm::make_error( + "Infeasible : target does not support memory instructions"); + ScratchSpaceAliasedRegs = + &State.getRATC().getRegister(ScratchSpacePointerInReg).aliasedBits(); + // If the instruction implicitly writes to ScratchSpacePointerInReg , abort. + // FIXME: We could make a copy of the scratch register. + for (const auto &Op : Instr.Operands) { + if (Op.isDef() && Op.isImplicitReg() && + ScratchSpaceAliasedRegs->test(Op.getImplicitReg())) + return llvm::make_error( + "Infeasible : memory instruction uses scratch memory register"); + } + } + return Error::success(); +} + llvm::Expected> generateSelfAliasingCodeTemplates(const Instruction &Instr) { const AliasingConfigurations SelfAliasing(Instr, Instr); Index: tools/llvm-exegesis/lib/Uops.cpp =================================================================== --- tools/llvm-exegesis/lib/Uops.cpp +++ tools/llvm-exegesis/lib/Uops.cpp @@ -127,24 +127,9 @@ llvm::Expected> UopsSnippetGenerator::generateCodeTemplates(const Instruction &Instr) const { CodeTemplate CT; - const llvm::BitVector *ScratchSpaceAliasedRegs = nullptr; - if (Instr.hasMemoryOperands()) { - const auto &ET = State.getExegesisTarget(); - CT.ScratchSpacePointerInReg = - ET.getScratchMemoryRegister(State.getTargetMachine().getTargetTriple()); - if (CT.ScratchSpacePointerInReg == 0) - return llvm::make_error( - "Infeasible : target does not support memory instructions"); - ScratchSpaceAliasedRegs = - &State.getRATC().getRegister(CT.ScratchSpacePointerInReg).aliasedBits(); - // If the instruction implicitly writes to ScratchSpacePointerInReg , abort. - // FIXME: We could make a copy of the scratch register. - for (const auto &Op : Instr.Operands) { - if (Op.isDef() && Op.isImplicitReg() && - ScratchSpaceAliasedRegs->test(Op.getImplicitReg())) - return llvm::make_error( - "Infeasible : memory instruction uses scratch memory register"); - } + const llvm::BitVector *ScratchSpaceAliasedRegs; + if (auto E = handleInstrWithMemoryOperand(Instr, CT.ScratchSpacePointerInReg, ScratchSpaceAliasedRegs)) { + return std::move(E); } const AliasingConfigurations SelfAliasing(Instr, Instr);