Index: utils/TableGen/X86FoldTablesEmitter.cpp =================================================================== --- utils/TableGen/X86FoldTablesEmitter.cpp +++ utils/TableGen/X86FoldTablesEmitter.cpp @@ -14,14 +14,120 @@ #include "CodeGenDAGPatterns.h" #include "CodeGenTarget.h" +#include "X86RecognizableInstr.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/TableGenBackend.h" -#include "X86RecognizableInstr.h" using namespace llvm; namespace { +// 3 possible strategies for the unfolding flag (TB_NO_REVERSE) of the +// manual added entries. +enum UnfoldStrategy { + UNFOLD, // Allow unfolding + NO_UNFOLD, // Prevent unfolding + NO_STRATEGY // Make decision according to operands' sizes +}; + +// Represents an entry in the manual mapped instructions set. +struct ManualMapEntry { + const char *RegInstStr; + const char *MemInstStr; + UnfoldStrategy Strategy; + + ManualMapEntry(const char *RegInstStr, const char *MemInstStr, + UnfoldStrategy Strategy = NO_STRATEGY) + : RegInstStr(RegInstStr), MemInstStr(MemInstStr), Strategy(Strategy) {} +}; + +class IsMatch; + +// List of instructions requiring explicitly aligned memory. +const char *ExplicitAlign[] = {"MOVDQA", "MOVAPS", "MOVAPD", "MOVNTPS", + "MOVNTPD", "MOVNTDQ", "MOVNTDQA"}; + +// List of instructions NOT requiring explicit memory alignment. +const char *ExplicitUnalign[] = {"MOVDQU", "MOVUPS", "MOVUPD"}; + +// For manually mapping instructions that do not match by their encoding. +const ManualMapEntry ManualMapSet[] = { + { "ADD16ri_DB", "ADD16mi", NO_UNFOLD }, + { "ADD16ri8_DB", "ADD16mi8", NO_UNFOLD }, + { "ADD16rr_DB", "ADD16mr", NO_UNFOLD }, + { "ADD32ri_DB", "ADD32mi", NO_UNFOLD }, + { "ADD32ri8_DB", "ADD32mi8", NO_UNFOLD }, + { "ADD32rr_DB", "ADD32mr", NO_UNFOLD }, + { "ADD64ri32_DB", "ADD64mi32", NO_UNFOLD }, + { "ADD64ri8_DB", "ADD64mi8", NO_UNFOLD }, + { "ADD64rr_DB", "ADD64mr", NO_UNFOLD }, + { "ADD16rr_DB", "ADD16rm", NO_UNFOLD }, + { "ADD32rr_DB", "ADD32rm", NO_UNFOLD }, + { "ADD64rr_DB", "ADD64rm", NO_UNFOLD }, + { "PUSH16r", "PUSH16rmm", NO_UNFOLD }, + { "PUSH32r", "PUSH32rmm", NO_UNFOLD }, + { "PUSH64r", "PUSH64rmm", NO_UNFOLD }, + { "TAILJMPr", "TAILJMPm", UNFOLD }, + { "TAILJMPr64", "TAILJMPm64", UNFOLD }, + { "TAILJMPr64_REX", "TAILJMPm64_REX", UNFOLD }, +}; + +// Do not add these instructions to any of the folding tables. +const std::vector NoFoldSet = { + "TCRETURNri64", + "TCRETURNmi64", // Special dealing (in X86InstrCompiler.td under + "TCRETURNri", // "tailcall stuff" section). + "TCRETURNmi" + + // Different calculations of the folded operand between + // memory and register forms (folding is illegal). + // - In their register form, the second register operand's relevant + // bits are only the first 4/5/6 (depending on mode and reg size). + // - In their memory form, the second register operand's relevant + // bits are only the first 16/32/64 (depending on mode and reg size). + "BT16rr", "BT32rr", "BT64rr", + "BT16mr", "BT32mr", "BT64mr", + "BTC16rr", "BTC32rr", "BTC64rr", + "BTC16mr", "BTC32mr", "BTC64mr", + "BTR16rr", "BTR32rr", "BTR64rr", + "BTR16mr", "BTR32mr", "BTR64mr", + "BTS16rr", "BTS32rr", "BTS64rr", + "BTS16mr", "BTS32mr", "BTS64mr", + + // Memory folding is enabled only when optimizing for size by DAG + // patterns only. (issue detailed in D28744 review) + "VCVTSS2SDrm", "VCVTSS2SDrr", + "VCVTSS2SDZrm", "VCVTSS2SDZrr", + "VCVTSS2SDZrmk", "VCVTSS2SDZrrk", + "VCVTSS2SDZrmkz", "VCVTSS2SDZrrkz", + "VCVTSS2SDZrm_Int", "VCVTSS2SDZrr_Int", + "VCVTSS2SDZrm_Intk", "VCVTSS2SDZrr_Intk", + "VCVTSS2SDZrm_Intkz", "VCVTSS2SDZrr_Intkz", + "VCVTSD2SSrm", "VCVTSD2SSrr", + "VCVTSD2SSZrm", "VCVTSD2SSZrr", + "VCVTSD2SSZrmk", "VCVTSD2SSZrrk", + "VCVTSD2SSZrmkz", "VCVTSD2SSZrrkz", + "VCVTSD2SSZrm_Int", "VCVTSD2SSZrr_Int", + "VCVTSD2SSZrm_Intk", "VCVTSD2SSZrr_Intk", + "VCVTSD2SSZrm_Intkz", "VCVTSD2SSZrr_Intkz", + "VRCP14SSrm", "VRCP14SSrr", + "VRCP14SDrm", "VRCP14SDrr", + "VRSQRT14SSrm", "VRSQRT14SSrr", + "VRSQRT14SDrm", "VRSQRT14SDrr", + "VSQRTSSm", "VSQRTSSr", + "VSQRTSSm_Int", "VSQRTSSr_Int", + "VSQRTSSZm", "VSQRTSSZr", + "VSQRTSSZm_Int", "VSQRTSSZr_Int", + "VSQRTSSZm_Intk", "VSQRTSSZr_Intk", + "VSQRTSSZm_Intkz", "VSQRTSSZr_Intkz", + "VSQRTSDm", "VSQRTSDr", + "VSQRTSDm_Int", "VSQRTSDr_Int", + "VSQRTSDZm", "VSQRTSDZr", + "VSQRTSDZm_Int", "VSQRTSDZr_Int", + "VSQRTSDZm_Intk", "VSQRTSDZr_Intk", + "VSQRTSDZm_Intkz", "VSQRTSDZr_Intkz", +}; + class X86FoldTablesEmitter { RecordKeeper &Records; CodeGenTarget Target; @@ -32,21 +138,15 @@ const CodeGenInstruction *MemInst; public: - bool CannotUnfold; - bool IsLoad; - bool IsStore; - bool IsAligned; - unsigned int Alignment; + bool CannotUnfold = false; + bool IsLoad = false; + bool IsStore = false; + bool IsAligned = false; + unsigned int Alignment = 0; X86FoldTableEntry(const CodeGenInstruction *RegInst, const CodeGenInstruction *MemInst) - : RegInst(RegInst), MemInst(MemInst) { - CannotUnfold = false; - IsLoad = false; - IsStore = false; - IsAligned = false; - Alignment = 0; - } + : RegInst(RegInst), MemInst(MemInst) {} friend raw_ostream &operator<<(raw_ostream &OS, const X86FoldTableEntry &E) { @@ -83,113 +183,6 @@ FoldTable Table3; FoldTable Table4; - // 3 possible strategies for the unfolding flag (TB_NO_REVERSE) of the - // manual added entries. - typedef enum { - UNFOLD, // Allow unfolding - NO_UNFOLD, // Prevent unfolding - NO_STRATEGY // Make decision according to operands' sizes - } UnfoldStrategy; - - // Represents an entry in the manual mapped instructions set. - struct ManualMapEntry { - const char *RegInstStr; - const char *MemInstStr; - UnfoldStrategy Strategy; - - ManualMapEntry(const char *RegInstStr, const char *MemInstStr, - UnfoldStrategy Strategy = NO_STRATEGY) - : RegInstStr(RegInstStr), MemInstStr(MemInstStr), Strategy(Strategy) {} - }; - - class IsMatch; - - // List of instructions requiring explicitly aligned memory. - const StringRef ExplicitAlign[7] = {"MOVDQA", "MOVAPS", "MOVAPD", - "MOVNTPS", "MOVNTPD", "MOVNTDQ", - "MOVNTDQA"}; - - // List of instructions NOT requiring explicit memory alignment. - const StringRef ExplicitUnalign[3] = {"MOVDQU", "MOVUPS", "MOVUPD"}; - - // For manually mapping instructions that do not match by their encoding. - const std::vector ManualMapSet = { - { "ADD16ri_DB", "ADD16mi", NO_UNFOLD }, - { "ADD16ri8_DB", "ADD16mi8", NO_UNFOLD }, - { "ADD16rr_DB", "ADD16mr", NO_UNFOLD }, - { "ADD32ri_DB", "ADD32mi", NO_UNFOLD }, - { "ADD32ri8_DB", "ADD32mi8", NO_UNFOLD }, - { "ADD32rr_DB", "ADD32mr", NO_UNFOLD }, - { "ADD64ri32_DB", "ADD64mi32", NO_UNFOLD }, - { "ADD64ri8_DB", "ADD64mi8", NO_UNFOLD }, - { "ADD64rr_DB", "ADD64mr", NO_UNFOLD }, - { "ADD16rr_DB", "ADD16rm", NO_UNFOLD }, - { "ADD32rr_DB", "ADD32rm", NO_UNFOLD }, - { "ADD64rr_DB", "ADD64rm", NO_UNFOLD }, - { "PUSH16r", "PUSH16rmm", NO_UNFOLD }, - { "PUSH32r", "PUSH32rmm", NO_UNFOLD }, - { "PUSH64r", "PUSH64rmm", NO_UNFOLD }, - { "TAILJMPr", "TAILJMPm", UNFOLD }, - { "TAILJMPr64", "TAILJMPm64", UNFOLD }, - { "TAILJMPr64_REX", "TAILJMPm64_REX", UNFOLD }, - }; - - // Do not add these instructions to any of the folding tables. - const std::vector NoFoldSet = { - "TCRETURNri64", - "TCRETURNmi64", // Special dealing (in X86InstrCompiler.td under - "TCRETURNri", // "tailcall stuff" section). - "TCRETURNmi" - - // Different calculations of the folded operand between - // memory and register forms (folding is illegal). - // - In their register form, the second register operand's relevant - // bits are only the first 4/5/6 (depending on mode and reg size). - // - In their memory form, the second register operand's relevant - // bits are only the first 16/32/64 (depending on mode and reg size). - "BT16rr", "BT32rr", "BT64rr", - "BT16mr", "BT32mr", "BT64mr", - "BTC16rr", "BTC32rr", "BTC64rr", - "BTC16mr", "BTC32mr", "BTC64mr", - "BTR16rr", "BTR32rr", "BTR64rr", - "BTR16mr", "BTR32mr", "BTR64mr", - "BTS16rr", "BTS32rr", "BTS64rr", - "BTS16mr", "BTS32mr", "BTS64mr", - - // Memory folding is enabled only when optimizing for size by DAG - // patterns only. (issue detailed in D28744 review) - "VCVTSS2SDrm", "VCVTSS2SDrr", - "VCVTSS2SDZrm", "VCVTSS2SDZrr", - "VCVTSS2SDZrmk", "VCVTSS2SDZrrk", - "VCVTSS2SDZrmkz", "VCVTSS2SDZrrkz", - "VCVTSS2SDZrm_Int", "VCVTSS2SDZrr_Int", - "VCVTSS2SDZrm_Intk", "VCVTSS2SDZrr_Intk", - "VCVTSS2SDZrm_Intkz", "VCVTSS2SDZrr_Intkz", - "VCVTSD2SSrm", "VCVTSD2SSrr", - "VCVTSD2SSZrm", "VCVTSD2SSZrr", - "VCVTSD2SSZrmk", "VCVTSD2SSZrrk", - "VCVTSD2SSZrmkz", "VCVTSD2SSZrrkz", - "VCVTSD2SSZrm_Int", "VCVTSD2SSZrr_Int", - "VCVTSD2SSZrm_Intk", "VCVTSD2SSZrr_Intk", - "VCVTSD2SSZrm_Intkz", "VCVTSD2SSZrr_Intkz", - "VRCP14SSrm", "VRCP14SSrr", - "VRCP14SDrm", "VRCP14SDrr", - "VRSQRT14SSrm", "VRSQRT14SSrr", - "VRSQRT14SDrm", "VRSQRT14SDrr", - "VSQRTSSm", "VSQRTSSr", - "VSQRTSSm_Int", "VSQRTSSr_Int", - "VSQRTSSZm", "VSQRTSSZr", - "VSQRTSSZm_Int", "VSQRTSSZr_Int", - "VSQRTSSZm_Intk", "VSQRTSSZr_Intk", - "VSQRTSSZm_Intkz", "VSQRTSSZr_Intkz", - "VSQRTSDm", "VSQRTSDr", - "VSQRTSDm_Int", "VSQRTSDr_Int", - "VSQRTSDZm", "VSQRTSDZr", - "VSQRTSDZm_Int", "VSQRTSDZr_Int", - "VSQRTSDZm_Intk", "VSQRTSDZr_Intk", - "VSQRTSDZm_Intkz", "VSQRTSDZr_Intkz", - }; - public: X86FoldTablesEmitter(RecordKeeper &R) : Records(R), Target(R) {} @@ -198,19 +191,19 @@ private: // Decides to which table to add the entry with the given instructions. - // Strgy sets the strategy of adding the TB_NO_REVERSE flag. + // S sets the strategy of adding the TB_NO_REVERSE flag. void updateTables(const CodeGenInstruction *RegInstr, const CodeGenInstruction *MemInstr, - const UnfoldStrategy Strgy = NO_STRATEGY); + const UnfoldStrategy S = NO_STRATEGY); // Generates X86FoldTableEntry with the given instructions and fill it with // the appropriate flags - then adds it to Table. void addEntryWithFlags(FoldTable &Table, const CodeGenInstruction *RegInstr, const CodeGenInstruction *MemInstr, - const UnfoldStrategy Strgy); + const UnfoldStrategy S, const unsigned int FoldedInd); bool isExplicitAlign(const CodeGenInstruction *Inst) { - for (const StringRef &InstStr : ExplicitAlign) { + for (const char *InstStr : ExplicitAlign) { if (Inst->TheDef->getName().find(InstStr) != StringRef::npos) return true; } @@ -218,35 +211,17 @@ } bool isExplicitUnalign(const CodeGenInstruction *Inst) { - for (const StringRef &InstStr : ExplicitUnalign) { + for (const char *InstStr : ExplicitUnalign) { if (Inst->TheDef->getName().find(InstStr) != StringRef::npos) return true; } return false; } - // Return the index of the folded operand for each table - unsigned int getIndexOfFoldedOperand(const FoldTable &Table) { - if (&Table == &Table2Addr) - return 0; - if (&Table == &Table0) - return 0; - if (&Table == &Table1) - return 1; - if (&Table == &Table2) - return 2; - if (&Table == &Table3) - return 3; - if (&Table == &Table4) - return 4; - - llvm_unreachable("No such table!"); - } - // Print the given table as a static const C++ array of type // X86MemoryFoldTableEntry. - inline void printTable(const FoldTable &Table, std::string TableName, - raw_ostream &OS) { + void printTable(const FoldTable &Table, std::string TableName, + raw_ostream &OS) { OS << "static const X86MemoryFoldTableEntry MemoryFold" << TableName << "[] = {\n"; @@ -257,30 +232,29 @@ } // Return true if one of the instruction's operands is a RST register class - inline bool hasRSTRegClass(const CodeGenInstruction *Inst) { - + bool hasRSTRegClass(const CodeGenInstruction *Inst) { return llvm::any_of(Inst->Operands, [](const CGIOperandList::OperandInfo &OpIn) { - return OpIn.Rec->getName() == "RST"; - }); + return OpIn.Rec->getName() == "RST"; + }); } - // Return true if one of the instruction's operands is a RST register class - inline bool hasPtrTailcallRegClass(const CodeGenInstruction *Inst) { - + // Return true if one of the instruction's operands is a ptr_rc_tailcall + bool hasPtrTailcallRegClass(const CodeGenInstruction *Inst) { return llvm::any_of(Inst->Operands, [](const CGIOperandList::OperandInfo &OpIn) { - return OpIn.Rec->getName() == "ptr_rc_tailcall"; - }); + return OpIn.Rec->getName() == "ptr_rc_tailcall"; + }); } }; -} // end anonymous namespace // Calculates the integer value representing the BitsInit object static inline uint64_t getValueFromBitsInit(const BitsInit *B) { + assert(B->getNumBits() <= sizeof(uint64_t) * 8 && "BitInits' too long!"); + uint64_t Value = 0; for (unsigned i = 0, e = B->getNumBits(); i != e; ++i) { - if (BitInit *Bit = dyn_cast(B->getBit(i))) + if (BitInit *Bit = cast(B->getBit(i))) Value |= uint64_t(Bit->getValue()) << i; else PrintFatalError("Invalid VectSize bit"); @@ -294,8 +268,8 @@ PrintFatalError("Comparing two BitsInits with different sizes!"); for (unsigned i = 0, e = B1->getNumBits(); i != e; ++i) { - if (BitInit *Bit1 = dyn_cast(B1->getBit(i))) { - if (BitInit *Bit2 = dyn_cast(B2->getBit(i))) { + if (BitInit *Bit1 = cast(B1->getBit(i))) { + if (BitInit *Bit2 = cast(B2->getBit(i))) { if (Bit1->getValue() != Bit2->getValue()) return false; } else @@ -308,10 +282,10 @@ // Return the size of the register operand static inline unsigned int getRegOperandSize(const Record *RegRec) { - if (RegRec->isSubClassOf("RegisterClass")) - return RegRec->getValueAsInt("Alignment"); if (RegRec->isSubClassOf("RegisterOperand")) - return RegRec->getValueAsDef("RegClass")->getValueAsInt("Alignment"); + RegRec = RegRec->getValueAsDef("RegClass"); + if (RegRec->isSubClassOf("RegisterClass")) + return RegRec->getValueAsListOfDefs("RegTypes")[0]->getValueAsInt("Size"); llvm_unreachable("Register operand's size not known!"); } @@ -349,11 +323,12 @@ } // Returns true if the record's list of defs includes the given def. -static bool hasDefInList(const Record *Rec, const StringRef List, - const StringRef Def) { +static inline bool hasDefInList(const Record *Rec, const StringRef List, + const StringRef Def) { if (!Rec->isValueUnset(List)) { - return llvm::any_of(*(Rec->getValueAsListInit(List)), - [Def](const Init *I) {return I->getAsString() == Def;}); + return llvm::any_of(*(Rec->getValueAsListInit(List)), [Def](const Init *I) { + return I->getAsString() == Def; + }); } return false; } @@ -409,130 +384,9 @@ return &AltRegInst; } -void X86FoldTablesEmitter::addEntryWithFlags(FoldTable &Table, - const CodeGenInstruction *RegInstr, - const CodeGenInstruction *MemInstr, - const UnfoldStrategy Strgy) { - - X86FoldTableEntry Result = X86FoldTableEntry(RegInstr, MemInstr); - Record *RegRec = RegInstr->TheDef; - Record *MemRec = MemInstr->TheDef; - - // Only table0 entries should explicitly specify a load or store flag. - if (&Table == &Table0) { - unsigned MemInOpsNum = MemRec->getValueAsDag("InOperandList")->getNumArgs(); - unsigned RegInOpsNum = RegRec->getValueAsDag("InOperandList")->getNumArgs(); - // If the instruction writes to the folded operand, it will appear as an - // output in the register form instruction and as an input in the memory - // form instruction. - // If the instruction reads from the folded operand, it well appear as in - // input in both forms. - if (MemInOpsNum == RegInOpsNum) - Result.IsLoad = true; - else - Result.IsStore = true; - } - - unsigned int FoldedInd = getIndexOfFoldedOperand(Table); - Record *RegOpRec = RegInstr->Operands[FoldedInd].Rec; - Record *MemOpRec = MemInstr->Operands[FoldedInd].Rec; - - // Unfolding code generates a load/store instruction according to the size of - // the register in the register form instruction. - // If the register's size is greater than the memory's operand size, do not - // allow unfolding. - if (Strgy == UNFOLD) - Result.CannotUnfold = false; - else if (Strgy == NO_UNFOLD) - Result.CannotUnfold = true; - else if (getRegOperandSize(RegOpRec) > getMemOperandSize(MemOpRec)) - Result.CannotUnfold = true; // Strgy == NO_STRATEGY - - if (isExplicitAlign(RegInstr)) { - // The instruction require explicitly aligned memory. - BitsInit *VectSize = RegRec->getValueAsBitsInit("VectSize"); - uint64_t Value = getValueFromBitsInit(VectSize); - Result.IsAligned = true; - Result.Alignment = Value; - } else if (!RegRec->isSubClassOf("VEX")) { - // Instructions with VEX encoding do not require alignment. - if (!isExplicitUnalign(RegInstr) && getMemOperandSize(MemOpRec) > 64 && - !RegInstr->TheDef->isSubClassOf("XOP")) { - // SSE packed vector instructions require a 16 byte alignment. - Result.IsAligned = true; - Result.Alignment = 16; - } - } - - Table.push_back(Result); -} - -void X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInstr, - const CodeGenInstruction *MemInstr, - const UnfoldStrategy Strgy) { - - Record *RegRec = RegInstr->TheDef; - Record *MemRec = MemInstr->TheDef; - unsigned MemOutSize = MemRec->getValueAsDag("OutOperandList")->getNumArgs(); - unsigned RegOutSize = RegRec->getValueAsDag("OutOperandList")->getNumArgs(); - unsigned MemInSize = MemRec->getValueAsDag("InOperandList")->getNumArgs(); - unsigned RegInSize = RegRec->getValueAsDag("InOperandList")->getNumArgs(); - - // Instructions which have the WriteRMW value (Read-Modify-Write) should be - // added to Table2Addr. - if (hasDefInList(MemRec, "SchedRW", "WriteRMW") && MemOutSize != RegOutSize && - MemInSize == RegInSize) { - addEntryWithFlags(Table2Addr, RegInstr, MemInstr, Strgy); - return; - } - - if (MemInSize == RegInSize && MemOutSize == RegOutSize) { - // Load-Folding cases. - // If the i'th register form operand is a register and the i'th memory form - // operand is a memory operand, add instructions to Table#i. - for (unsigned i = RegOutSize; i < RegInstr->Operands.size(); i++) { - Record *RegOpRec = RegInstr->Operands[i].Rec; - Record *MemOpRec = MemInstr->Operands[i].Rec; - if (isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec)) { - switch (i) { - case 0: - addEntryWithFlags(Table0, RegInstr, MemInstr, Strgy); - return; - case 1: - addEntryWithFlags(Table1, RegInstr, MemInstr, Strgy); - return; - case 2: - addEntryWithFlags(Table2, RegInstr, MemInstr, Strgy); - return; - case 3: - addEntryWithFlags(Table3, RegInstr, MemInstr, Strgy); - return; - case 4: - addEntryWithFlags(Table4, RegInstr, MemInstr, Strgy); - return; - } - } - } - } else if (MemInSize == RegInSize + 1 && MemOutSize + 1 == RegOutSize) { - // Store-Folding cases. - // If the memory form instruction performs performs a store, the *output* - // register of the register form instructions disappear and instead a - // memory *input* operand appears in the memory form instruction. - // For example: - // MOVAPSrr => (outs VR128:$dst), (ins VR128:$src) - // MOVAPSmr => (outs), (ins f128mem:$dst, VR128:$src) - Record *RegOpRec = RegInstr->Operands[RegOutSize - 1].Rec; - Record *MemOpRec = MemInstr->Operands[RegOutSize - 1].Rec; - if (isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec)) - addEntryWithFlags(Table0, RegInstr, MemInstr, Strgy); - } - - return; -} - // Function object - Operator() returns true if the given VEX instruction // matches the EVEX instruction of this object. -class X86FoldTablesEmitter::IsMatch { +class IsMatch { const CodeGenInstruction *MemInst; const RecordKeeper &Records; @@ -629,9 +483,9 @@ } private: -// Return true of the 2 given forms are the opposite of each other. + // Return true of the 2 given forms are the opposite of each other. bool areOppositeForms(const BitsInit *RegFormBits, - const BitsInit *MemFormBits) { + const BitsInit *MemFormBits) { uint64_t MemFormNum = getValueFromBitsInit(MemFormBits); uint64_t RegFormNum = getValueFromBitsInit(RegFormBits); @@ -645,19 +499,142 @@ (MemFormNum == X86Local::MRM7m && RegFormNum == X86Local::MRM7r) || (MemFormNum == X86Local::MRMXm && RegFormNum == X86Local::MRMXr) || (MemFormNum == X86Local::MRMDestMem && - RegFormNum == X86Local::MRMDestReg) || + RegFormNum == X86Local::MRMDestReg) || (MemFormNum == X86Local::MRMSrcMem && - RegFormNum == X86Local::MRMSrcReg) || + RegFormNum == X86Local::MRMSrcReg) || (MemFormNum == X86Local::MRMSrcMem4VOp3 && - RegFormNum == X86Local::MRMSrcReg4VOp3) || + RegFormNum == X86Local::MRMSrcReg4VOp3) || (MemFormNum == X86Local::MRMSrcMemOp4 && - RegFormNum == X86Local::MRMSrcRegOp4)) + RegFormNum == X86Local::MRMSrcRegOp4)) return true; return false; } }; +} // end anonymous namespace + +void X86FoldTablesEmitter::addEntryWithFlags(FoldTable &Table, + const CodeGenInstruction *RegInstr, + const CodeGenInstruction *MemInstr, + const UnfoldStrategy S, + const unsigned int FoldedInd) { + + X86FoldTableEntry Result = X86FoldTableEntry(RegInstr, MemInstr); + Record *RegRec = RegInstr->TheDef; + Record *MemRec = MemInstr->TheDef; + + // Only table0 entries should explicitly specify a load or store flag. + if (&Table == &Table0) { + unsigned MemInOpsNum = MemRec->getValueAsDag("InOperandList")->getNumArgs(); + unsigned RegInOpsNum = RegRec->getValueAsDag("InOperandList")->getNumArgs(); + // If the instruction writes to the folded operand, it will appear as an + // output in the register form instruction and as an input in the memory + // form instruction. + // If the instruction reads from the folded operand, it well appear as in + // input in both forms. + if (MemInOpsNum == RegInOpsNum) + Result.IsLoad = true; + else + Result.IsStore = true; + } + + Record *RegOpRec = RegInstr->Operands[FoldedInd].Rec; + Record *MemOpRec = MemInstr->Operands[FoldedInd].Rec; + + // Unfolding code generates a load/store instruction according to the size of + // the register in the register form instruction. + // If the register's size is greater than the memory's operand size, do not + // allow unfolding. + if (S == UNFOLD) + Result.CannotUnfold = false; + else if (S == NO_UNFOLD) + Result.CannotUnfold = true; + else if (getRegOperandSize(RegOpRec) > getMemOperandSize(MemOpRec)) + Result.CannotUnfold = true; // S == NO_STRATEGY + + if (isExplicitAlign(RegInstr)) { + // The instruction require explicitly aligned memory. + BitsInit *VectSize = RegRec->getValueAsBitsInit("VectSize"); + uint64_t Value = getValueFromBitsInit(VectSize); + Result.IsAligned = true; + Result.Alignment = Value; + } else if (!RegRec->isSubClassOf("VEX")) { + // Instructions with VEX encoding do not require alignment. + if (!isExplicitUnalign(RegInstr) && getMemOperandSize(MemOpRec) > 64 && + !RegInstr->TheDef->isSubClassOf("XOP")) { + // SSE packed vector instructions require a 16 byte alignment. + Result.IsAligned = true; + Result.Alignment = 16; + } + } + + Table.push_back(Result); +} + +void X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInstr, + const CodeGenInstruction *MemInstr, + const UnfoldStrategy S) { + + Record *RegRec = RegInstr->TheDef; + Record *MemRec = MemInstr->TheDef; + unsigned MemOutSize = MemRec->getValueAsDag("OutOperandList")->getNumArgs(); + unsigned RegOutSize = RegRec->getValueAsDag("OutOperandList")->getNumArgs(); + unsigned MemInSize = MemRec->getValueAsDag("InOperandList")->getNumArgs(); + unsigned RegInSize = RegRec->getValueAsDag("InOperandList")->getNumArgs(); + + // Instructions which have the WriteRMW value (Read-Modify-Write) should be + // added to Table2Addr. + if (hasDefInList(MemRec, "SchedRW", "WriteRMW") && MemOutSize != RegOutSize && + MemInSize == RegInSize) { + addEntryWithFlags(Table2Addr, RegInstr, MemInstr, S, 0); + return; + } + + if (MemInSize == RegInSize && MemOutSize == RegOutSize) { + // Load-Folding cases. + // If the i'th register form operand is a register and the i'th memory form + // operand is a memory operand, add instructions to Table#i. + for (unsigned i = RegOutSize; i < RegInstr->Operands.size(); i++) { + Record *RegOpRec = RegInstr->Operands[i].Rec; + Record *MemOpRec = MemInstr->Operands[i].Rec; + if (isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec)) { + switch (i) { + case 0: + addEntryWithFlags(Table0, RegInstr, MemInstr, S, 0); + return; + case 1: + addEntryWithFlags(Table1, RegInstr, MemInstr, S, 1); + return; + case 2: + addEntryWithFlags(Table2, RegInstr, MemInstr, S, 2); + return; + case 3: + addEntryWithFlags(Table3, RegInstr, MemInstr, S, 3); + return; + case 4: + addEntryWithFlags(Table4, RegInstr, MemInstr, S, 4); + return; + } + } + } + } else if (MemInSize == RegInSize + 1 && MemOutSize + 1 == RegOutSize) { + // Store-Folding cases. + // If the memory form instruction performs performs a store, the *output* + // register of the register form instructions disappear and instead a + // memory *input* operand appears in the memory form instruction. + // For example: + // MOVAPSrr => (outs VR128:$dst), (ins VR128:$src) + // MOVAPSmr => (outs), (ins f128mem:$dst, VR128:$src) + Record *RegOpRec = RegInstr->Operands[RegOutSize - 1].Rec; + Record *MemOpRec = MemInstr->Operands[RegOutSize - 1].Rec; + if (isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec)) + addEntryWithFlags(Table0, RegInstr, MemInstr, S, 0); + } + + return; +} + void X86FoldTablesEmitter::run(raw_ostream &OS) { emitSourceFileHeader("X86 fold tables", OS); @@ -721,18 +698,17 @@ // this field if (RegInst->TheDef->isValueUnset("FoldGenRegForm")) { updateTables(RegInst, MemInst); - OpcRegInsts.erase(Match); } else { const CodeGenInstruction *AltRegInst = getAltRegInst(RegInst, Records, Target); updateTables(AltRegInst, MemInst); } + OpcRegInsts.erase(Match); } } // Add the manually mapped instructions listed above. for (const ManualMapEntry &Entry : ManualMapSet) { - Record *RegInstIter = Records.getDef(Entry.RegInstStr); Record *MemInstIter = Records.getDef(Entry.MemInstStr);