@@ -420,6 +420,8 @@ namespace {
420
420
return Subtarget->hasAVX512 ();
421
421
}
422
422
}
423
+
424
+ bool foldLoadStoreIntoMemOperand (SDNode *Node);
423
425
};
424
426
}
425
427
@@ -2035,6 +2037,57 @@ static unsigned getFusedLdStOpcode(EVT &LdVT, unsigned Opc) {
2035
2037
llvm_unreachable (" unrecognized size for LdVT" );
2036
2038
}
2037
2039
2040
+ // Change a chain of {load; incr or dec; store} of the same value into
2041
+ // a simple increment or decrement through memory of that value, if the
2042
+ // uses of the modified value and its address are suitable.
2043
+ // The DEC64m tablegen pattern is currently not able to match the case where
2044
+ // the EFLAGS on the original DEC are used. (This also applies to
2045
+ // {INC,DEC}X{64,32,16,8}.)
2046
+ // We'll need to improve tablegen to allow flags to be transferred from a
2047
+ // node in the pattern to the result node. probably with a new keyword
2048
+ // for example, we have this
2049
+ // def DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst), "dec{q}\t$dst",
2050
+ // [(store (add (loadi64 addr:$dst), -1), addr:$dst),
2051
+ // (implicit EFLAGS)]>;
2052
+ // but maybe need something like this
2053
+ // def DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst), "dec{q}\t$dst",
2054
+ // [(store (add (loadi64 addr:$dst), -1), addr:$dst),
2055
+ // (transferrable EFLAGS)]>;
2056
+ //
2057
+ // FIXME: This should handle a wide range of operations which support RMW
2058
+ // memory operands, not just inc and dec.
2059
+ bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand (SDNode *Node) {
2060
+ StoreSDNode *StoreNode = cast<StoreSDNode>(Node);
2061
+ SDValue StoredVal = StoreNode->getOperand (1 );
2062
+ unsigned Opc = StoredVal->getOpcode ();
2063
+
2064
+ LoadSDNode *LoadNode = nullptr ;
2065
+ SDValue InputChain;
2066
+ if (!isLoadIncOrDecStore (StoreNode, Opc, StoredVal, CurDAG, LoadNode,
2067
+ InputChain))
2068
+ return false ;
2069
+
2070
+ SDValue Base, Scale, Index, Disp, Segment;
2071
+ if (!selectAddr (LoadNode, LoadNode->getBasePtr (), Base, Scale, Index, Disp,
2072
+ Segment))
2073
+ return false ;
2074
+
2075
+ MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray (2 );
2076
+ MemOp[0 ] = StoreNode->getMemOperand ();
2077
+ MemOp[1 ] = LoadNode->getMemOperand ();
2078
+ const SDValue Ops[] = {Base, Scale, Index, Disp, Segment, InputChain};
2079
+ EVT LdVT = LoadNode->getMemoryVT ();
2080
+ unsigned newOpc = getFusedLdStOpcode (LdVT, Opc);
2081
+ MachineSDNode *Result =
2082
+ CurDAG->getMachineNode (newOpc, SDLoc (Node), MVT::i32, MVT::Other, Ops);
2083
+ Result->setMemRefs (MemOp, MemOp + 2 );
2084
+
2085
+ ReplaceUses (SDValue (StoreNode, 0 ), SDValue (Result, 1 ));
2086
+ ReplaceUses (SDValue (StoredVal.getNode (), 1 ), SDValue (Result, 0 ));
2087
+ CurDAG->RemoveDeadNode (Node);
2088
+ return true ;
2089
+ }
2090
+
2038
2091
void X86DAGToDAGISel::Select (SDNode *Node) {
2039
2092
MVT NVT = Node->getSimpleValueType (0 );
2040
2093
unsigned Opc, MOpc;
@@ -2697,55 +2750,10 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
2697
2750
}
2698
2751
break ;
2699
2752
}
2700
- case ISD::STORE: {
2701
- // Change a chain of {load; incr or dec; store} of the same value into
2702
- // a simple increment or decrement through memory of that value, if the
2703
- // uses of the modified value and its address are suitable.
2704
- // The DEC64m tablegen pattern is currently not able to match the case where
2705
- // the EFLAGS on the original DEC are used. (This also applies to
2706
- // {INC,DEC}X{64,32,16,8}.)
2707
- // We'll need to improve tablegen to allow flags to be transferred from a
2708
- // node in the pattern to the result node. probably with a new keyword
2709
- // for example, we have this
2710
- // def DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst), "dec{q}\t$dst",
2711
- // [(store (add (loadi64 addr:$dst), -1), addr:$dst),
2712
- // (implicit EFLAGS)]>;
2713
- // but maybe need something like this
2714
- // def DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst), "dec{q}\t$dst",
2715
- // [(store (add (loadi64 addr:$dst), -1), addr:$dst),
2716
- // (transferrable EFLAGS)]>;
2717
-
2718
- StoreSDNode *StoreNode = cast<StoreSDNode>(Node);
2719
- SDValue StoredVal = StoreNode->getOperand (1 );
2720
- unsigned Opc = StoredVal->getOpcode ();
2721
-
2722
- LoadSDNode *LoadNode = nullptr ;
2723
- SDValue InputChain;
2724
- if (!isLoadIncOrDecStore (StoreNode, Opc, StoredVal, CurDAG,
2725
- LoadNode, InputChain))
2726
- break ;
2727
-
2728
- SDValue Base, Scale, Index, Disp, Segment;
2729
- if (!selectAddr (LoadNode, LoadNode->getBasePtr (),
2730
- Base, Scale, Index, Disp, Segment))
2731
- break ;
2732
-
2733
- MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray (2 );
2734
- MemOp[0 ] = StoreNode->getMemOperand ();
2735
- MemOp[1 ] = LoadNode->getMemOperand ();
2736
- const SDValue Ops[] = { Base, Scale, Index, Disp, Segment, InputChain };
2737
- EVT LdVT = LoadNode->getMemoryVT ();
2738
- unsigned newOpc = getFusedLdStOpcode (LdVT, Opc);
2739
- MachineSDNode *Result = CurDAG->getMachineNode (newOpc,
2740
- SDLoc (Node),
2741
- MVT::i32, MVT::Other, Ops);
2742
- Result->setMemRefs (MemOp, MemOp + 2 );
2743
-
2744
- ReplaceUses (SDValue (StoreNode, 0 ), SDValue (Result, 1 ));
2745
- ReplaceUses (SDValue (StoredVal.getNode (), 1 ), SDValue (Result, 0 ));
2746
- CurDAG->RemoveDeadNode (Node);
2747
- return ;
2748
- }
2753
+ case ISD::STORE:
2754
+ if (foldLoadStoreIntoMemOperand (Node))
2755
+ return ;
2756
+ break ;
2749
2757
}
2750
2758
2751
2759
SelectCode (Node);
0 commit comments