Index: lib/Target/X86/X86ISelLowering.cpp =================================================================== --- lib/Target/X86/X86ISelLowering.cpp +++ lib/Target/X86/X86ISelLowering.cpp @@ -5776,21 +5776,21 @@ } } - auto CreateLoad = [&DAG, &DL](EVT VT, LoadSDNode *LDBase) { + auto CreateLoad = [&DAG, &DL, NumElems, LoadMask, Elts](EVT VT, + LoadSDNode *LDBase) { SDValue NewLd = DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(), LDBase->getPointerInfo(), - LDBase->isVolatile(), LDBase->isNonTemporal(), + false /*isVolatile*/, LDBase->isNonTemporal(), LDBase->isInvariant(), LDBase->getAlignment()); - - if (LDBase->hasAnyUseOfValue(1)) { - SDValue NewChain = - DAG.getNode(ISD::TokenFactor, DL, MVT::Other, SDValue(LDBase, 1), - SDValue(NewLd.getNode(), 1)); - DAG.ReplaceAllUsesOfValueWith(SDValue(LDBase, 1), NewChain); - DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(LDBase, 1), - SDValue(NewLd.getNode(), 1)); - } - + // all references to chain for each component load should now point + // to new Load's chain + for (unsigned i = 0; i < NumElems; ++i) + if (LoadMask[i]) { + SDValue Elt = peekThroughBitcasts(Elts[i]); + LoadSDNode *Ld = cast(Elt); + DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), + SDValue(NewLd.getNode(), 1)); + } return NewLd; }; @@ -5848,18 +5848,15 @@ false/*isVolatile*/, true/*ReadMem*/, false/*WriteMem*/); - // Make sure the newly-created LOAD is in the same position as LDBase in - // terms of dependency. We create a TokenFactor for LDBase and ResNode, - // and update uses of LDBase's output chain to use the TokenFactor. - if (LDBase->hasAnyUseOfValue(1)) { - SDValue NewChain = - DAG.getNode(ISD::TokenFactor, DL, MVT::Other, SDValue(LDBase, 1), - SDValue(ResNode.getNode(), 1)); - DAG.ReplaceAllUsesOfValueWith(SDValue(LDBase, 1), NewChain); - DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(LDBase, 1), - SDValue(ResNode.getNode(), 1)); - } - + // all references to chain for each component load should now + // point to new Load's chain + for (unsigned i = 0; i < NumElems; ++i) + if (LoadMask[i]) { + SDValue Elt = peekThroughBitcasts(Elts[i]); + LoadSDNode *Ld = cast(Elt); + DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), + SDValue(ResNode.getNode(), 1)); + } return DAG.getBitcast(VT, ResNode); } } @@ -24046,16 +24043,9 @@ false/*WriteMem*/); // Make sure the newly-created LOAD is in the same position as Ld in - // terms of dependency. We create a TokenFactor for Ld and ResNode, - // and update uses of Ld's output chain to use the TokenFactor. - if (Ld->hasAnyUseOfValue(1)) { - SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, - SDValue(Ld, 1), SDValue(ResNode.getNode(), 1)); - DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), NewChain); - DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(Ld, 1), - SDValue(ResNode.getNode(), 1)); - } - + // terms of chain dependency. + DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), + SDValue(ResNode.getNode(), 1)); return DAG.getBitcast(VT, ResNode); } }