diff --git a/llvm/lib/Target/PowerPC/PPCLoopPreIncPrep.cpp b/llvm/lib/Target/PowerPC/PPCLoopPreIncPrep.cpp --- a/llvm/lib/Target/PowerPC/PPCLoopPreIncPrep.cpp +++ b/llvm/lib/Target/PowerPC/PPCLoopPreIncPrep.cpp @@ -6,16 +6,39 @@ // //===----------------------------------------------------------------------===// // -// This file implements a pass to prepare loops for pre-increment addressing -// modes. Additional PHIs are created for loop induction variables used by -// load/store instructions so that the pre-increment forms can be used. -// Generically, this means transforming loops like this: -// for (int i = 0; i < n; ++i) -// array[i] = c; -// to look like this: -// T *p = array[-1]; -// for (int i = 0; i < n; ++i) -// *++p = c; +// This file implements a pass to prepare loops for ppc preferred addressing +// modes, leveraging different instruction form. (eg: DS/DQ form, D/DS form with +// update) +// Additional PHIs are created for loop induction variables used by load/store +// instructions so that preferred addressing modes can be used. +// +// 1: DS/DQ form preparation, prepare the load/store instructions so that they +// can satisfy the DS/DQ form displacement requirements. +// Generically, this means transforming loops like this: +// for (int i = 0; i < n; ++i) { +// unsigned long x1 = *(unsigned long *)(p + i + 5); +// unsigned long x2 = *(unsigned long *)(p + i + 9); +// } +// +// to look like this: +// +// unsigned NewP = p + 5; +// for (int i = 0; i < n; ++i) { +// unsigned long x1 = *(unsigned long *)(i + NewP); +// unsigned long x2 = *(unsigned long *)(i + NewP + 4); +// } +// +// 2: D/DS form with update preparation, prepare the load/store instructions so +// that we can use update form to do pre-increment. +// Generically, this means transforming loops like this: +// for (int i = 0; i < n; ++i) +// array[i] = c; +// +// to look like this: +// +// T *p = array[-1]; +// for (int i = 0; i < n; ++i) +// *++p = c; //===----------------------------------------------------------------------===// #define DEBUG_TYPE "ppc-loop-preinc-prep" @@ -57,13 +80,49 @@ using namespace llvm; -// By default, we limit this to creating 16 PHIs (which is a little over half -// of the allocatable register set). -static cl::opt MaxVars("ppc-preinc-prep-max-vars", +// By default, we limit this to creating 16 common bases out of loops per +// function. 16 is a little over half of the allocatable register set. +static cl::opt MaxVarsPrep("ppc-formprep-max-vars", cl::Hidden, cl::init(16), - cl::desc("Potential PHI threshold for PPC preinc loop prep")); - -STATISTIC(PHINodeAlreadyExists, "PHI node already in pre-increment form"); + cl::desc("Potential common base number threshold per function for PPC loop " + "prep")); + +static cl::opt PreferUpdateForm("ppc-formprep-prefer-update", + cl::init(true), cl::Hidden, + cl::desc("prefer update form when ds form is also a update form")); + +// Sum of following 3 per loop thresholds for all loops can not be larger +// than MaxVarsPrep. +// By default, we limit this to creating 9 PHIs for one loop. +// 9 and 3 for each kind prep are exterimental values on Power9. +static cl::opt MaxVarsUpdateForm("ppc-preinc-prep-max-vars", + cl::Hidden, cl::init(3), + cl::desc("Potential PHI threshold per loop for PPC loop prep of update " + "form")); + +static cl::opt MaxVarsDSForm("ppc-dsprep-max-vars", + cl::Hidden, cl::init(3), + cl::desc("Potential PHI threshold per loop for PPC loop prep of DS form")); + +static cl::opt MaxVarsDQForm("ppc-dqprep-max-vars", + cl::Hidden, cl::init(3), + cl::desc("Potential PHI threshold per loop for PPC loop prep of DQ form")); + + +// If would not be profitable if the common base has only one load/store, ISEL +// should already be able to choose best load/store form based on offset for +// single load/store. Set minimal profitable value default to 2 and make it as +// an option. +static cl::opt DispFormPrepMinThreshold("ppc-dispprep-min-threshold", + cl::Hidden, cl::init(2), + cl::desc("Minimal common base load/store instructions triggering DS/DQ form " + "preparation")); + +STATISTIC(PHINodeAlreadyExistsUpdate, "PHI node already in pre-increment form"); +STATISTIC(PHINodeAlreadyExistsDS, "PHI node already in DS form"); +STATISTIC(PHINodeAlreadyExistsDQ, "PHI node already in DQ form"); +STATISTIC(DSFormChainRewritten, "Num of DS form chain rewritten"); +STATISTIC(DQFormChainRewritten, "Num of DQ form chain rewritten"); STATISTIC(UpdFormChainRewritten, "Num of update form chain rewritten"); namespace { @@ -83,6 +142,12 @@ SmallVector Elements; }; + // "UpdateForm" is not a real PPC instruction form, it stands for dform + // load/store with update like ldu/stdu, or Prefetch intrinsic. + // For DS form instructions, their displacements must be multiple of 4. + // For DQ form instructions, their displacements must be multiple of 16. + enum InstrForm { UpdateForm = 1, DSForm = 4, DQForm = 16 }; + class PPCLoopPreIncPrep : public FunctionPass { public: static char ID; // Pass ID, replacement for typeid @@ -112,12 +177,19 @@ ScalarEvolution *SE; bool PreserveLCSSA; + /// Successful preparation number for Update/DS/DQ form in all inner most + /// loops. One successful preparation will put one common base out of loop, + /// this may leads to register presure like LICM does. + /// Make sure total preparation number can be controlled by option. + unsigned SuccPrepCount; + bool runOnLoop(Loop *L); /// Check if required PHI node is already exist in Loop \p L. bool alreadyPrepared(Loop *L, Instruction* MemI, const SCEV *BasePtrStartSCEV, - const SCEVConstant *BasePtrIncSCEV); + const SCEVConstant *BasePtrIncSCEV, + InstrForm Form); /// Collect condition matched(\p isValidCandidate() returns true) /// candidates in Loop \p L. @@ -135,14 +207,32 @@ /// Prepare all candidates in \p Buckets for update form. bool updateFormPrep(Loop *L, SmallVector &Buckets); + /// Prepare all candidates in \p Buckets for displacement form, now for + /// ds/dq. + bool dispFormPrep(Loop *L, SmallVector &Buckets, + InstrForm Form); + /// Prepare for one chain \p BucketChain, find the best base element and /// update all other elements in \p BucketChain accordingly. + /// \p Form is used to find the best base element. + /// If success, best base element must be stored as the first element of + /// \p BucketChain. + /// Return false if no base element found, otherwise return true. + bool prepareBaseForDispFormChain(Bucket &BucketChain, + InstrForm Form); + + /// Prepare for one chain \p BucketChain, find the best base element and + /// update all other elements in \p BucketChain accordingly. + /// If success, best base element must be stored as the first element of + /// \p BucketChain. + /// Return false if no base element found, otherwise return true. bool prepareBaseForUpdateFormChain(Bucket &BucketChain); /// Rewrite load/store instructions in \p BucketChain according to /// preparation. bool rewriteLoadStores(Loop *L, Bucket &BucketChain, - SmallSet &BBChanged); + SmallSet &BBChanged, + InstrForm Form); }; } // end anonymous namespace @@ -204,6 +294,7 @@ DT = DTWP ? &DTWP->getDomTree() : nullptr; PreserveLCSSA = mustPreserveAnalysisID(LCSSAID); ST = TM ? TM->getSubtargetImpl(F) : nullptr; + SuccPrepCount = 0; bool MadeChange = false; @@ -278,7 +369,76 @@ return Buckets; } -// TODO: implement a more clever base choosing policy. +bool PPCLoopPreIncPrep::prepareBaseForDispFormChain(Bucket &BucketChain, + InstrForm Form) { + // RemainderOffsetInfo details: + // key: value of (Offset urem DispConstraint). For DSForm, it can + // be [0, 4). + // first of pair: the index of first BucketElement whose remainder is equal + // to key. For key 0, this value must be 0. + // second of pair: number of load/stores with the same remainder. + DenseMap> RemainderOffsetInfo; + + for (unsigned j = 0, je = BucketChain.Elements.size(); j != je; ++j) { + if (!BucketChain.Elements[j].Offset) + RemainderOffsetInfo[0] = std::make_pair(0, 1); + else { + unsigned Remainder = + BucketChain.Elements[j].Offset->getAPInt().urem(Form); + if (RemainderOffsetInfo.find(Remainder) == RemainderOffsetInfo.end()) + RemainderOffsetInfo[Remainder] = std::make_pair(j, 1); + else + RemainderOffsetInfo[Remainder].second++; + } + } + // Currently we choose the most profitable base as the one which has the max + // number of load/store with same remainder. + // FIXME: adjust the base selection strategy according to load/store offset + // distribution. + // For example, if we have one candidate chain for DS form preparation, which + // contains following load/stores with different remainders: + // 1: 10 load/store whose remainder is 1; + // 2: 9 load/store whose remainder is 2; + // 3: 1 for remainder 3 and 0 for remainder 0; + // Now we will choose the first load/store whose remainder is 1 as base and + // adjust all other load/stores according to new base, so we will get 10 DS + // form and 10 X form. + // But we should be more clever, for this case we could use two bases, one for + // remainder 1 and the other for remainder 2, thus we could get 19 DS form and 1 + // X form. + unsigned MaxCountRemainder = 0; + for (unsigned j = 0; j < Form; j++) + if ((RemainderOffsetInfo.find(j) != RemainderOffsetInfo.end()) && + RemainderOffsetInfo[j].second > + RemainderOffsetInfo[MaxCountRemainder].second) + MaxCountRemainder = j; + + // Abort when there are too few insts with common base. + if (RemainderOffsetInfo[MaxCountRemainder].second < DispFormPrepMinThreshold) + return false; + + // If the first value is most profitable, no needed to adjust BucketChain + // elements as they are substracted the first value when collecting. + if (MaxCountRemainder == 0) + return true; + + // Adjust load/store to the new chosen base. + const SCEV *Offset = + BucketChain.Elements[RemainderOffsetInfo[MaxCountRemainder].first].Offset; + BucketChain.BaseSCEV = SE->getAddExpr(BucketChain.BaseSCEV, Offset); + for (auto &E : BucketChain.Elements) { + if (E.Offset) + E.Offset = cast(SE->getMinusSCEV(E.Offset, Offset)); + else + E.Offset = cast(SE->getNegativeSCEV(Offset)); + } + + std::swap(BucketChain.Elements[RemainderOffsetInfo[MaxCountRemainder].first], + BucketChain.Elements[0]); + return true; +} + +// FIXME: implement a more clever base choosing policy. // Currently we always choose an exist load/store offset. This maybe lead to // suboptimal code sequences. For example, for one DS chain with offsets // {-32769, 2003, 2007, 2011}, we choose -32769 as base offset, and left disp @@ -324,8 +484,9 @@ return true; } -bool PPCLoopPreIncPrep::rewriteLoadStores( - Loop *L, Bucket &BucketChain, SmallSet &BBChanged) { +bool PPCLoopPreIncPrep::rewriteLoadStores(Loop *L, Bucket &BucketChain, + SmallSet &BBChanged, + InstrForm Form) { bool MadeChange = false; const SCEVAddRecExpr *BasePtrSCEV = cast(BucketChain.BaseSCEV); @@ -346,19 +507,30 @@ Type *I8PtrTy = Type::getInt8PtrTy(MemI->getParent()->getContext(), BasePtr->getType()->getPointerAddressSpace()); - const SCEV *BasePtrStartSCEV = BasePtrSCEV->getStart(); - if (!SE->isLoopInvariant(BasePtrStartSCEV, L)) + if (!SE->isLoopInvariant(BasePtrSCEV->getStart(), L)) return MadeChange; const SCEVConstant *BasePtrIncSCEV = dyn_cast(BasePtrSCEV->getStepRecurrence(*SE)); if (!BasePtrIncSCEV) return MadeChange; - BasePtrStartSCEV = SE->getMinusSCEV(BasePtrStartSCEV, BasePtrIncSCEV); + + // For some DS form load/store instructions, it can also be an update form, + // if the stride is a multipler of 4. Use update form if prefer it. + bool CanPreInc = (Form == UpdateForm || + ((Form == DSForm) && !BasePtrIncSCEV->getAPInt().urem(4) && + PreferUpdateForm)); + const SCEV *BasePtrStartSCEV = nullptr; + if (CanPreInc) + BasePtrStartSCEV = + SE->getMinusSCEV(BasePtrSCEV->getStart(), BasePtrIncSCEV); + else + BasePtrStartSCEV = BasePtrSCEV->getStart(); + if (!isSafeToExpand(BasePtrStartSCEV, *SE)) return MadeChange; - if (alreadyPrepared(L, MemI, BasePtrStartSCEV, BasePtrIncSCEV)) + if (alreadyPrepared(L, MemI, BasePtrStartSCEV, BasePtrIncSCEV, Form)) return MadeChange; LLVM_DEBUG(dbgs() << "PIP: New start is: " << *BasePtrStartSCEV << "\n"); @@ -385,24 +557,54 @@ NewPHI->addIncoming(BasePtrStart, LoopPredecessor); } - Instruction *InsPoint = &*Header->getFirstInsertionPt(); - GetElementPtrInst *PtrInc = GetElementPtrInst::Create( - I8Ty, NewPHI, BasePtrIncSCEV->getValue(), - getInstrName(MemI, GEPNodeIncNameSuffix), InsPoint); - PtrInc->setIsInBounds(IsPtrInBounds(BasePtr)); - for (const auto &PI : predecessors(Header)) { - if (PI == LoopPredecessor) - continue; + Instruction *PtrInc = nullptr; + Instruction *NewBasePtr = nullptr; + if (CanPreInc) { + Instruction *InsPoint = &*Header->getFirstInsertionPt(); + PtrInc = GetElementPtrInst::Create( + I8Ty, NewPHI, BasePtrIncSCEV->getValue(), + getInstrName(MemI, GEPNodeIncNameSuffix), InsPoint); + cast(PtrInc)->setIsInBounds(IsPtrInBounds(BasePtr)); + for (const auto &PI : predecessors(Header)) { + if (PI == LoopPredecessor) + continue; - NewPHI->addIncoming(PtrInc, PI); - } + NewPHI->addIncoming(PtrInc, PI); + } + if (PtrInc->getType() != BasePtr->getType()) + NewBasePtr = new BitCastInst( + PtrInc, BasePtr->getType(), + getInstrName(PtrInc, CastNodeNameSuffix), InsPoint); + else + NewBasePtr = PtrInc; + } else { + // Note that LoopPredecessor might occur in the predecessor list multiple + // times, and we need to make sure no more incoming value for them in PHI. + for (const auto &PI : predecessors(Header)) { + if (PI == LoopPredecessor) + continue; - Instruction *NewBasePtr; - if (PtrInc->getType() != BasePtr->getType()) - NewBasePtr = new BitCastInst(PtrInc, BasePtr->getType(), - getInstrName(PtrInc, CastNodeNameSuffix), InsPoint); - else - NewBasePtr = PtrInc; + // For the latch predecessor, we need to insert a GEP just before the + // terminator to increase the address. + BasicBlock *BB = PI; + Instruction *InsPoint = BB->getTerminator(); + PtrInc = GetElementPtrInst::Create( + I8Ty, NewPHI, BasePtrIncSCEV->getValue(), + getInstrName(MemI, GEPNodeIncNameSuffix), InsPoint); + + cast(PtrInc)->setIsInBounds(IsPtrInBounds(BasePtr)); + + NewPHI->addIncoming(PtrInc, PI); + } + PtrInc = NewPHI; + if (NewPHI->getType() != BasePtr->getType()) + NewBasePtr = + new BitCastInst(NewPHI, BasePtr->getType(), + getInstrName(NewPHI, CastNodeNameSuffix), + &*Header->getFirstInsertionPt()); + else + NewBasePtr = NewPHI; + } if (Instruction *IDel = dyn_cast(BasePtr)) BBChanged.insert(IDel->getParent()); @@ -461,7 +663,15 @@ } MadeChange = true; - UpdFormChainRewritten++; + + SuccPrepCount++; + + if (Form == DSForm && !CanPreInc) + DSFormChainRewritten++; + else if (Form == DQForm) + DQFormChainRewritten++; + else if (Form == UpdateForm || (Form == DSForm && CanPreInc)) + UpdFormChainRewritten++; return MadeChange; } @@ -476,7 +686,8 @@ // The base address of each bucket is transformed into a phi and the others // are rewritten based on new base. if (prepareBaseForUpdateFormChain(Bucket)) - MadeChange |= rewriteLoadStores(L, Bucket, BBChanged); + MadeChange |= rewriteLoadStores(L, Bucket, BBChanged, UpdateForm); + if (MadeChange) for (auto &BB : L->blocks()) if (BBChanged.count(BB)) @@ -484,13 +695,36 @@ return MadeChange; } -// In order to prepare for the pre-increment a PHI is added. +bool PPCLoopPreIncPrep::dispFormPrep(Loop *L, SmallVector &Buckets, + InstrForm Form) { + bool MadeChange = false; + + if (Buckets.empty()) + return MadeChange; + + SmallSet BBChanged; + for (auto &Bucket : Buckets) { + if (Bucket.Elements.size() < DispFormPrepMinThreshold) + continue; + if (prepareBaseForDispFormChain(Bucket, Form)) + MadeChange |= rewriteLoadStores(L, Bucket, BBChanged, Form); + } + + if (MadeChange) + for (auto &BB : L->blocks()) + if (BBChanged.count(BB)) + DeleteDeadPHIs(BB); + return MadeChange; +} + +// In order to prepare for the preferred instruction form, a PHI is added. // This function will check to see if that PHI already exists and will return -// true if it found an existing PHI with the same start and increment as the +// true if it found an existing PHI with the matched start and increment as the // one we wanted to create. bool PPCLoopPreIncPrep::alreadyPrepared(Loop *L, Instruction* MemI, const SCEV *BasePtrStartSCEV, - const SCEVConstant *BasePtrIncSCEV) { + const SCEVConstant *BasePtrIncSCEV, + InstrForm Form) { BasicBlock *BB = MemI->getParent(); if (!BB) return false; @@ -527,12 +761,25 @@ CurrentPHINode->getIncomingBlock(1) == PredBB) || (CurrentPHINode->getIncomingBlock(1) == LatchBB && CurrentPHINode->getIncomingBlock(0) == PredBB)) { - if (PHIBasePtrSCEV->getStart() == BasePtrStartSCEV && - PHIBasePtrIncSCEV == BasePtrIncSCEV) { + if (PHIBasePtrIncSCEV == BasePtrIncSCEV) { // The existing PHI (CurrentPHINode) has the same start and increment - // as the PHI that we wanted to create. - ++PHINodeAlreadyExists; - return true; + // as the PHI that we wanted to create. + if (Form == UpdateForm && + PHIBasePtrSCEV->getStart() == BasePtrStartSCEV) { + ++PHINodeAlreadyExistsUpdate; + return true; + } + if (Form == DSForm || Form == DQForm) { + const SCEVConstant *Diff = dyn_cast( + SE->getMinusSCEV(PHIBasePtrSCEV->getStart(), BasePtrStartSCEV)); + if (Diff && !Diff->getAPInt().urem(Form)) { + if (Form == DSForm) + ++PHINodeAlreadyExistsDS; + else + ++PHINodeAlreadyExistsDQ; + return true; + } + } } } } @@ -547,6 +794,10 @@ if (!L->empty()) return MadeChange; + // Return if already done enough preparation. + if (SuccPrepCount >= MaxVarsPrep) + return MadeChange; + LLVM_DEBUG(dbgs() << "PIP: Examining: " << *L << "\n"); BasicBlock *LoopPredecessor = L->getLoopPredecessor(); @@ -563,7 +814,6 @@ LLVM_DEBUG(dbgs() << "PIP fails since no predecessor for current loop.\n"); return MadeChange; } - // Check if a load/store has update form. This lambda is used by function // collectCandidates which can collect candidates for types defined by lambda. auto isUpdateFormCandidate = [&] (const Instruction *I, @@ -592,15 +842,49 @@ } return true; }; + + // Check if a load/store has DS form. + auto isDSFormCandidate = [] (const Instruction *I, const Value *PtrValue) { + assert((PtrValue && I) && "Invalid parameter!"); + // FIXME: 32 bit instruction lwa is also DS form. + return !isa(I) && + ((PtrValue->getType()->getPointerElementType()->isIntegerTy(64)) || + (PtrValue->getType()->getPointerElementType()->isFloatTy()) || + (PtrValue->getType()->getPointerElementType()->isDoubleTy())); + }; + + // Check if a load/store has DQ form. + auto isDQFormCandidate = [&] (const Instruction *I, const Value *PtrValue) { + assert((PtrValue && I) && "Invalid parameter!"); + return !isa(I) && ST && ST->hasP9Vector() && + (PtrValue->getType()->getPointerElementType()->isVectorTy()); + }; - // Collect buckets of comparable addresses used by loads, stores and prefetch // intrinsic for update form. SmallVector UpdateFormBuckets = - collectCandidates(L, isUpdateFormCandidate, MaxVars); + collectCandidates(L, isUpdateFormCandidate, MaxVarsUpdateForm); // Prepare for update form. if (!UpdateFormBuckets.empty()) MadeChange |= updateFormPrep(L, UpdateFormBuckets); + // Collect buckets of comparable addresses used by loads and stores for DS + // form. + SmallVector DSFormBuckets = + collectCandidates(L, isDSFormCandidate, MaxVarsDSForm); + + // Prepare for DS form. + if (!DSFormBuckets.empty()) + MadeChange |= dispFormPrep(L, DSFormBuckets, DSForm); + + // Collect buckets of comparable addresses used by loads and stores for DQ + // form. + SmallVector DQFormBuckets = + collectCandidates(L, isDQFormCandidate, MaxVarsDQForm); + + // Prepare for DQ form. + if (!DQFormBuckets.empty()) + MadeChange |= dispFormPrep(L, DQFormBuckets, DQForm); + return MadeChange; } diff --git a/llvm/test/CodeGen/PowerPC/loop-instr-form-prepare.ll b/llvm/test/CodeGen/PowerPC/loop-instr-form-prepare.ll --- a/llvm/test/CodeGen/PowerPC/loop-instr-form-prepare.ll +++ b/llvm/test/CodeGen/PowerPC/loop-instr-form-prepare.ll @@ -83,17 +83,17 @@ define i64 @test_ds_prep(i8* %0, i32 signext %1) { ; CHECK-LABEL: test_ds_prep: -; CHECK: addi r6, r3, 4001 +; CHECK: addi r6, r3, 4002 ; CHECK: .LBB1_2: # +; CHECK-NEXT: ldx r9, r6, r7 ; CHECK-NEXT: ld r10, 0(r6) +; CHECK-NEXT: mulld r9, r10, r9 ; CHECK-NEXT: ldx r11, r6, r5 -; CHECK-NEXT: mulld r10, r11, r10 -; CHECK-NEXT: ldx r12, r6, r7 -; CHECK-NEXT: mulld r10, r10, r12 -; CHECK-NEXT: addi r9, r6, 1 -; CHECK-NEXT: ldx r6, r6, r8 -; CHECK-NEXT: maddld r3, r10, r6, r3 -; CHECK-NEXT: mr r6, r9 +; CHECK-NEXT: mulld r9, r9, r11 +; CHECK-NEXT: addi r8, r6, 1 +; CHECK-NEXT: ld r6, 4(r6) +; CHECK-NEXT: maddld r3, r9, r6, r3 +; CHECK-NEXT: mr r6, r8 ; CHECK-NEXT: bdnz .LBB1_2 %3 = sext i32 %1 to i64 %4 = icmp eq i32 %1, 0 @@ -158,27 +158,27 @@ define i64 @test_max_number_reminder(i8* %0, i32 signext %1) { ; CHECK-LABEL: test_max_number_reminder: -; CHECK: addi r8, r3, 4001 +; CHECK: addi r9, r3, 4002 ; CHECK: .LBB2_2: # -; CHECK-NEXT: ld r30, 0(r8) -; CHECK-NEXT: ldx r29, r8, r5 -; CHECK-NEXT: mulld r30, r29, r30 -; CHECK-NEXT: addi r0, r8, 1 -; CHECK-NEXT: ld r28, 4(r8) -; CHECK-NEXT: ldx r27, r8, r7 -; CHECK-NEXT: ldx r26, r8, r9 -; CHECK-NEXT: ldx r25, r8, r10 -; CHECK-NEXT: ldx r24, r8, r11 -; CHECK-NEXT: ldx r23, r8, r12 -; CHECK-NEXT: ldx r8, r8, r6 -; CHECK-NEXT: mulld r8, r30, r8 -; CHECK-NEXT: mulld r8, r8, r28 -; CHECK-NEXT: mulld r8, r8, r27 -; CHECK-NEXT: mulld r8, r8, r26 -; CHECK-NEXT: mulld r8, r8, r25 -; CHECK-NEXT: mulld r8, r8, r24 -; CHECK-NEXT: maddld r3, r8, r23, r3 -; CHECK-NEXT: mr r8, r0 +; CHECK-NEXT: ldx r12, r9, r6 +; CHECK-NEXT: ld r0, 0(r9) +; CHECK-NEXT: mulld r12, r0, r12 +; CHECK-NEXT: addi r11, r9, 1 +; CHECK-NEXT: ldx r30, r9, r7 +; CHECK-NEXT: ld r29, 4(r9) +; CHECK-NEXT: ldx r28, r9, r8 +; CHECK-NEXT: ld r27, 12(r9) +; CHECK-NEXT: ld r26, 8(r9) +; CHECK-NEXT: ldx r25, r9, r10 +; CHECK-NEXT: ldx r9, r9, r5 +; CHECK-NEXT: mulld r9, r12, r9 +; CHECK-NEXT: mulld r9, r9, r30 +; CHECK-NEXT: mulld r9, r9, r29 +; CHECK-NEXT: mulld r9, r9, r28 +; CHECK-NEXT: mulld r9, r9, r27 +; CHECK-NEXT: mulld r9, r9, r26 +; CHECK-NEXT: maddld r3, r9, r25, r3 +; CHECK-NEXT: mr r9, r11 ; CHECK-NEXT: bdnz .LBB2_2 %3 = sext i32 %1 to i64 %4 = icmp eq i32 %1, 0 @@ -253,15 +253,15 @@ define dso_local i64 @test_update_ds_prep_interact(i8* %0, i32 signext %1) { ; CHECK-LABEL: test_update_ds_prep_interact: -; CHECK: addi r3, r3, 3997 +; CHECK: addi r3, r3, 3998 ; CHECK: .LBB3_2: # -; CHECK-NEXT: ldu r9, 4(r3) +; CHECK-NEXT: ldu r8, 4(r3) +; CHECK-NEXT: ldx r9, r3, r7 +; CHECK-NEXT: mulld r8, r8, r9 ; CHECK-NEXT: ldx r10, r3, r6 -; CHECK-NEXT: mulld r9, r10, r9 -; CHECK-NEXT: ldx r11, r3, r7 -; CHECK-NEXT: mulld r9, r9, r11 -; CHECK-NEXT: ldx r12, r3, r8 -; CHECK-NEXT: maddld r5, r9, r12, r5 +; CHECK-NEXT: mulld r8, r8, r10 +; CHECK-NEXT: ld r11, 4(r3) +; CHECK-NEXT: maddld r5, r8, r11, r5 ; CHECK-NEXT: bdnz .LBB3_2 %3 = sext i32 %1 to i64 %4 = icmp eq i32 %1, 0 @@ -317,15 +317,17 @@ define i64 @test_update_ds_prep_nointeract(i8* %0, i32 signext %1) { ; CHECK-LABEL: test_update_ds_prep_nointeract: -; CHECK: addi r3, r3, 4000 +; CHECK: addi r5, r3, 4000 +; CHECK: addi r3, r3, 4003 ; CHECK: .LBB4_2: # -; CHECK-NEXT: lbzu r9, 1(r3) -; CHECK-NEXT: ldx r10, r3, r6 -; CHECK-NEXT: mulld r9, r10, r9 -; CHECK-NEXT: ldx r11, r3, r7 -; CHECK-NEXT: mulld r9, r9, r11 -; CHECK-NEXT: ldx r12, r3, r8 -; CHECK-NEXT: maddld r5, r9, r12, r5 +; CHECK-NEXT: lbzu r8, 1(r5) +; CHECK-NEXT: ldx r9, r3, r7 +; CHECK-NEXT: ld r10, 0(r3) +; CHECK-NEXT: ld r11, 4(r3) +; CHECK-NEXT: addi r3, r3, 1 +; CHECK-NEXT: mulld r8, r9, r8 +; CHECK-NEXT: mulld r8, r8, r10 +; CHECK-NEXT: maddld r6, r8, r11, r6 ; CHECK-NEXT: bdnz .LBB4_2 %3 = sext i32 %1 to i64 %4 = icmp eq i32 %1, 0 @@ -384,26 +386,26 @@ define dso_local i64 @test_ds_multiple_chains(i8* %0, i8* %1, i32 signext %2) { ; CHECK-LABEL: test_ds_multiple_chains: -; CHECK: addi r3, r3, 4010 -; CHECK: addi r4, r4, 4010 +; CHECK: addi r3, r3, 4001 +; CHECK: addi r4, r4, 4001 ; CHECK: .LBB5_2: # -; CHECK-NEXT: ldx r10, r3, r7 -; CHECK-NEXT: ld r11, 0(r3) -; CHECK-NEXT: mulld r10, r11, r10 -; CHECK-NEXT: ldx r11, r3, r8 -; CHECK-NEXT: mulld r10, r10, r11 -; CHECK-NEXT: ldx r12, r3, r9 +; CHECK-NEXT: ld r8, 0(r3) +; CHECK-NEXT: ldx r9, r3, r7 +; CHECK-NEXT: mulld r8, r9, r8 +; CHECK-NEXT: ld r9, 4(r3) +; CHECK-NEXT: mulld r8, r8, r9 +; CHECK-NEXT: ld r10, 8(r3) ; CHECK-NEXT: addi r3, r3, 1 -; CHECK-NEXT: mulld r10, r10, r12 -; CHECK-NEXT: ldx r0, r4, r7 -; CHECK-NEXT: mulld r10, r10, r0 -; CHECK-NEXT: ld r30, 0(r4) -; CHECK-NEXT: mulld r10, r10, r30 -; CHECK-NEXT: ldx r29, r4, r8 -; CHECK-NEXT: mulld r10, r10, r29 -; CHECK-NEXT: ldx r28, r4, r9 +; CHECK-NEXT: mulld r8, r8, r10 +; CHECK-NEXT: ld r11, 0(r4) +; CHECK-NEXT: mulld r8, r8, r11 +; CHECK-NEXT: ldx r12, r4, r7 +; CHECK-NEXT: mulld r8, r8, r12 +; CHECK-NEXT: ld r0, 4(r4) +; CHECK-NEXT: mulld r8, r8, r0 +; CHECK-NEXT: ld r30, 8(r4) ; CHECK-NEXT: addi r4, r4, 1 -; CHECK-NEXT: maddld r6, r10, r28, r6 +; CHECK-NEXT: maddld r6, r8, r30, r6 ; CHECK-NEXT: bdnz .LBB5_2 %4 = sext i32 %2 to i64 %5 = icmp eq i32 %2, 0 @@ -491,28 +493,28 @@ define i64 @test_ds_cross_basic_blocks(i8* %0, i32 signext %1) { ; CHECK-LABEL: test_ds_cross_basic_blocks: -; CHECK: addi r5, r3, 4000 +; CHECK: addi r6, r3, 4009 ; CHECK: .LBB6_2: # -; CHECK-NEXT: ld r0, 0(r5) -; CHECK-NEXT: add r26, r0, r26 -; CHECK-NEXT: ldx r0, r5, r7 -; CHECK-NEXT: add r27, r0, r27 +; CHECK-NEXT: ldx r0, r6, r8 +; CHECK-NEXT: add r28, r0, r28 +; CHECK-NEXT: ld r0, -8(r6) +; CHECK-NEXT: add r29, r0, r29 ; CHECK-NEXT: .LBB6_3: # -; CHECK-NEXT: mulld r0, r27, r26 -; CHECK-NEXT: mulld r0, r0, r28 -; CHECK-NEXT: mulld r0, r0, r29 +; CHECK-NEXT: mulld r0, r29, r28 ; CHECK-NEXT: mulld r0, r0, r30 -; CHECK-NEXT: maddld r3, r0, r12, r3 -; CHECK-NEXT: addi r5, r5, 1 +; CHECK-NEXT: mulld r0, r0, r12 +; CHECK-NEXT: mulld r0, r0, r11 +; CHECK-NEXT: maddld r3, r0, r7, r3 +; CHECK-NEXT: addi r6, r6, 1 ; CHECK-NEXT: bdz .LBB6_9 ; CHECK-NEXT: .LBB6_4: # -; CHECK-NEXT: lbzu r0, 1(r6) -; CHECK-NEXT: clrldi r25, r0, 32 -; CHECK-NEXT: mulld r25, r25, r4 -; CHECK-NEXT: rldicl r25, r25, 31, 33 -; CHECK-NEXT: slwi r24, r25, 1 -; CHECK-NEXT: add r25, r25, r24 -; CHECK-NEXT: subf r0, r25, r0 +; CHECK-NEXT: lbzu r0, 1(r5) +; CHECK-NEXT: clrldi r27, r0, 32 +; CHECK-NEXT: mulld r27, r27, r4 +; CHECK-NEXT: rldicl r27, r27, 31, 33 +; CHECK-NEXT: slwi r26, r27, 1 +; CHECK-NEXT: add r27, r27, r26 +; CHECK-NEXT: subf r0, r27, r0 ; CHECK-NEXT: cmplwi r0, 1 ; CHECK-NEXT: beq cr0, .LBB6_2 ; CHECK-NEXT: # %bb.5: # @@ -520,17 +522,17 @@ ; CHECK-NEXT: cmplwi r0, 2 ; CHECK-NEXT: bne cr0, .LBB6_7 ; CHECK-NEXT: # %bb.6: # -; CHECK-NEXT: ldx r0, r5, r8 -; CHECK-NEXT: add r28, r0, r28 -; CHECK-NEXT: ldx r0, r5, r9 -; CHECK-NEXT: add r29, r0, r29 +; CHECK-NEXT: ldx r0, r6, r9 +; CHECK-NEXT: add r30, r0, r30 +; CHECK-NEXT: ld r0, -4(r6) +; CHECK-NEXT: add r12, r0, r12 ; CHECK-NEXT: b .LBB6_3 ; CHECK-NEXT: .p2align 4 ; CHECK-NEXT: .LBB6_7: # -; CHECK-NEXT: ldx r0, r5, r10 -; CHECK-NEXT: add r30, r0, r30 -; CHECK-NEXT: ldx r0, r5, r11 -; CHECK-NEXT: add r12, r0, r12 +; CHECK-NEXT: ldx r0, r6, r10 +; CHECK-NEXT: add r11, r0, r11 +; CHECK-NEXT: ld r0, 0(r6) +; CHECK-NEXT: add r7, r0, r7 %3 = sext i32 %1 to i64 %4 = icmp eq i32 %1, 0 br i1 %4, label %66, label %5 @@ -635,14 +637,15 @@ define float @test_ds_float(i8* %0, i32 signext %1) { ; CHECK-LABEL: test_ds_float: -; CHECK: addi r3, r3, 4000 +; CHECK: addi r3, r3, 4002 ; CHECK: .LBB7_2: # -; CHECK-NEXT: lfsu f0, 1(r3) -; CHECK-NEXT: lfsx f2, r3, r4 -; CHECK-NEXT: lfsx f3, r3, r5 +; CHECK-NEXT: lfsx f0, r3, r4 +; CHECK-NEXT: lfs f2, 0(r3) ; CHECK-NEXT: xsmulsp f0, f0, f2 -; CHECK-NEXT: lfsx f4, r3, r6 +; CHECK-NEXT: lfs f3, 20(r3) ; CHECK-NEXT: xsmulsp f0, f0, f3 +; CHECK-NEXT: lfs f4, 60(r3) +; CHECK-NEXT: addi r3, r3, 1 ; CHECK-NEXT: xsmulsp f0, f0, f4 ; CHECK-NEXT: xsaddsp f1, f1, f0 ; CHECK-NEXT: bdnz .LBB7_2 @@ -702,16 +705,16 @@ define float @test_ds_combine_float_int(i8* %0, i32 signext %1) { ; CHECK-LABEL: test_ds_combine_float_int: -; CHECK: addi r4, r3, 4001 -; CHECK: addi r3, r3, 4000 +; CHECK: addi r3, r3, 4002 ; CHECK: .LBB8_2: # -; CHECK-NEXT: lfdu f4, 1(r4) -; CHECK-NEXT: lfsu f0, 1(r3) +; CHECK-NEXT: lfd f4, 0(r3) +; CHECK-NEXT: lfsx f0, r3, r4 ; CHECK-NEXT: xscvuxdsp f4, f4 -; CHECK-NEXT: lfsx f2, r3, r5 -; CHECK-NEXT: lfsx f3, r3, r6 +; CHECK-NEXT: lfs f2, 20(r3) ; CHECK-NEXT: xsmulsp f0, f0, f4 ; CHECK-NEXT: xsmulsp f0, f2, f0 +; CHECK-NEXT: lfs f3, 60(r3) +; CHECK-NEXT: addi r3, r3, 1 ; CHECK-NEXT: xsmulsp f0, f3, f0 ; CHECK-NEXT: xsaddsp f1, f1, f0 ; CHECK-NEXT: bdnz .LBB8_2 diff --git a/llvm/test/CodeGen/PowerPC/swaps-le-1.ll b/llvm/test/CodeGen/PowerPC/swaps-le-1.ll --- a/llvm/test/CodeGen/PowerPC/swaps-le-1.ll +++ b/llvm/test/CodeGen/PowerPC/swaps-le-1.ll @@ -164,9 +164,9 @@ ; NOOPTSWAP: stxvd2x ; CHECK-P9-LABEL: @foo -; CHECK-P9-DAG: lxvx -; CHECK-P9-DAG: lxvx -; CHECK-P9-DAG: lxvx +; CHECK-P9-DAG: lxv +; CHECK-P9-DAG: lxv +; CHECK-P9-DAG: lxv ; CHECK-P9-DAG: lxv ; CHECK-P9-DAG: lxv ; CHECK-P9-DAG: lxv @@ -184,7 +184,7 @@ ; CHECK-P9-DAG: vmuluwm ; CHECK-P9-DAG: vmuluwm ; CHECK-P9-DAG: vmuluwm -; CHECK-P9-DAG: stxvx +; CHECK-P9-DAG: stxv ; CHECK-P9-DAG: stxv ; CHECK-P9-DAG: stxv ; CHECK-P9-DAG: stxv