[ModuloSchedule] Allow illegal phis to be moved across stages.

[hgreving]

Summary

Fixes a trivial but impactful bug where we did not move illegal phis across stages. This
led to incorrect mappings in certain cases.

llvm/lib/CodeGen/ModuloSchedule.cpp

 void PeelingModuloScheduleExpander::moveStageBetweenBlocks(
     MachineBasicBlock *DestBB, MachineBasicBlock *SourceBB, unsigned Stage) {
   auto InsertPt = DestBB->getFirstNonPHI();
   DenseMap<Register, Register> Remaps;
   for (auto I = SourceBB->getFirstNonPHI(); I != SourceBB->end();) {
     MachineInstr *MI = &*I++;
     if (MI->isPHI()) {
       // This is an illegal PHI. If we move any instructions using an illegal
-      // PHI, we need to create a legal Phi
+      // PHI, we need to create a legal Phi.
+      if (getStage(MI) != Stage) {
+        // The legal Phi is not necessary if the illegal phi's stage
+        // is being moved.
         Register PhiR = MI->getOperand(0).getReg();
         auto RC = MRI.getRegClass(PhiR);
         Register NR = MRI.createVirtualRegister(RC);
-      MachineInstr *NI = BuildMI(*DestBB, DestBB->getFirstNonPHI(), DebugLoc(),
-                                 TII->get(TargetOpcode::PHI), NR)
+        MachineInstr *NI = BuildMI(*DestBB, DestBB->getFirstNonPHI(),
+                                   DebugLoc(), TII->get(TargetOpcode::PHI), NR)
                                .addReg(PhiR)
                                .addMBB(SourceBB);
         BlockMIs[{DestBB, CanonicalMIs[MI]}] = NI;
         CanonicalMIs[NI] = CanonicalMIs[MI];
         Remaps[PhiR] = NR;
-      continue;
+      }
     }
     if (getStage(MI) != Stage)
       continue;
     MI->removeFromParent();
     DestBB->insert(InsertPt, MI);
     auto *KernelMI = CanonicalMIs[MI];
     BlockMIs[{DestBB, KernelMI}] = MI;
     BlockMIs.erase({SourceBB, KernelMI});