[PowerPC][AIX][RFC] Generate inlined quadword lock free atomic operations

[lkail]

Summary

Generate inlined quadword lock free atomic operations for PPC64 on AIX. It keeps compatible with current i128 implementation for PPC64. Need more input for if this patch is at a proper direction.

[jsji]

I would prefer we implementing it using AtomicExpand , not using AtomicExpansionKind::LLSC , but with AtomicExpansionKind::MaskedIntrinsic.

[lkail]

All related work moved to https://reviews.llvm.org/D103614

llvm/lib/Target/PowerPC/PPC.td

def FeatureP8Vector  : SubtargetFeature<"power8-vector", "HasP8Vector", "true",
                                         [FeatureVSX, FeatureP8Altivec]>;
 def FeatureDirectMove :
   SubtargetFeature<"direct-move", "HasDirectMove", "true",
                    "Enable Power8 direct move instructions",
                    [FeatureVSX]>;
 def FeaturePartwordAtomic : SubtargetFeature<"partword-atomics",
                                              "HasPartwordAtomics", "true",
                                              "Enable l[bh]arx and st[bh]cx.">;
+def FeatureQuadwordAtomic : SubtargetFeature<"quadword-atomics",
+                                             "HasQuadwordAtomics", "true",
+                                             "Enable lqarx and stqcx.">;
 def FeatureInvariantFunctionDescriptors :
   SubtargetFeature<"invariant-function-descriptors",
                    "HasInvariantFunctionDescriptors", "true",
                    "Assume function descriptors are invariant">;
 def FeatureLongCall : SubtargetFeature<"longcall", "UseLongCalls", "true",
                                        "Always use indirect calls">;
 def FeatureHTM : SubtargetFeature<"htm", "HasHTM", "true",
                                   "Enable Hardware Transactional Memory instructions">;
list<SubtargetFeature> P8AdditionalFeatures =
     [DirectivePwr8,
      FeatureP8Altivec,
      FeatureP8Vector,
      FeatureP8Crypto,
      FeatureHTM,
      FeatureDirectMove,
      FeatureICBT,
      FeaturePartwordAtomic,
+     FeatureQuadwordAtomic,
      FeaturePredictableSelectIsExpensive
     ];
 
   list<SubtargetFeature> P8SpecificFeatures = [FeatureAddiLoadFusion,
                                                FeatureAddisLoadFusion];
   list<SubtargetFeature> P8InheritableFeatures =
     !listconcat(P7InheritableFeatures, P8AdditionalFeatures);
   list<SubtargetFeature> P8Features =

llvm/lib/Target/PowerPC/PPCISelLowering.h

enum NodeType : unsigned {
     ST_VSR_SCAL_INT,
 
     /// ATOMIC_CMP_SWAP - the exact same as the target-independent nodes
     /// except they ensure that the compare input is zero-extended for
     /// sub-word versions because the atomic loads zero-extend.
     ATOMIC_CMP_SWAP_8,
     ATOMIC_CMP_SWAP_16,
 
+    ATOMIC_SWAP_128,
+
     /// GPRC = TOC_ENTRY GA, TOC
     /// Loads the entry for GA from the TOC, where the TOC base is given by
     /// the last operand.
     TOC_ENTRY
   };
 
   } // end namespace PPCISD
 
public:
     Instruction *emitLeadingFence(IRBuilder<> &Builder, Instruction *Inst,
                                   AtomicOrdering Ord) const override;
     Instruction *emitTrailingFence(IRBuilder<> &Builder, Instruction *Inst,
                                    AtomicOrdering Ord) const override;
 
     MachineBasicBlock *
     EmitInstrWithCustomInserter(MachineInstr &MI,
                                 MachineBasicBlock *MBB) const override;
+    MachineBasicBlock *EmitQuadwordAtomicBinary(MachineInstr &MI,
+                                                MachineBasicBlock *MBB,
+                                                unsigned BinOpcode,
+                                                unsigned CmpOpcode = 0,
+                                                unsigned CmpPred = 0) const;
     MachineBasicBlock *EmitAtomicBinary(MachineInstr &MI,
                                         MachineBasicBlock *MBB,
                                         unsigned AtomicSize,
                                         unsigned BinOpcode,
                                         unsigned CmpOpcode = 0,
                                         unsigned CmpPred = 0) const;
     MachineBasicBlock *EmitPartwordAtomicBinary(MachineInstr &MI,
                                                 MachineBasicBlock *MBB,
private:
     SDValue LowerFunnelShift(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerBSWAP(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerQuadwordAtomicOps(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerROTL(SDValue Op, SelectionDAG &DAG) const;
 
     SDValue LowerVectorLoad(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerVectorStore(SDValue Op, SelectionDAG &DAG) const;
 

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

 
   setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, isPPC64 ? Legal : Custom);
 
   if (!isPPC64) {
     setOperationAction(ISD::ATOMIC_LOAD,  MVT::i64, Expand);
     setOperationAction(ISD::ATOMIC_STORE, MVT::i64, Expand);
   }
 
+  if (isPPC64 && Subtarget.hasQuadwordAtomics() && Subtarget.isAIXABI()) {
+    setOperationAction(ISD::ATOMIC_SWAP, MVT::i128, Custom);
+  }
+
   setBooleanContents(ZeroOrOneBooleanContent);
 
   if (Subtarget.hasAltivec()) {
     // Altivec instructions set fields to all zeros or all ones.
     setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
   }
 
   if (!isPPC64) {
const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::XXPERMDI:        return "PPCISD::XXPERMDI";
   case PPCISD::VECSHL:          return "PPCISD::VECSHL";
   case PPCISD::CMPB:            return "PPCISD::CMPB";
   case PPCISD::Hi:              return "PPCISD::Hi";
   case PPCISD::Lo:              return "PPCISD::Lo";
   case PPCISD::TOC_ENTRY:       return "PPCISD::TOC_ENTRY";
   case PPCISD::ATOMIC_CMP_SWAP_8: return "PPCISD::ATOMIC_CMP_SWAP_8";
   case PPCISD::ATOMIC_CMP_SWAP_16: return "PPCISD::ATOMIC_CMP_SWAP_16";
+  case PPCISD::ATOMIC_SWAP_128: return "PPCISD::ATOMIC_SWAP_128";
   case PPCISD::DYNALLOC:        return "PPCISD::DYNALLOC";
   case PPCISD::DYNAREAOFFSET:   return "PPCISD::DYNAREAOFFSET";
   case PPCISD::PROBED_ALLOCA:   return "PPCISD::PROBED_ALLOCA";
   case PPCISD::GlobalBaseReg:   return "PPCISD::GlobalBaseReg";
   case PPCISD::SRL:             return "PPCISD::SRL";
   case PPCISD::SRA:             return "PPCISD::SRA";
   case PPCISD::SHL:             return "PPCISD::SHL";
   case PPCISD::SRA_ADDZE:       return "PPCISD::SRA_ADDZE";
   case PPCISD::CALL:            return "PPCISD::CALL";
   case PPCISD::CALL_NOP:        return "PPCISD::CALL_NOP";
SDValue PPCTargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op,
   Ops[2] = NewCmpOp;
   MachineMemOperand *MMO = AtomicNode->getMemOperand();
   SDVTList Tys = DAG.getVTList(MVT::i32, MVT::Other);
   auto NodeTy =
     (MemVT == MVT::i8) ? PPCISD::ATOMIC_CMP_SWAP_8 : PPCISD::ATOMIC_CMP_SWAP_16;
   return DAG.getMemIntrinsicNode(NodeTy, dl, Tys, Ops, MemVT, MMO);
 }
 
+SDValue PPCTargetLowering::LowerQuadwordAtomicOps(SDValue Op,
+                                                  SelectionDAG &DAG) const {
+  AtomicSDNode *N = cast<AtomicSDNode>(Op.getNode());
+  EVT MemVT = N->getMemoryVT();
+  MVT VT = MemVT.getSimpleVT();
+  assert(VT == MVT::i128 && "Expect quadword atomic operations");
+  SDLoc dl(N);
+  unsigned Opc = N->getOpcode();
+  switch (Opc) {
+  case ISD::ATOMIC_SWAP: {
+    SDValue Incr = N->getOperand(2);
+    SDValue RHSHi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i64, Incr,
+                                DAG.getIntPtrConstant(1, dl));
+    SDValue RHSLo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i64, Incr,
+                                DAG.getIntPtrConstant(0, dl));
+    SDVTList Tys = DAG.getVTList(MVT::i64, MVT::i64, MVT::Other);
+    SmallVector<SDValue, 4> Ops = {N->getOperand(0), N->getOperand(1), RHSHi,
+                                   RHSLo};
+    MachineMemOperand *MMO = N->getMemOperand();
+    SDValue Res = DAG.getMemIntrinsicNode(PPCISD::ATOMIC_SWAP_128, dl, Tys, Ops,
+                                          MemVT, MMO);
+    return Res;
+  }
+  default:
+    llvm_unreachable("Unhandle quadword operations");
+  }
+}
+
 SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
                                                  SelectionDAG &DAG) const {
   SDLoc dl(Op);
   // Create a stack slot that is 16-byte aligned.
   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
   int FrameIdx = MFI.CreateStackObject(16, Align(16), false);
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);

 
 void PPCTargetLowering::ReplaceNodeResults(SDNode *N,
                                            SmallVectorImpl<SDValue>&Results,
                                            SelectionDAG &DAG) const {
   SDLoc dl(N);
   switch (N->getOpcode()) {
   default:
     llvm_unreachable("Do not know how to custom type legalize this operation!");
+  case ISD::ATOMIC_SWAP: {
+    SDValue Res = LowerQuadwordAtomicOps(SDValue(N, 0), DAG);
+    if (Res) {
+      SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i128,
+                                 Res.getValue(1), Res);
+      Results.push_back(Pair);
+      Results.push_back(Res.getValue(2));
+    }
+    return;
+  }
   case ISD::READCYCLECOUNTER: {
     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
     SDValue RTB = DAG.getNode(PPCISD::READ_TIME_BASE, dl, VTs, N->getOperand(0));
 
     Results.push_back(
         DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, RTB, RTB.getValue(1)));
     Results.push_back(RTB.getValue(2));
     break;
if (isa<LoadInst>(Inst) && Subtarget.isPPC64())
               Intrinsic::ppc_cfence, {Inst->getType()}),
           {Inst});
     // FIXME: Can use isync for rmw operation.
     return callIntrinsic(Builder, Intrinsic::ppc_lwsync);
   }
   return nullptr;
 }
 
+MachineBasicBlock *PPCTargetLowering::EmitQuadwordAtomicBinary(
+    MachineInstr &MI, MachineBasicBlock *MBB, unsigned BinOpcode,
+    unsigned CmpOpcode, unsigned CmpPred) const {
+  assert(Subtarget.isPPC64() &&
+         "Quadword lock free atomic operations is not supported");
+  // This also handles ATOMIC_SWAP, indicated by BinOpcode==0.
+  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
+  auto LoadMnemonic = PPC::LQARX;
+  auto StoreMnemonic = PPC::STQCX;
+  DebugLoc DL = MI.getDebugLoc();
+  MachineFunction *MF = MBB->getParent();
+  MachineRegisterInfo &MRI = MF->getRegInfo();
+  const BasicBlock *BB = MBB->getBasicBlock();
+  MachineFunction::iterator It = ++MBB->getIterator();
+  MachineBasicBlock *LoopMBB = MF->CreateMachineBasicBlock(BB);
+  MachineBasicBlock *CmpFallThroughMBB =
+      CmpOpcode ? MF->CreateMachineBasicBlock(BB) : nullptr;
+  MachineBasicBlock *ExitMBB = MF->CreateMachineBasicBlock(BB);
+  MF->insert(It, LoopMBB);
+  if (CmpFallThroughMBB)
+    MF->insert(It, CmpFallThroughMBB);
+  MF->insert(It, ExitMBB);
+  ExitMBB->splice(ExitMBB->begin(), MBB, std::next(MI.getIterator()),
+                  MBB->end());
+  ExitMBB->transferSuccessorsAndUpdatePHIs(MBB);
+  // For min/max binary operations
+  // MBB:
+  //   ...
+  // LoopMBB:
+  //   lqarx in, ptr
+  //   cmpld in.sub_x0, op.sub_x0
+  //   %cmphi = crandc cr0.sub_lt, cr0.sub_eq
+  //   %hi_eq  = copy cr0.sub_eq
+  //   cmpld in.sub_x1, op.sub_x1
+  //   %cmplow = crand %hi_eq, cr0.sub_lt
+  //   %res = cror %cmphi, %cmplow
+  //   bc %res, ExitMBB
+  // CmpFallThroughMBB:
+  //   stqcx op, ptr
+  //   bne- LoopMBB
+  // ExitMBB:
+  //   ...
+
+  // Otherwise
+  // MBB:
+  //   ...
+  // LoopMBB:
+  //   lqarx in, ptr
+  //   addc out.sub_x1, in.sub_x1, op.sub_x1
+  //   adde out.sub_x0, in.sub_x0, op.sub_x0
+  //   stqcx out, ptr
+  //   bne- LoopMBB
+  // ExitMBB:
+  //   ...
+  MBB->addSuccessor(LoopMBB);
+  MachineBasicBlock *CurrentMBB = LoopMBB;
+  Register OutRegSub0 = MI.getOperand(0).getReg();
+  Register OutRegSub1 = MI.getOperand(1).getReg();
+  Register RAReg = MI.getOperand(2).getReg();
+  Register RBReg = MI.getOperand(3).getReg();
+  Register DeltaSub0 = MI.getOperand(4).getReg();
+  Register DeltaSub1 = MI.getOperand(5).getReg();
+  Register PairedLoadReg = MRI.createVirtualRegister(&PPC::G8pRCRegClass);
+  Register PairedStoreReg = MRI.createVirtualRegister(&PPC::G8pRCRegClass);
+
+  BuildMI(CurrentMBB, DL, TII->get(LoadMnemonic), PairedLoadReg)
+      .addReg(RAReg)
+      .addReg(RBReg);
+
+  if (BinOpcode == 0) {
+    // The return value of min/max is the original value that was stored in
+    // memory before comparison.
+    // See https://llvm.org/docs/LangRef.html#atomicrmw-instruction
+    BuildMI(CurrentMBB, DL, TII->get(TargetOpcode::COPY))
+        .addDef(OutRegSub0)
+        .addUse(PairedLoadReg, 0, PPC::sub_gp8_x0);
+    BuildMI(CurrentMBB, DL, TII->get(TargetOpcode::COPY))
+        .addDef(OutRegSub1)
+        .addUse(PairedLoadReg, 0, PPC::sub_gp8_x1);
+    if (CmpOpcode == PPC::CMPD) {
+      // Indicate signed comparison.
+      BuildMI(CurrentMBB, DL, TII->get(PPC::CMPLD))
+          .addUse(PairedLoadReg, 0, PPC::sub_gp8_x0)
+          .addUse(DeltaSub0);
+      Register HiEq = MRI.createVirtualRegister(&PPC::CRBITRCRegClass);
+      BuildMI(CurrentMBB, DL, TII->get(TargetOpcode::COPY), HiEq)
+          .addUse(PPC::CR0, 0, PPC::sub_eq);
+      BuildMI(CurrentMBB, DL, TII->get(PPC::CMPD))
+          .addUse(PairedLoadReg, 0, PPC::sub_gp8_x0)
+          .addUse(DeltaSub0);
+      Register CmpHi = MRI.createVirtualRegister(&PPC::CRBITRCRegClass);
+      BuildMI(CurrentMBB, DL, TII->get(PPC::CRANDC), CmpHi)
+          .addUse(PPC::CR0, 0, CmpPred)
+          .addUse(HiEq);
+      BuildMI(CurrentMBB, DL, TII->get(PPC::CMPLD))
+          .addUse(PairedLoadReg, 0, PPC::sub_gp8_x1)
+          .addUse(DeltaSub1);
+      Register CmpLow = MRI.createVirtualRegister(&PPC::CRBITRCRegClass);
+      BuildMI(CurrentMBB, DL, TII->get(PPC::CRAND), CmpLow)
+          .addUse(HiEq)
+          .addUse(PPC::CR0, 0, CmpPred);
+      Register Res = MRI.createVirtualRegister(&PPC::CRBITRCRegClass);
+      BuildMI(CurrentMBB, DL, TII->get(PPC::CROR), Res)
+          .addReg(CmpHi)
+          .addReg(CmpHi);
+      BuildMI(CurrentMBB, DL, TII->get(PPC::BC)).addReg(Res).addMBB(ExitMBB);
+    } else if (CmpOpcode == PPC::CMPLD) {
+      // Indicate unsigned comparison.
+      BuildMI(CurrentMBB, DL, TII->get(PPC::CMPLD))
+          .addUse(PairedLoadReg, 0, PPC::sub_gp8_x0)
+          .addUse(DeltaSub0);
+      Register HiEq = MRI.createVirtualRegister(&PPC::CRBITRCRegClass);
+      BuildMI(CurrentMBB, DL, TII->get(TargetOpcode::COPY), HiEq)
+          .addUse(PPC::CR0, 0, PPC::sub_eq);
+      Register CmpHi = MRI.createVirtualRegister(&PPC::CRBITRCRegClass);
+      BuildMI(CurrentMBB, DL, TII->get(PPC::CRANDC), CmpHi)
+          .addUse(PPC::CR0, 0, CmpPred)
+          .addUse(HiEq);
+      BuildMI(CurrentMBB, DL, TII->get(PPC::CMPLD))
+          .addUse(PairedLoadReg, 0, PPC::sub_gp8_x1)
+          .addUse(DeltaSub1);
+      Register CmpLow = MRI.createVirtualRegister(&PPC::CRBITRCRegClass);
+      BuildMI(CurrentMBB, DL, TII->get(PPC::CRAND), CmpLow)
+          .addUse(HiEq)
+          .addUse(PPC::CR0, 0, CmpPred);
+      Register Res = MRI.createVirtualRegister(&PPC::CRBITRCRegClass);
+      BuildMI(CurrentMBB, DL, TII->get(PPC::CROR), Res)
+        .addReg(CmpHi)
+        .addReg(CmpHi);
+      BuildMI(CurrentMBB, DL, TII->get(PPC::BC)).addReg(Res).addMBB(ExitMBB);
+    }
+    if (CmpOpcode) {
+      CurrentMBB->addSuccessor(ExitMBB);
+      CurrentMBB->addSuccessor(CmpFallThroughMBB);
+      CurrentMBB = CmpFallThroughMBB;
+    }
+    BuildMI(CurrentMBB, DL, TII->get(PPC::BUILD_QUADWORD), PairedStoreReg)
+        .addReg(DeltaSub0)
+        .addReg(DeltaSub1);
+  } else {
+    switch (BinOpcode) {
+    case PPC::ADD8:
+      BuildMI(CurrentMBB, DL, TII->get(PPC::ADDC8))
+          .addDef(OutRegSub1, 0, PPC::sub_gp8_x1)
+          .addUse(DeltaSub1)
+          .addUse(PairedLoadReg, 0, PPC::sub_gp8_x1);
+      BuildMI(CurrentMBB, DL, TII->get(PPC::ADDE8))
+          .addDef(OutRegSub0, 0, PPC::sub_gp8_x0)
+          .addUse(DeltaSub0)
+          .addUse(PairedLoadReg, 0, PPC::sub_gp8_x0);
+      break;
+    case PPC::SUBF8:
+      BuildMI(CurrentMBB, DL, TII->get(PPC::SUBFC8))
+          .addDef(OutRegSub1, 0, PPC::sub_gp8_x1)
+          .addUse(DeltaSub1)
+          .addUse(PairedLoadReg, 0, PPC::sub_gp8_x1);
+      BuildMI(CurrentMBB, DL, TII->get(PPC::SUBFE8))
+          .addDef(OutRegSub0, 0, PPC::sub_gp8_x0)
+          .addUse(DeltaSub0)
+          .addUse(PairedLoadReg, 0, PPC::sub_gp8_x0);
+      break;
+    case PPC::AND8:
+    case PPC::XOR8:
+    case PPC::OR8:
+      BuildMI(CurrentMBB, DL, TII->get(BinOpcode))
+          .addDef(OutRegSub0, 0, PPC::sub_gp8_x0)
+          .addUse(DeltaSub0)
+          .addUse(PairedLoadReg, 0, PPC::sub_gp8_x0);
+      BuildMI(CurrentMBB, DL, TII->get(BinOpcode))
+          .addDef(OutRegSub1, 0, PPC::sub_gp8_x1)
+          .addUse(DeltaSub1)
+          .addUse(PairedLoadReg, 0, PPC::sub_gp8_x1);
+      break;
+    default:
+      llvm_unreachable("Unexpected quadword atomic binary opcode");
+    }
+    BuildMI(CurrentMBB, DL, TII->get(PPC::BUILD_QUADWORD), PairedStoreReg)
+        .addReg(OutRegSub0)
+        .addReg(OutRegSub1);
+  }
+  BuildMI(CurrentMBB, DL, TII->get(StoreMnemonic))
+      .addReg(PairedStoreReg)
+      .addReg(RAReg)
+      .addReg(RBReg);
+  BuildMI(CurrentMBB, DL, TII->get(PPC::BCC))
+      .addImm(PPC::PRED_NE)
+      .addReg(PPC::CR0)
+      .addMBB(LoopMBB);
+  CurrentMBB->addSuccessor(LoopMBB);
+  CurrentMBB->addSuccessor(ExitMBB);
+  return ExitMBB;
+}
+
 MachineBasicBlock *
 PPCTargetLowering::EmitAtomicBinary(MachineInstr &MI, MachineBasicBlock *BB,
                                     unsigned AtomicSize,
                                     unsigned BinOpcode,
                                     unsigned CmpOpcode,
                                     unsigned CmpPred) const {
   // This also handles ATOMIC_SWAP, indicated by BinOpcode==0.
   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
else if (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I32 ||
     BB->addSuccessor(exitMBB);
 
     //  exitMBB:
     //   ...
     BB = exitMBB;
     BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest)
         .addReg(TmpReg)
         .addReg(ShiftReg);
-  } else if (MI.getOpcode() == PPC::FADDrtz) {
+  } else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I128) {
+    BB = EmitQuadwordAtomicBinary(MI, BB, 0, 0, 0);
+  }else if (MI.getOpcode() == PPC::FADDrtz) {
     // This pseudo performs an FADD with rounding mode temporarily forced
     // to round-to-zero.  We emit this via custom inserter since the FPSCR
     // is not modeled at the SelectionDAG level.
     Register Dest = MI.getOperand(0).getReg();
     Register Src1 = MI.getOperand(1).getReg();
     Register Src2 = MI.getOperand(2).getReg();
     DebugLoc dl = MI.getDebugLoc();
 
     MachineRegisterInfo &RegInfo = F->getRegInfo();
     Register MFFSReg = RegInfo.createVirtualRegister(&PPC::F8RCRegClass);

llvm/lib/Target/PowerPC/PPCInstr64Bit.td

let Defs = [CR0] in {
 
   def ATOMIC_CMP_SWAP_I64 : PPCCustomInserterPseudo<
     (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$old, g8rc:$new), "#ATOMIC_CMP_SWAP_I64",
     [(set i64:$dst, (atomic_cmp_swap_64 ForceXForm:$ptr, i64:$old, i64:$new))]>;
 
   def ATOMIC_SWAP_I64 : PPCCustomInserterPseudo<
     (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$new), "#ATOMIC_SWAP_I64",
     [(set i64:$dst, (atomic_swap_64 ForceXForm:$ptr, i64:$new))]>;
+
+  def ATOMIC_SWAP_I128 : PPCCustomInserterPseudo<
+    (outs g8rc:$res_hi, g8rc:$res_lo),
+    (ins memrr:$ptr, g8rc:$incr_hi, g8rc:$incr_lo),
+    "#ATOMIC_SWAP_I128",
+    [(set i64:$res_hi, i64:$res_lo,
+      (PPCatomicSwap_128 ForceXForm:$ptr, i64:$incr_hi, i64:$incr_lo))]>;
 }
 
 // Instructions to support atomic operations
 let mayLoad = 1, hasSideEffects = 0 in {
 def LDARX : XForm_1_memOp<31,  84, (outs g8rc:$rD), (ins memrr:$ptr),
                           "ldarx $rD, $ptr", IIC_LdStLDARX, []>;
 // TODO: Add scheduling info.
 let hasNoSchedulingInfo = 1 in
 def LQARX : XForm_1_memOp<31, 276, (outs g8prc:$RTp), (ins memrr:$ptr),
-                          "lqarx $RTp, $ptr", IIC_LdStLQARX, []>;
+                          "lqarx $RTp, $ptr", IIC_LdStLQARX, []>,
+                          Requires<[HasQuadwordAtomics]>;
 
 // Instruction to support lock versions of atomics
 // (EH=1 - see Power ISA 2.07 Book II 4.4.2)
 def LDARXL : XForm_1<31,  84, (outs g8rc:$rD), (ins memrr:$ptr),
                      "ldarx $rD, $ptr, 1", IIC_LdStLDARX, []>, isRecordForm;
 // TODO: Add scheduling info.
 let hasNoSchedulingInfo = 1 in
 def LQARXL : XForm_1<31, 276, (outs g8prc:$RTp), (ins memrr:$ptr),
                      "lqarx $RTp, $ptr, 1", IIC_LdStLQARX, []>, isRecordForm;
 
 let hasExtraDefRegAllocReq = 1 in
 def LDAT : X_RD5_RS5_IM5<31, 614, (outs g8rc:$rD), (ins g8rc:$rA, u5imm:$FC),
                          "ldat $rD, $rA, $FC", IIC_LdStLoad>, isPPC64,
            Requires<[IsISA3_0]>;
 }
 
 let Defs = [CR0], mayStore = 1, mayLoad = 0, hasSideEffects = 0 in {
 def STDCX : XForm_1_memOp<31, 214, (outs), (ins g8rc:$rS, memrr:$dst),
                           "stdcx. $rS, $dst", IIC_LdStSTDCX, []>, isRecordForm;
 // TODO: Add scheduling info.
 let hasNoSchedulingInfo = 1 in
 def STQCX : XForm_1_memOp<31, 182, (outs), (ins g8prc:$RSp, memrr:$dst),
-                          "stqcx. $RSp, $dst", IIC_LdStSTQCX, []>, isRecordForm;
+                          "stqcx. $RSp, $dst", IIC_LdStSTQCX, []>, isRecordForm,
+                          Requires<[HasQuadwordAtomics]>;
 }
 
 let mayStore = 1, mayLoad = 0, hasSideEffects = 0 in
 def STDAT : X_RD5_RS5_IM5<31, 742, (outs), (ins g8rc:$rS, g8rc:$rA, u5imm:$FC),
                           "stdat $rS, $rA, $FC", IIC_LdStStore>, isPPC64,
             Requires<[IsISA3_0]>;
 
 let Interpretation64Bit = 1, isCodeGenOnly = 1 in {
def LDUX : XForm_1_memOp<31, 53, (outs g8rc:$rD, ptr_rc_nor0:$ea_result),
                         []>, RegConstraint<"$addr.ptrreg = $ea_result">,
                         NoEncode<"$ea_result">, isPPC64;
 
 def LDMX : XForm_1<31, 309, (outs g8rc:$rD), (ins memrr:$src),
                    "ldmx $rD, $src", IIC_LdStLD, []>, isPPC64,
                    Requires<[IsISA3_0]>;
 }
 
+def BUILD_QUADWORD : PPCPostRAExpPseudo<(outs g8prc:$RTp),
+                                        (ins g8rc:$src0, g8rc:$src1),
+                                        "#BUILD_QUADWORD", []>,
+                                        isPPC64;
+
 let mayLoad = 1, hasNoSchedulingInfo = 1 in {
 // Full 16-byte load.
 // Early clobber $RTp to avoid assigned to the same register as RA.
 // TODO: Add scheduling info.
 def LQ   : DQForm_RTp5_RA17_MEM<56, 0,
                                 (outs g8prc:$RTp),
                                 (ins memrix16:$src),
                                 "lq $RTp, $src", IIC_LdStLQ,

llvm/lib/Target/PowerPC/PPCInstrInfo.cpp

case PPC::CFENCE8: {
     BuildMI(MBB, MI, DL, get(PPC::CTRL_DEP))
         .addImm(PPC::PRED_NE_MINUS)
         .addReg(PPC::CR7)
         .addImm(1);
     MI.setDesc(get(PPC::ISYNC));
     MI.RemoveOperand(0);
     return true;
   }
+  case PPC::BUILD_QUADWORD: {
+    unsigned DestIdx = MI.getOperand(0).getReg() - PPC::G8p0;
+    Register DestX0 = PPC::X0 + 2 * DestIdx;
+    Register DestX1 = DestX0 + 1;
+    // We have to check we don't early clobber Src registers.
+    Register Src0 = MI.getOperand(1).getReg();
+    Register Src1 = MI.getOperand(2).getReg();
+    // The most tricky case, swapping values.
+    if (DestX0 == Src1 && DestX1 == Src0) {
+      BuildMI(MBB, MI, DL, get(PPC::XOR8), DestX0)
+          .addReg(DestX0)
+          .addReg(DestX1);
+      BuildMI(MBB, MI, DL, get(PPC::XOR8), DestX1)
+          .addReg(DestX0)
+          .addReg(DestX1);
+      BuildMI(MBB, MI, DL, get(PPC::XOR8), DestX0)
+          .addReg(DestX0)
+          .addReg(DestX1);
+    } else if (DestX0 != Src0 || DestX1 != Src1) {
+      if (DestX0 == Src1 || DestX1 != Src0) {
+        BuildMI(MBB, MI, DL, get(PPC::OR8), DestX1).addReg(Src1).addReg(Src1);
+        BuildMI(MBB, MI, DL, get(PPC::OR8), DestX0).addReg(Src0).addReg(Src0);
+      } else {
+        BuildMI(MBB, MI, DL, get(PPC::OR8), DestX0).addReg(Src0).addReg(Src0);
+        BuildMI(MBB, MI, DL, get(PPC::OR8), DestX1).addReg(Src1).addReg(Src1);
+      }
+    }
+    MI.setDesc(get(PPC::UNENCODED_NOP));
+    MI.RemoveOperand(2);
+    MI.RemoveOperand(1);
+    MI.RemoveOperand(0);
+    return true;
+  }
   }
   return false;
 }
 
 // Essentially a compile-time implementation of a compare->isel sequence.
 // It takes two constants to compare, along with the true/false registers
 // and the comparison type (as a subreg to a CR field) and returns one
 // of the true/false registers, depending on the comparison results.

llvm/lib/Target/PowerPC/PPCInstrInfo.td

 def SDT_PPCextswsli : SDTypeProfile<1, 2, [  // extswsli
   SDTCisInt<0>, SDTCisInt<1>, SDTCisOpSmallerThanOp<1, 0>, SDTCisInt<2>
 ]>;
 
 def SDT_PPCFPMinMax : SDTypeProfile<1, 2, [
   SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisFP<0>
 ]>;
 
+def SDT_PPCI128Atomic2 : SDTypeProfile<2, 3, [
+  SDTCisInt<0>, SDTCisSameAs<0, 1>, SDTCisPtrTy<2>, SDTCisSameAs<0, 3>,
+  SDTCisSameAs<3, 4>
+]>;
+
+def SDT_PPCI128Atomic3 : SDTypeProfile<2, 5, [
+  SDTCisSameAs<0, 1>, SDTCisPtrTy<2>, SDTCisSameAs<3, 4>, SDTCisSameAs<5, 6>
+]>;
+
 //===----------------------------------------------------------------------===//
 // PowerPC specific DAG Nodes.
 //
 
 def PPCfre    : SDNode<"PPCISD::FRE",     SDTFPUnaryOp, []>;
 def PPCfrsqrte: SDNode<"PPCISD::FRSQRTE", SDTFPUnaryOp, []>;
 def PPCfsqrt  : SDNode<"PPCISD::FSQRT",   SDTFPUnaryOp, []>;
 def PPCftsqrt : SDNode<"PPCISD::FTSQRT",  SDT_PPCFtsqrt,[]>;
SDNode<"PPCISD::ATOMIC_CMP_SWAP_8", SDTAtomic3,
          [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
 def PPCatomicCmpSwap_16 :
   SDNode<"PPCISD::ATOMIC_CMP_SWAP_16", SDTAtomic3,
          [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
 def PPClbrx       : SDNode<"PPCISD::LBRX", SDT_PPClbrx,
                            [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
 def PPCstbrx      : SDNode<"PPCISD::STBRX", SDT_PPCstbrx,
                            [SDNPHasChain, SDNPMayStore]>;
+def PPCatomicSwap_128 :
+  SDNode<"PPCISD::ATOMIC_SWAP_128", SDT_PPCI128Atomic2,
+         [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
 
 // Instructions to set/unset CR bit 6 for SVR4 vararg calls
 def PPCcr6set   : SDNode<"PPCISD::CR6SET", SDTNone,
                          [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
 def PPCcr6unset : SDNode<"PPCISD::CR6UNSET", SDTNone,
                          [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
 
 // Instructions to support dynamic alloca.

 def HasOnlyMSYNC : Predicate<"Subtarget->hasOnlyMSYNC()">;
 def HasSYNC   : Predicate<"!Subtarget->hasOnlyMSYNC()">;
 def IsPPC4xx  : Predicate<"Subtarget->isPPC4xx()">;
 def IsPPC6xx  : Predicate<"Subtarget->isPPC6xx()">;
 def IsE500  : Predicate<"Subtarget->isE500()">;
 def HasSPE  : Predicate<"Subtarget->hasSPE()">;
 def HasICBT : Predicate<"Subtarget->hasICBT()">;
 def HasPartwordAtomics : Predicate<"Subtarget->hasPartwordAtomics()">;
+def HasQuadwordAtomics : Predicate<"Subtarget->hasQuadwordAtomics()">;
 def NoNaNsFPMath
     : Predicate<"Subtarget->getTargetMachine().Options.NoNaNsFPMath">;
 def NaNsFPMath
     : Predicate<"!Subtarget->getTargetMachine().Options.NoNaNsFPMath">;
 def HasBPERMD : Predicate<"Subtarget->hasBPERMD()">;
 def HasExtDiv : Predicate<"Subtarget->hasExtDiv()">;
 def IsISA3_0 : Predicate<"Subtarget->isISA3_0()">;
 def HasFPU : Predicate<"Subtarget->hasFPU()">;

llvm/lib/Target/PowerPC/PPCSubtarget.h

protected:
   bool IsPPC6xx;
   bool FeatureMFTB;
   bool AllowsUnalignedFPAccess;
   bool DeprecatedDST;
   bool IsLittleEndian;
   bool HasICBT;
   bool HasInvariantFunctionDescriptors;
   bool HasPartwordAtomics;
+  bool HasQuadwordAtomics;
   bool HasDirectMove;
   bool HasHTM;
   bool HasFloat128;
   bool HasFusion;
   bool HasStoreFusion;
   bool HasAddiLoadFusion;
   bool HasAddisLoadFusion;
   bool IsISA3_0;
public:
   bool isDeprecatedDST() const { return DeprecatedDST; }
   bool hasICBT() const { return HasICBT; }
   bool hasInvariantFunctionDescriptors() const {
     return HasInvariantFunctionDescriptors;
   }
   bool usePPCPreRASchedStrategy() const { return UsePPCPreRASchedStrategy; }
   bool usePPCPostRASchedStrategy() const { return UsePPCPostRASchedStrategy; }
   bool hasPartwordAtomics() const { return HasPartwordAtomics; }
+  bool hasQuadwordAtomics() const { return HasQuadwordAtomics; }
   bool hasDirectMove() const { return HasDirectMove; }
 
   Align getPlatformStackAlignment() const {
     return Align(16);
   }
 
   unsigned  getRedZoneSize() const {
     if (isPPC64())

llvm/lib/Target/PowerPC/PPCSubtarget.cpp

void PPCSubtarget::initializeEnvironment() {
   IsPPC6xx = false;
   IsE500 = false;
   FeatureMFTB = false;
   AllowsUnalignedFPAccess = false;
   DeprecatedDST = false;
   HasICBT = false;
   HasInvariantFunctionDescriptors = false;
   HasPartwordAtomics = false;
+  HasQuadwordAtomics = false;
   HasDirectMove = false;
   HasHTM = false;
   HasFloat128 = false;
   HasFusion = false;
   HasStoreFusion = false;
   HasAddiLoadFusion = false;
   HasAddisLoadFusion = false;
   IsISA3_0 = false;

llvm/test/CodeGen/PowerPC/atomics-i128.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-ibm-aix-xcoff -mcpu=pwr8 \
+; RUN:   < %s | FileCheck %s
+
+define i128 @swap(i128* %a, i128 %x) {
+; CHECK-LABEL: swap:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sync
+; CHECK-NEXT:  L..BB0_1: # %entry
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lqarx 6, 0, 3
+; CHECK-NEXT:    mr 9, 5
+; CHECK-NEXT:    mr 8, 4
+; CHECK-NEXT:    stqcx. 8, 0, 3
+; CHECK-NEXT:    bne 0, L..BB0_1
+; CHECK-NEXT:  # %bb.2: # %entry
+; CHECK-NEXT:    mr 3, 6
+; CHECK-NEXT:    mr 4, 7
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    blr
+entry:
+  %0 = atomicrmw xchg i128* %a, i128 %x seq_cst, align 16
+  ret i128 %0
+}