[PowerPC] Implement quadword atomic load/store

[lkail]

Summary

Add support to load/store i128 atomically.

[lkail]

ping.

[jsji]

LGTM with some nits. Thanks!

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

10463

nit: multiline after if should be within {/}?

10543

Add comments to all the following code please.

12741

Looks like we can common up most of these code for LQX_PSEUDO/STQX_PSEUDO? Just need additional check for last BuildMI?

17390–17391

So we will set Flag to PPC::MOF_DoubleWordInt for 128. Although we haven't used the flags for now, I think it would be better if we add one more flag for 128?

llvm/lib/Target/PowerPC/PPCInstrInfo.cpp

3107

nit: indent is off?

[lkail]

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

17390–17391

Good catch. I'll add TODO at PPCTargetLowering::initializeAddrModeMap and address it with following patches.

llvm/lib/Target/PowerPC/PPCInstrInfo.cpp

3107

Already run clang-format on it, it should have matched clang-format's behavior.

llvm/include/llvm/IR/IntrinsicsPowerPC.td

let TargetPrefix = "ppc" in {
   def int_ppc_atomicrmw_or_i128   : AtomicRMW128Intrinsic;
   def int_ppc_atomicrmw_xor_i128  : AtomicRMW128Intrinsic;
   def int_ppc_atomicrmw_nand_i128 : AtomicRMW128Intrinsic;
   def int_ppc_cmpxchg_i128 : Intrinsic<[llvm_i64_ty, llvm_i64_ty],
                                        [llvm_ptr_ty,
                                         llvm_i64_ty, llvm_i64_ty,
                                         llvm_i64_ty, llvm_i64_ty],
                                        [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
+  def int_ppc_atomic_load_i128 :
+    Intrinsic<[llvm_i64_ty, llvm_i64_ty],
+              [llvm_ptr_ty],
+              [IntrArgMemOnly, IntrReadMem, NoCapture<ArgIndex<0>>]>;
+  def int_ppc_atomic_store_i128 :
+    Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty],
+              [IntrArgMemOnly, IntrWriteMem, NoCapture<ArgIndex<2>>]>;
 }

llvm/lib/Target/PowerPC/PPCExpandAtomicPseudoInsts.cpp

bool PPCExpandAtomicPseudo::expandMI(MachineBasicBlock &MBB, MachineInstr &MI,
   case PPC::ATOMIC_LOAD_SUB_I128:
   case PPC::ATOMIC_LOAD_XOR_I128:
   case PPC::ATOMIC_LOAD_NAND_I128:
   case PPC::ATOMIC_LOAD_AND_I128:
   case PPC::ATOMIC_LOAD_OR_I128:
     return expandAtomicRMW128(MBB, MI, NMBBI);
   case PPC::ATOMIC_CMP_SWAP_I128:
     return expandAtomicCmpSwap128(MBB, MI, NMBBI);
+  case PPC::BUILD_QUADWORD: {
+    Register Dst = MI.getOperand(0).getReg();
+    Register DstHi = TRI->getSubReg(Dst, PPC::sub_gp8_x0);
+    Register DstLo = TRI->getSubReg(Dst, PPC::sub_gp8_x1);
+    Register Lo = MI.getOperand(1).getReg();
+    Register Hi = MI.getOperand(2).getReg();
+    PairedCopy(TII, MBB, MI, MI.getDebugLoc(), DstHi, DstLo, Hi, Lo);
+    MI.eraseFromParent();
+    return true;
+  }
   default:
     return false;
   }
 }
 
 bool PPCExpandAtomicPseudo::expandAtomicRMW128(
     MachineBasicBlock &MBB, MachineInstr &MI,
     MachineBasicBlock::iterator &NMBBI) {

llvm/lib/Target/PowerPC/PPCISelLowering.h

private:
     SDValue LowerFunnelShift(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerBSWAP(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerATOMIC_LOAD_STORE(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerROTL(SDValue Op, SelectionDAG &DAG) const;
 
     SDValue LowerVectorLoad(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerVectorStore(SDValue Op, SelectionDAG &DAG) const;

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

 
   setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, isPPC64 ? Legal : Custom);
 
   if (!isPPC64) {
     setOperationAction(ISD::ATOMIC_LOAD,  MVT::i64, Expand);
     setOperationAction(ISD::ATOMIC_STORE, MVT::i64, Expand);
   }
 
-  if (EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics())
+  if (EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics()) {
     setMaxAtomicSizeInBitsSupported(128);
+    setOperationAction(ISD::ATOMIC_LOAD, MVT::i128, Custom);
+    setOperationAction(ISD::ATOMIC_STORE, MVT::i128, Custom);
+    setOperationAction(ISD::INTRINSIC_VOID, MVT::i128, Custom);
+  }
 
   setBooleanContents(ZeroOrOneBooleanContent);
 
   if (Subtarget.hasAltivec()) {
     // Altivec instructions set fields to all zeros or all ones.
     setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
   }
 
AddrModesMap[PPC::AM_DQForm] = {
       // LXV, STXV
       PPC::MOF_RPlusSImm16Mult16 | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
       PPC::MOF_NotAddNorCst | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
       PPC::MOF_AddrIsSImm32 | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
       PPC::MOF_RPlusSImm16Mult16 | PPC::MOF_Vector256 | PPC::MOF_SubtargetP10,
       PPC::MOF_NotAddNorCst | PPC::MOF_Vector256 | PPC::MOF_SubtargetP10,
       PPC::MOF_AddrIsSImm32 | PPC::MOF_Vector256 | PPC::MOF_SubtargetP10,
   };
+  // TODO: Add mapping for quadword load/store.
 }
 
 /// getMaxByValAlign - Helper for getByValTypeAlignment to determine
 /// the desired ByVal argument alignment.
 static void getMaxByValAlign(Type *Ty, Align &MaxAlign, Align MaxMaxAlign) {
   if (MaxAlign == MaxMaxAlign)
     return;
   if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
SDValue PPCTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
   // SelectionDAGBuilder::visitTargetIntrinsic may insert one extra chain to
   // the beginning of the argument list.
   int ArgStart = isa<ConstantSDNode>(Op.getOperand(0)) ? 0 : 1;
   SDLoc DL(Op);
   switch (cast<ConstantSDNode>(Op.getOperand(ArgStart))->getZExtValue()) {
   case Intrinsic::ppc_cfence: {
     assert(ArgStart == 1 && "llvm.ppc.cfence must carry a chain argument.");
     assert(Subtarget.isPPC64() && "Only 64-bit is supported for now.");
-    return SDValue(DAG.getMachineNode(PPC::CFENCE8, DL, MVT::Other,
-                                      DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64,
-                                                  Op.getOperand(ArgStart + 1)),
-                                      Op.getOperand(0)),
-                   0);
+    SDValue Val = Op.getOperand(ArgStart + 1);
+    EVT Ty = Val.getValueType();
+    if (Ty == MVT::i128) {
+      // FIXME: Testing one of two paired registers is sufficient to guarantee
+      // ordering?
+      Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, Val);
+    }
+    return SDValue(
+        DAG.getMachineNode(PPC::CFENCE8, DL, MVT::Other,
+                           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Val),
   }
   default:
     break;
   }
   return SDValue();
 }
 
 // Lower scalar BSWAP64 to xxbrd.
SDValue PPCTargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op,
   Ops[2] = NewCmpOp;
   MachineMemOperand *MMO = AtomicNode->getMemOperand();
   SDVTList Tys = DAG.getVTList(MVT::i32, MVT::Other);
   auto NodeTy =
     (MemVT == MVT::i8) ? PPCISD::ATOMIC_CMP_SWAP_8 : PPCISD::ATOMIC_CMP_SWAP_16;
   return DAG.getMemIntrinsicNode(NodeTy, dl, Tys, Ops, MemVT, MMO);
 }
 
+SDValue PPCTargetLowering::LowerATOMIC_LOAD_STORE(SDValue Op,
+                                                  SelectionDAG &DAG) const {
+  AtomicSDNode *N = cast<AtomicSDNode>(Op.getNode());
+  EVT MemVT = N->getMemoryVT();
+  MVT VT = MemVT.getSimpleVT();
+  assert(VT == MVT::i128 && "Expect quadword atomic operations");
+  SDLoc dl(N);
+  unsigned Opc = N->getOpcode();
+  switch (Opc) {
+  case ISD::ATOMIC_LOAD: {
+    // Lower quadword atomic load to int_ppc_atomic_load_i128 which will be
+    // lowered to ppc instructions by pattern matching instruction selector.
+    SDVTList Tys = DAG.getVTList(MVT::i64, MVT::i64, MVT::Other);
+    SmallVector<SDValue, 4> Ops{
+        N->getOperand(0),
+        DAG.getConstant(Intrinsic::ppc_atomic_load_i128, dl, MVT::i32)};
+    for (int I = 1, E = N->getNumOperands(); I < E; ++I)
+      Ops.push_back(N->getOperand(I));
+    SDValue LoadedVal = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl, Tys,
+                                                Ops, MemVT, N->getMemOperand());
+    SDValue ValLo = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i128, LoadedVal);
+    SDValue ValHi =
+        DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i128, LoadedVal.getValue(1));
+    ValHi = DAG.getNode(ISD::SHL, dl, MVT::i128, ValHi,
+                        DAG.getConstant(64, dl, MVT::i32));
+    SDValue Val =
+        DAG.getNode(ISD::OR, dl, {MVT::i128, MVT::Other}, {ValLo, ValHi});
+    return DAG.getNode(ISD::MERGE_VALUES, dl, {MVT::i128, MVT::Other},
+                       {Val, LoadedVal.getValue(2)});
+  }
+  case ISD::ATOMIC_STORE: {
+    // Lower quadword atomic store to int_ppc_atomic_store_i128 which will be
+    // lowered to ppc instructions by pattern matching instruction selector.
+    SDVTList Tys = DAG.getVTList(MVT::Other);
+    SmallVector<SDValue, 4> Ops{
+        N->getOperand(0),
+        DAG.getConstant(Intrinsic::ppc_atomic_store_i128, dl, MVT::i32)};
+    SDValue Val = N->getOperand(2);
+    SDValue ValLo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i64, Val);
+    SDValue ValHi = DAG.getNode(ISD::SRL, dl, MVT::i128, Val,
+                                DAG.getConstant(64, dl, MVT::i32));
+    ValHi = DAG.getNode(ISD::TRUNCATE, dl, MVT::i64, ValHi);
+    Ops.push_back(ValLo);
+    Ops.push_back(ValHi);
+    Ops.push_back(N->getOperand(1));
+    return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, dl, Tys, Ops, MemVT,
+                                   N->getMemOperand());
+  }
+  default:
+    llvm_unreachable("Unexpected atomic opcode");
+  }
+}
+
 SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
                                                  SelectionDAG &DAG) const {
   SDLoc dl(Op);
   // Create a stack slot that is 16-byte aligned.
   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
   int FrameIdx = MFI.CreateStackObject(16, Align(16), false);
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::FRAMEADDR:          return LowerFRAMEADDR(Op, DAG);
 
   case ISD::INTRINSIC_VOID:
     return LowerINTRINSIC_VOID(Op, DAG);
   case ISD::BSWAP:
     return LowerBSWAP(Op, DAG);
   case ISD::ATOMIC_CMP_SWAP:
     return LowerATOMIC_CMP_SWAP(Op, DAG);
+  case ISD::ATOMIC_STORE:
+    return LowerATOMIC_LOAD_STORE(Op, DAG);
   }
 }
 
 void PPCTargetLowering::ReplaceNodeResults(SDNode *N,
                                            SmallVectorImpl<SDValue>&Results,
                                            SelectionDAG &DAG) const {
   SDLoc dl(N);
   switch (N->getOpcode()) {
   default:
     llvm_unreachable("Do not know how to custom type legalize this operation!");
+  case ISD::ATOMIC_LOAD: {
+    SDValue Res = LowerATOMIC_LOAD_STORE(SDValue(N, 0), DAG);
+    Results.push_back(Res);
+    Results.push_back(Res.getValue(1));
+    break;
+  }
   case ISD::READCYCLECOUNTER: {
     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
     SDValue RTB = DAG.getNode(PPCISD::READ_TIME_BASE, dl, VTs, N->getOperand(0));
 
     Results.push_back(
         DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, RTB, RTB.getValue(1)));
     Results.push_back(RTB.getValue(2));
     break;
else if (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I32 ||
     Register Lo = MI.getOperand(0).getReg();
     Register Hi = MI.getOperand(1).getReg();
     BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY))
         .addDef(Lo)
         .addUse(Src, 0, PPC::sub_gp8_x1);
     BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY))
         .addDef(Hi)
         .addUse(Src, 0, PPC::sub_gp8_x0);
+  } else if (MI.getOpcode() == PPC::LQX_PSEUDO ||
+             MI.getOpcode() == PPC::STQX_PSEUDO) {
+    DebugLoc DL = MI.getDebugLoc();
+    // Ptr is used as the ptr_rc_no_r0 part
+    // of LQ/STQ's memory operand and adding result of RA and RB,
+    // so it has to be g8rc_and_g8rc_nox0.
+    Register Ptr =
+        F->getRegInfo().createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass);
+    Register Val = MI.getOperand(0).getReg();
+    Register RA = MI.getOperand(1).getReg();
+    Register RB = MI.getOperand(2).getReg();
+    BuildMI(*BB, MI, DL, TII->get(PPC::ADD8), Ptr).addReg(RA).addReg(RB);
+    BuildMI(*BB, MI, DL,
+            MI.getOpcode() == PPC::LQX_PSEUDO ? TII->get(PPC::LQ)
+                                              : TII->get(PPC::STQ))
+        .addReg(Val, MI.getOpcode() == PPC::LQX_PSEUDO ? RegState::Define : 0)
+        .addImm(0)
+        .addReg(Ptr);
   } else {
     llvm_unreachable("Unexpected instr type to insert");
   }
 
   MI.eraseFromParent(); // The pseudo instruction is gone now.
   return BB;
 }
 
case Intrinsic::ppc_cmpxchg_i128:
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     Info.memVT = MVT::i128;
     Info.ptrVal = I.getArgOperand(0);
     Info.offset = 0;
     Info.align = Align(16);
     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
                  MachineMemOperand::MOVolatile;
     return true;
+  case Intrinsic::ppc_atomic_load_i128:
+    Info.opc = ISD::INTRINSIC_W_CHAIN;
+    Info.memVT = MVT::i128;
+    Info.ptrVal = I.getArgOperand(0);
+    Info.offset = 0;
+    Info.align = Align(16);
+    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
+    return true;
+  case Intrinsic::ppc_atomic_store_i128:
+    Info.opc = ISD::INTRINSIC_VOID;
+    Info.memVT = MVT::i128;
+    Info.ptrVal = I.getArgOperand(2);
+    Info.offset = 0;
+    Info.align = Align(16);
+    Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
+    return true;
   case Intrinsic::ppc_altivec_lvx:
   case Intrinsic::ppc_altivec_lvxl:
   case Intrinsic::ppc_altivec_lvebx:
   case Intrinsic::ppc_altivec_lvehx:
   case Intrinsic::ppc_altivec_lvewx:
   case Intrinsic::ppc_vsx_lxvd2x:
   case Intrinsic::ppc_vsx_lxvw4x:
   case Intrinsic::ppc_vsx_lxvd2x_be:
unsigned PPCTargetLowering::computeMOFlags(const SDNode *Parent, SDValue N,
 
   // Compute in-memory type flags. This is based on if there are scalars,
   // floats or vectors.
   const MemSDNode *MN = dyn_cast<MemSDNode>(Parent);
   assert(MN && "Parent should be a MemSDNode!");
   EVT MemVT = MN->getMemoryVT();
   unsigned Size = MemVT.getSizeInBits();
   if (MemVT.isScalarInteger()) {
-    assert(Size <= 64 && "Not expecting scalar integers larger than 8 bytes!");
+    assert(Size <= 128 &&
+           "Not expecting scalar integers larger than 16 bytes!");
     if (Size < 32)
       FlagSet |= PPC::MOF_SubWordInt;
     else if (Size == 32)
       FlagSet |= PPC::MOF_WordInt;
     else
       FlagSet |= PPC::MOF_DoubleWordInt;
   } else if (MemVT.isVector() && !MemVT.isFloatingPoint()) { // Integer vectors.
     if (Size == 128)

llvm/lib/Target/PowerPC/PPCInstr64Bit.td

 // TODO: Add scheduling info.
 def LQ   : DQForm_RTp5_RA17_MEM<56, 0,
                                 (outs g8prc:$RTp),
                                 (ins memrix16:$src),
                                 "lq $RTp, $src", IIC_LdStLQ,
                                 []>,
                                 RegConstraint<"@earlyclobber $RTp">,
                                 isPPC64;
+// We don't really have LQX in the ISA, make a pseudo one so that we can
+// handle x-form during isel. Make it pre-ra may expose
+// oppotunities to some opts(CSE, LICM and etc.) for the result of adding
+// RA and RB.
+def LQX_PSEUDO : PPCCustomInserterPseudo<(outs g8prc:$RTp),
+                                         (ins memrr:$src), "#LQX_PSEUDO", []>;
+
 def RESTORE_QUADWORD : PPCEmitTimePseudo<(outs g8prc:$RTp), (ins memrix:$src),
                                          "#RESTORE_QUADWORD", []>;
 }
 
 }
 
+def : Pat<(int_ppc_atomic_load_i128 iaddrX16:$src),
+          (SPLIT_QUADWORD (LQ memrix16:$src))>;
+
+def : Pat<(int_ppc_atomic_load_i128 ForceXForm:$src),
+          (SPLIT_QUADWORD (LQX_PSEUDO memrr:$src))>;
+
 // Support for medium and large code model.
 let hasSideEffects = 0 in {
 let isReMaterializable = 1 in {
 def ADDIStocHA8: PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, tocentry:$disp),
                        "#ADDIStocHA8", []>, isPPC64;
 def ADDItocL: PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, tocentry:$disp),
                      "#ADDItocL", []>, isPPC64;
 }
def STDBRX: XForm_8_memOp<31, 660, (outs), (ins g8rc:$rS, memrr:$dst),
                           PPC970_DGroup_Cracked;
 
 let mayStore = 1, hasNoSchedulingInfo = 1 in {
 // Normal 16-byte stores.
 // TODO: Add scheduling info.
 def STQ : DSForm_1<62, 2, (outs), (ins g8prc:$RSp, memrix:$dst),
                    "stq $RSp, $dst", IIC_LdStSTQ,
                    []>, isPPC64;
+
+def STQX_PSEUDO : PPCCustomInserterPseudo<(outs),
+                                          (ins g8prc:$RSp, memrr:$dst),
+                                          "#STQX_PSEUDO", []>;
+
 def SPILL_QUADWORD : PPCEmitTimePseudo<(outs), (ins g8prc:$RSp, memrix:$dst),
                                        "#SPILL_QUADWORD", []>;
 }
 
 }
 
+def BUILD_QUADWORD : PPCPostRAExpPseudo<
+                       (outs g8prc:$RTp),
+                       (ins g8rc:$lo, g8rc:$hi),
+                       "#BUILD_QUADWORD", []>;
+
+def : Pat<(int_ppc_atomic_store_i128 i64:$lo, i64:$hi, DSForm:$dst),
+          (STQ (BUILD_QUADWORD g8rc:$lo, g8rc:$hi), memrix:$dst)>;
+
+def : Pat<(int_ppc_atomic_store_i128 i64:$lo, i64:$hi, ForceXForm:$dst),
+          (STQX_PSEUDO (BUILD_QUADWORD g8rc:$lo, g8rc:$hi), memrr:$dst)>;
+
 // Stores with Update (pre-inc).
 let PPC970_Unit = 2, mayStore = 1, mayLoad = 0 in {
 let Interpretation64Bit = 1, isCodeGenOnly = 1 in {
 def STBU8 : DForm_1<39, (outs ptr_rc_nor0:$ea_res), (ins g8rc:$rS, memri:$dst),
                    "stbu $rS, $dst", IIC_LdStSTU, []>,
                    RegConstraint<"$dst.reg = $ea_res">, NoEncode<"$ea_res">;
 def STHU8 : DForm_1<45, (outs ptr_rc_nor0:$ea_res), (ins g8rc:$rS, memri:$dst),
                    "sthu $rS, $dst", IIC_LdStSTU, []>,

llvm/lib/Target/PowerPC/PPCInstrInfo.cpp

if (PPC::VSFRCRegClass.contains(SrcReg)) {
       MI.setDesc(get(PPC::STXSDX));
     } else {
       NumStoreSPILLVSRRCAsGpr++;
       MI.setDesc(get(PPC::STDX));
     }
     return true;
   }
 
+    // FIXME: Maybe we can expand it in 'PowerPC Expand Atomic' pass.
   case PPC::CFENCE8: {
     auto Val = MI.getOperand(0).getReg();
     BuildMI(MBB, MI, DL, get(PPC::CMPD), PPC::CR7).addReg(Val).addReg(Val);
     BuildMI(MBB, MI, DL, get(PPC::CTRL_DEP))
         .addImm(PPC::PRED_NE_MINUS)
         .addReg(PPC::CR7)
         .addImm(1);
     MI.setDesc(get(PPC::ISYNC));

llvm/test/CodeGen/PowerPC/atomics-i128-ldst.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-unknown \
+; RUN:   -mcpu=pwr8 -ppc-asm-full-reg-names -ppc-track-subreg-liveness \
+; RUN:   -ppc-quadword-atomics < %s | FileCheck --check-prefix=P8 %s
+
+define dso_local i128 @lq_unordered(i128* %src) {
+; P8-LABEL: lq_unordered:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lq r4, 0(r3)
+; P8-NEXT:    mr r3, r4
+; P8-NEXT:    mr r4, r5
+; P8-NEXT:    blr
+entry:
+  %0 = load atomic i128, i128* %src unordered, align 16
+  ret i128 %0
+}
+
+define dso_local i128 @lqx_unordered(i128* %src, i64 %idx) {
+; P8-LABEL: lqx_unordered:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    sldi r4, r4, 4
+; P8-NEXT:    add r3, r3, r4
+; P8-NEXT:    lq r4, 0(r3)
+; P8-NEXT:    mr r3, r4
+; P8-NEXT:    mr r4, r5
+; P8-NEXT:    blr
+entry:
+  %0 = getelementptr i128, i128* %src, i64 %idx
+  %1 = load atomic i128, i128* %0 unordered, align 16
+  ret i128 %1
+}
+
+define dso_local i128 @lq_big_offset_unordered(i128* %src) {
+; P8-LABEL: lq_big_offset_unordered:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lis r4, 32
+; P8-NEXT:    add r3, r3, r4
+; P8-NEXT:    lq r4, 0(r3)
+; P8-NEXT:    mr r3, r4
+; P8-NEXT:    mr r4, r5
+; P8-NEXT:    blr
+entry:
+  %0 = getelementptr i128, i128* %src, i64 131072
+  %1 = load atomic i128, i128* %0 unordered, align 16
+  ret i128 %1
+}
+
+define dso_local i128 @lq_monotonic(i128* %src) {
+; P8-LABEL: lq_monotonic:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lq r4, 0(r3)
+; P8-NEXT:    mr r3, r4
+; P8-NEXT:    mr r4, r5
+; P8-NEXT:    blr
+entry:
+  %0 = load atomic i128, i128* %src monotonic, align 16
+  ret i128 %0
+}
+
+define dso_local i128 @lq_acquire(i128* %src) {
+; P8-LABEL: lq_acquire:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lq r4, 0(r3)
+; P8-NEXT:    cmpd cr7, r5, r5
+; P8-NEXT:    mr r3, r4
+; P8-NEXT:    mr r4, r5
+; P8-NEXT:    bne- cr7, .+4
+; P8-NEXT:    isync
+; P8-NEXT:    blr
+entry:
+  %0 = load atomic i128, i128* %src acquire, align 16
+  ret i128 %0
+}
+
+define dso_local i128 @lq_seqcst(i128* %src) {
+; P8-LABEL: lq_seqcst:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    sync
+; P8-NEXT:    lq r4, 0(r3)
+; P8-NEXT:    cmpd cr7, r5, r5
+; P8-NEXT:    mr r3, r4
+; P8-NEXT:    mr r4, r5
+; P8-NEXT:    bne- cr7, .+4
+; P8-NEXT:    isync
+; P8-NEXT:    blr
+entry:
+  %0 = load atomic i128, i128* %src seq_cst, align 16
+  ret i128 %0
+}
+
+define dso_local void @stq_unordered(i128 %val, i128* %dst) {
+; P8-LABEL: stq_unordered:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    mr r7, r4
+; P8-NEXT:    mr r6, r3
+; P8-NEXT:    stq r6, 0(r5)
+; P8-NEXT:    blr
+entry:
+  store atomic i128 %val, i128* %dst unordered, align 16
+  ret void
+}
+
+define dso_local void @stqx_unordered(i128 %val, i128* %dst, i64 %idx) {
+; P8-LABEL: stqx_unordered:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    sldi r6, r6, 4
+; P8-NEXT:    mr r9, r4
+; P8-NEXT:    mr r8, r3
+; P8-NEXT:    add r3, r5, r6
+; P8-NEXT:    stq r8, 0(r3)
+; P8-NEXT:    blr
+entry:
+  %0 = getelementptr i128, i128* %dst, i64 %idx
+  store atomic i128 %val, i128* %0 unordered, align 16
+  ret void
+}
+
+define dso_local void @stq_big_offset_unordered(i128 %val, i128* %dst) {
+; P8-LABEL: stq_big_offset_unordered:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lis r6, 32
+; P8-NEXT:    mr r9, r4
+; P8-NEXT:    mr r8, r3
+; P8-NEXT:    add r3, r5, r6
+; P8-NEXT:    stq r8, 0(r3)
+; P8-NEXT:    blr
+entry:
+  %0 = getelementptr i128, i128* %dst, i64 131072
+  store atomic i128 %val, i128* %0 unordered, align 16
+  ret void
+}
+
+define dso_local void @stq_monotonic(i128 %val, i128* %dst) {
+; P8-LABEL: stq_monotonic:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    mr r7, r4
+; P8-NEXT:    mr r6, r3
+; P8-NEXT:    stq r6, 0(r5)
+; P8-NEXT:    blr
+entry:
+  store atomic i128 %val, i128* %dst monotonic, align 16
+  ret void
+}
+
+define dso_local void @stq_release(i128 %val, i128* %dst) {
+; P8-LABEL: stq_release:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lwsync
+; P8-NEXT:    mr r7, r4
+; P8-NEXT:    mr r6, r3
+; P8-NEXT:    stq r6, 0(r5)
+; P8-NEXT:    blr
+entry:
+  store atomic i128 %val, i128* %dst release, align 16
+  ret void
+}
+
+define dso_local void @stq_seqcst(i128 %val, i128* %dst) {
+; P8-LABEL: stq_seqcst:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    sync
+; P8-NEXT:    mr r7, r4
+; P8-NEXT:    mr r6, r3
+; P8-NEXT:    stq r6, 0(r5)
+; P8-NEXT:    blr
+entry:
+  store atomic i128 %val, i128* %dst seq_cst, align 16
+  ret void
+}