[SystemZ] Support ATOMIC_FENCE

[bryanpkc]

Summary

A cross-thread sequentially consistent fence should be lowered into z/Architecture's BCR serialization instruction, instead of causing a fatal error in the back-end.

[uweigand]

This seems overly conservative: I think we only need a BCR serialization instruction for sequentially-consistent cross-thread fences. For everything else, a pure compiler barrier should be sufficient. See e.g. the X86 back-end for how to implement that check.

[bryanpkc]

I have updated the patch to mimic the behaviour of ATOMIC_FENCE on X86, as suggested.

[uweigand]

LGTM, thanks.

lib/Target/SystemZ/SystemZISelLowering.h

private:
   SDValue lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerSMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerUMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerSDIVREM(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerUDIVREM(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerATOMIC_LOAD(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerATOMIC_LOAD_OP(SDValue Op, SelectionDAG &DAG,
                               unsigned Opcode) const;
   SDValue lowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerLOAD_SEQUENCE_POINT(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const;

lib/Target/SystemZ/SystemZISelLowering.cpp

SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::ATOMIC_LOAD_XOR,  MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_MIN,  MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_MAX,  MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_CMP_SWAP,  MVT::i32, Custom);
 
+  setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
+
   // z10 has instructions for signed but not unsigned FP conversion.
   // Handle unsigned 32-bit types as signed 64-bit types.
   if (!Subtarget.hasFPExtension()) {
     setOperationAction(ISD::UINT_TO_FP, MVT::i32, Promote);
     setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand);
   }
 
   // We have native support for a 64-bit CTLZ, via FLOGR.
SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
   // Extract overall result from high byte.
   if (BitSize > 8)
     Op = DAG.getNode(ISD::SRL, DL, VT, Op,
                      DAG.getConstant(BitSize - 8, DL, VT));
 
   return Op;
 }
 
+SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
+                                                 SelectionDAG &DAG) const {
+  SDLoc DL(Op);
+  return SDValue(DAG.getMachineNode(SystemZ::Serialize, DL, MVT::Other), 0);
+}
+
 // Op is an atomic load.  Lower it into a normal volatile load.
 SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
                                                 SelectionDAG &DAG) const {
   auto *Node = cast<AtomicSDNode>(Op.getNode());
   return DAG.getExtLoad(ISD::EXTLOAD, SDLoc(Op), Op.getValueType(),
                         Node->getChain(), Node->getBasePtr(),
                         Node->getMemoryVT(), Node->getMemOperand());
 }
SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
   case ISD::CTPOP:
     return lowerCTPOP(Op, DAG);
   case ISD::CTLZ_ZERO_UNDEF:
     return DAG.getNode(ISD::CTLZ, SDLoc(Op),
                        Op.getValueType(), Op.getOperand(0));
   case ISD::CTTZ_ZERO_UNDEF:
     return DAG.getNode(ISD::CTTZ, SDLoc(Op),
                        Op.getValueType(), Op.getOperand(0));
+  case ISD::ATOMIC_FENCE:
+    return lowerATOMIC_FENCE(Op, DAG);
   case ISD::ATOMIC_SWAP:
     return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_SWAPW);
   case ISD::ATOMIC_STORE:
     return lowerATOMIC_STORE(Op, DAG);
   case ISD::ATOMIC_LOAD:
     return lowerATOMIC_LOAD(Op, DAG);
   case ISD::ATOMIC_LOAD_ADD:
     return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD);

test/CodeGen/SystemZ/atomic-fence.ll

+; Test (fast) serialization.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s --check-prefix=Z10
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z196  | FileCheck %s --check-prefix=Z196
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=zEC12 | FileCheck %s --check-prefix=ZEC12
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13   | FileCheck %s --check-prefix=Z13
+
+define void @test() {
+; Z10:   bcr 15, %r0
+; Z196:  bcr 14, %r0
+; ZEC12: bcr 14, %r0
+; Z13:   bcr 14, %r0
+  fence seq_cst
+  ret void
+}
+