AArch64: bitcast ops for i128 on big endian

[cpirker]

Summary

Hi All,

This patch enables the big endian data order for BITCAST operations from i128 to vector types and vice versa.
Please review.

Thanks,
Christian

[cpirker]

Hi Amara,

can you please review.

Thanks,
Christian

[t.p.northover]

Hi Christian,

I think this is another one that's barking up the wrong tree. The existing DAG looks correct to me and what probably needs implementing here (and quite possibly for your constant pool issue too, though I can't be sure until I see a more complete source) is the lane remapping during ISel that me, Albrecht & Jiangning discussed in the other large thread.

My idea for an implementation is that any pattern dealing with vector indices should use an operand (currently "neon_uimm2_bare" and so on) that applies an SDNodeXForm which reverses the lane actually chosen. The basic idea is that LLVM's indices refer to the lowest addressed element, but because of how we've loaded & stored vectors these elements are actually in "MaxLanes - index" of the register.

Some modification to the existing classes along the lines of:

def BE_Reverse_4Lanes : SDNodeXForm<imm, [{
  if (Subtarget->isLittle()) return SDValue(N, 0);
  return CurDAG->getTargetConstant(4 - N->getZExtValue(), MVT::i64);
}]>;

def neon_uimm2_bare : Operand<i64>,
                        ImmLeaf<i64, [{return Imm < 4;}], BE_Reverse_4Lanes> {
  let ParserMatchClass = neon_uimm2_asmoperand;
  let PrintMethod = "printUImmBareOperand";
}

seems to do the trick for me (and should make LD1LN instructions and lots of other things work as expected too). I'm *hoping* this and the LD1/LDR changes are all that will be needed in the backend.

Cheers.

Tim.

lib/Target/AArch64/AArch64ISelLowering.h

   virtual bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
                                   unsigned Intrinsic) const override;
 
+  virtual void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>& Results,
+                                  SelectionDAG &DAG) const;
+
 protected:
   std::pair<const TargetRegisterClass*, uint8_t>
   findRepresentativeClass(MVT VT) const;

lib/Target/AArch64/AArch64ISelLowering.cpp

     setOperationAction(ISD::FPOW, MVT::v2f64, Expand);
     setOperationAction(ISD::FPOW, MVT::v4f32, Expand);
     setOperationAction(ISD::FPOW, MVT::v2f32, Expand);
+
+    if (!Subtarget->isLittle())
+      setOperationAction(ISD::BITCAST, MVT::i128, Custom);
   }
 
   setTargetDAGCombine(ISD::SIGN_EXTEND);
   setTargetDAGCombine(ISD::VSELECT);
 }
 
+static void IntegerToVector(SDValue Op, SelectionDAG &DAG, unsigned NumElements,
+                            SmallVectorImpl<SDValue> &Ops,
+                            EVT EltVT) {
+  assert(Op.getValueType().isInteger());
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  SDLoc dl(Op);
+  SDValue Lo, Hi;
+
+  if (NumElements > 1) {
+    NumElements >>= 1;
+    EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(),
+                                   Op.getValueType().getSizeInBits()/2);
+    Hi = DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Op);
+    Lo = DAG.getNode(ISD::SRL, dl, Op.getValueType(), Op,
+                     DAG.getConstant(HalfVT.getSizeInBits(), TLI.getPointerTy()));
+    Lo = DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Lo);
+
+    IntegerToVector(Hi, DAG, NumElements, Ops, EltVT);
+    IntegerToVector(Lo, DAG, NumElements, Ops, EltVT);
+  } else {
+    Ops.push_back(DAG.getNode(ISD::BITCAST, dl, EltVT, Op));
+  }
+}
+
+static SDValue ExpandBITCAST(SDNode *N, SelectionDAG &DAG) {
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  EVT OutVT = N->getValueType(0);
+  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
+  SDValue InOp = N->getOperand(0);
+  EVT InVT = InOp.getValueType();
+  SDLoc dl(N);
+  assert((InVT == MVT::i128 || OutVT == MVT::i128) &&
+         "ExpandBITCAST called for non-i128 type");
+  if (TLI.getTypeAction(*DAG.getContext(), InVT) == TargetLowering::TypeLegal ||
+      TLI.getTypeAction(*DAG.getContext(), InVT) == TargetLowering::TypePromoteInteger) {
+    if (InVT.isVector() && OutVT.isInteger()) {
+      // Handle cases like i128 = BITCAST v4i32
+      unsigned NumElems = 2;
+      EVT ElemVT = NOutVT;
+      EVT NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
+
+      // If <ElemVT * N> is not a legal type, try <ElemVT/2 * (N*2)>.
+      while (!TLI.isTypeLegal(NVT)) {
+        unsigned NewSizeInBits = ElemVT.getSizeInBits() / 2;
+        // If the element size is smaller than byte, bail.
+        if (NewSizeInBits < 8)
+          break;
+        NumElems *= 2;
+        ElemVT = EVT::getIntegerVT(*DAG.getContext(), NewSizeInBits);
+        NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
+      }
+      if (TLI.isTypeLegal(NVT)) {
+        SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp);
+
+        SmallVector<SDValue, 8> Vals;
+        for (unsigned i = 0; i < NumElems; ++i)
+          SmallVector<SDValue, 8> Vals;
+        for (unsigned i = 0; i < NumElems; ++i)
+          Vals.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
+                                     ElemVT, CastInOp,
+                                     DAG.getConstant(i, TLI.getVectorIdxTy())));
+
+        // Build Lo, Hi pair by pairing extracted elements if needed.
+        unsigned Slot = 0;
+        for (unsigned e = Vals.size(); e - Slot > 2; Slot += 2, e += 1) {
+          // Each iteration will BUILD_PAIR two nodes and append the result until
+          // there are only two nodes left, i.e. Lo and Hi.
+          SDValue LHS = Vals[Slot];
+          SDValue RHS = Vals[Slot + 1];
+
+          if (TLI.isBigEndian())
+            std::swap(LHS, RHS);
+
+          Vals.push_back(DAG.getNode(ISD::BUILD_PAIR, dl,
+                                     EVT::getIntegerVT(
+                                       *DAG.getContext(),
+                                       LHS.getValueType().getSizeInBits() << 1),
+                                     LHS, RHS));
+        }
+        SDValue Lo = Vals[Slot++];
+        SDValue Hi = Vals[Slot];
+        return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i128, Lo, Hi);
+      }
+    }
+  }
+  if (InVT.isInteger() && OutVT.isVector()) {
+    unsigned NumElts = 2;
+    EVT NVT = EVT::getVectorVT(*DAG.getContext(),
+                               TLI.getTypeToTransformTo(*DAG.getContext(), InVT),
+                               NumElts);
+    if (!TLI.isTypeLegal(NVT)) {
+      // If we can't find a legal type by splitting the integer in half,
+      // then we can use the node's value type.
+      NumElts = OutVT.getVectorNumElements();
+      NVT = OutVT;
+    }
+    SmallVector<SDValue, 8> Ops;
+    IntegerToVector(InOp, DAG, NumElts, Ops, NVT.getVectorElementType());
+    SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], NumElts);
+    return DAG.getNode(ISD::BITCAST, dl, OutVT, Vec);
+  }
+  return SDValue();
+}
+
+void
+AArch64TargetLowering::ReplaceNodeResults(SDNode *N,
+                                          SmallVectorImpl<SDValue>& Results,
+                                          SelectionDAG &DAG) const {
+  SDValue Res;
+  switch (N->getOpcode()) {
+  default:
+    llvm_unreachable("Don't know how to custom expand this!");
+  case ISD::BITCAST:
+    Res = ExpandBITCAST(N, DAG);
+    break;
+  }
+  if (Res.getNode())
+    Results.push_back(Res);
+}
+
 EVT AArch64TargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const {
   // It's reasonably important that this value matches the "natural" legal
   // promotion from i1 for scalar types. Otherwise LegalizeTypes can get itself

     return LowerBUILD_VECTOR(Op, DAG, getSubtarget());
   case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
   case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
+  case ISD::BITCAST: return ExpandBITCAST(Op.getNode(), DAG);
   }
 
   return SDValue();

test/CodeGen/AArch64/bitcast-i128.ll

+; RUN: llc -verify-machineinstrs < %s -march aarch64 -mattr neon | FileCheck %s --check-prefix=CHECK-LE
+; RUN: llc -verify-machineinstrs < %s -march aarch64_be -mattr neon | FileCheck %s --check-prefix=CHECK-BE
+
+define void @callee_i128( i128 %var, <4 x i32 >* %storeaddr ) {
+; CHECK-LE-LABEL: callee_i128:
+; CHECK-LE: ins v1.d[0], x0
+; CHECK-LE: ins v1.d[1], x1
+; CHECK-BE-LABEL: callee_i128:
+; CHECK-BE: ins v1.d[0], x1
+; CHECK-BE: ins v1.d[1], x0
+  %v0 = bitcast i128 %var to <4 x i32>
+  %v1 = add <4 x i32> %v0, < i32 0, i32 1, i32 2, i32 3 >
+  store <4 x i32> %v1, <4 x i32>* %storeaddr
+  ret void
+}
+
+define void @caller_i128( <4 x i32>* %loadaddr, <4 x i32>* %storeaddr ) {
+; CHECK-LE-LABEL: caller_i128:
+; CHECK-LE: umov x0, v0.d[0]
+; CHECK-LE: umov x1, v0.d[1]
+; CHECK-BE-LABEL: caller_i128:
+; CHECK-BE: umov x0, v0.d[1]
+; CHECK-BE: umov x1, v0.d[0]
+  %v0 = load <4 x i32>* %loadaddr
+  %v1 = add <4 x i32> %v0, < i32 0, i32 1, i32 2, i32 3 >
+  %v2 = bitcast <4 x i32> %v1 to i128
+  call void (i128, <4 x i32>*)* @callee_i128( i128 %v2, <4 x i32>* %storeaddr )
+  ret void
+}
+