Add TargetLoweringInfo hook for explicitly setting the ABI calling convention endianess

[dylanmckay]

Summary

The endianess used in the calling convention does not always match the
endianess of the target on all architectures, namely AVR.

When an argument is too large to be legalised by the architecture and is
split for the ABI, a new hook TargetLoweringInfo::shouldSplitFunctionArgumentsAsLittleEndian
is queried to find the endianess that function arguments must be laid
out in.

This approach was recommended by Eli Friedman.

Originally reported in https://github.com/avr-rust/rust/issues/129.

Patch by Carl Peto.

[efriedma]

LGTM

include/llvm/CodeGen/TargetLowering.h

3541

Maybe "for some calls"; this applies to basically all targets.

[dylanmckay]

Thanks heaps for the quick turnaround @efriedma, I will amend the comment as I commit this patch.

include/llvm/CodeGen/TargetLowering.h

3541

Good point, I will amend this as I commit it.

include/llvm/CodeGen/TargetLowering.h

public:
   /// simple types, but the calling convention specifies that the entire struct
   /// must be passed in a block of consecutive registers.
   virtual bool
   functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv,
                                             bool isVarArg) const {
     return false;
   }
 
+  /// For some targets, an LLVM type must be broken down into multiple
+  /// smaller types. Usually the halves are ordered according to the endianness
+  /// but for some platform that would break. So this method will default to
+  /// matching the endianness but can be overridden.
+  virtual bool
+  shouldSplitFunctionArgumentsAsLittleEndian(const DataLayout &DL) const {
+    return DL.isLittleEndian();
+  }
+
   /// Returns a 0 terminated array of registers that can be safely used as
   /// scratch registers.
   virtual const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const {
     return nullptr;
   }
 
   /// This callback is used to prepare for a volatile or atomic load.
   /// It takes a chain node as input and returns the chain for the load itself.

lib/CodeGen/SelectionDAG/TargetLowering.cpp

if (isSigned) {
         HiRHS = DAG.getConstant(0, dl, VT);
     }
 
     // Here we're passing the 2 arguments explicitly as 4 arguments that are
     // pre-lowered to the correct types. This all depends upon WideVT not
     // being a legal type for the architecture and thus has to be split to
     // two arguments.
     SDValue Ret;
-    if (DAG.getDataLayout().isLittleEndian()) {
+    if (shouldSplitFunctionArgumentsAsLittleEndian(DAG.getDataLayout())) {
       // Halves of WideVT are packed into registers in different order
       // depending on platform endianness. This is usually handled by
       // the C calling convention, but we can't defer to it in
       // the legalizer.
       SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
       Ret = makeLibCall(DAG, LC, WideVT, Args, isSigned, dl,
           /* doesNotReturn */ false, /* isReturnValueUsed */ true,
           /* isPostTypeLegalization */ true).first;

lib/Target/AVR/AVRISelLowering.h

public:
 
   void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint,
                                     std::vector<SDValue> &Ops,
                                     SelectionDAG &DAG) const override;
 
   unsigned getRegisterByName(const char* RegName, EVT VT,
                              SelectionDAG &DAG) const override;
 
+  bool shouldSplitFunctionArgumentsAsLittleEndian(const DataLayout &DL)
+    const override {
+    return false;
+  }
+
 private:
   SDValue getAVRCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue &AVRcc,
                     SelectionDAG &DAG, SDLoc dl) const;
   SDValue LowerShifts(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerDivRem(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;

test/CodeGen/AVR/umul.with.overflow.i16-bug.ll

+; RUN: llc -O1 < %s -march=avr | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9"
+
+%Vs6UInt16 = type <{ i16 }>
+%Sb = type <{ i1 }>
+
+define hidden void @setServoAngle(i16) {
+  ; CHECK-LABEL: entry
+entry:
+  %adjustedAngle = alloca %Vs6UInt16, align 2
+  %1 = bitcast %Vs6UInt16* %adjustedAngle to i8*
+  %adjustedAngle._value = getelementptr inbounds %Vs6UInt16, %Vs6UInt16* %adjustedAngle, i32 0, i32 0
+  store i16 %0, i16* %adjustedAngle._value, align 2
+
+;print(unsignedInt: adjustedAngle &* UInt16(11))
+; breaks here
+  %adjustedAngle._value2 = getelementptr inbounds %Vs6UInt16, %Vs6UInt16* %adjustedAngle, i32 0, i32 0
+  %2 = load i16, i16* %adjustedAngle._value2, align 2
+
+; CHECK: mov r22, r24
+; CHECK: mov r23, r25
+
+; CHECK-DAG: ldi r20, 0
+; CHECK-DAG: ldi r21, 0
+; CHECK-DAG: ldi r18, 11
+; CHECK-DAG: ldi r19, 0
+
+; CHECK: mov r24, r20
+; CHECK: mov r25, r21
+; CHECK: call  __mulsi3
+  %3 = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 %2, i16 11)
+  %4 = extractvalue { i16, i1 } %3, 0
+  %5 = extractvalue { i16, i1 } %3, 1
+
+  ; above code looks fine, how is it lowered?
+  %6 = call i1 @printDefaultParam()
+  call void @print(i16 %4, i1 %6)
+
+; CHECK: ret
+  ret void
+}
+
+declare void @print(i16, i1)
+declare i1 @printDefaultParam()
+
+; Function Attrs: nounwind readnone speculatable
+declare { i16, i1 } @llvm.umul.with.overflow.i16(i16, i16)