[mips] Enable arithmetic and binary operations for the i128 data type.

[vkalintiris]

Summary

This patch adds support for some operations that were missing from
128-bit integer types (add/sub/mul/sdiv/udiv... etc.). With these
changes we can support the int128_t and uint128_t data types
from C/C++.

Depends on D7125

[dsanders]

LGTM with the comment. As with the other patch, please try to annotate the redundant instructions with FIXME's but there's quite a few so I'm not going to require that.

I was reminded last night that I hadn't committed the add/sub portion I did a few weeks back. I committed it today as r227003. Feel free to revert it if the conflict is awkward since your patch makes the same change and fixes all the other operations too.

lib/Target/Mips/MipsSEISelDAGToDAG.cpp

251–256 ↗

(On Diff #18671)

Please add a comment about this. I believe you are working around the fact that we currently use i32 for conditions on MIPS64 but we should probably be using i64.

llvm/trunk/lib/Target/Mips/Mips64InstrInfo.td

 
 // Sign extend in register
 def : MipsPat<(i64 (sext_inreg GPR64:$src, i32)),
               (SLL64_64 GPR64:$src)>;
 
 // bswap MipsPattern
 def : MipsPat<(bswap GPR64:$rt), (DSHD (DSBH GPR64:$rt))>;
 
+// Carry pattern
+def : MipsPat<(subc GPR64:$lhs, GPR64:$rhs),
+              (DSUBu GPR64:$lhs, GPR64:$rhs)>;
+let AdditionalPredicates = [NotDSP] in {
+  def : MipsPat<(addc GPR64:$lhs, GPR64:$rhs),
+                (DADDu GPR64:$lhs, GPR64:$rhs)>;
+  def : MipsPat<(addc GPR64:$lhs, immSExt16:$imm),
+                (DADDiu GPR64:$lhs, imm:$imm)>;
+}
+
 // Octeon bbit0/bbit1 MipsPattern
 let Predicates = [HasMips64, HasCnMips] in {
 def : MipsPat<(brcond (i32 (seteq (and i64:$lhs, PowerOf2LO:$mask), 0)), bb:$dst),
               (BBIT0 i64:$lhs, (Log2LO PowerOf2LO:$mask), bb:$dst)>;
 def : MipsPat<(brcond (i32 (seteq (and i64:$lhs, PowerOf2HI:$mask), 0)), bb:$dst),
               (BBIT032 i64:$lhs, (Log2HI PowerOf2HI:$mask), bb:$dst)>;
 def : MipsPat<(brcond (i32 (setne (and i64:$lhs, PowerOf2LO:$mask), 0)), bb:$dst),
               (BBIT1 i64:$lhs, (Log2LO PowerOf2LO:$mask), bb:$dst)>;

llvm/trunk/lib/Target/Mips/MipsISelLowering.cpp

if (Subtarget.isGP64bit()) {
     setOperationAction(ISD::BlockAddress,       MVT::i64,   Custom);
     setOperationAction(ISD::GlobalTLSAddress,   MVT::i64,   Custom);
     setOperationAction(ISD::JumpTable,          MVT::i64,   Custom);
     setOperationAction(ISD::ConstantPool,       MVT::i64,   Custom);
     setOperationAction(ISD::SELECT,             MVT::i64,   Custom);
     setOperationAction(ISD::LOAD,               MVT::i64,   Custom);
     setOperationAction(ISD::STORE,              MVT::i64,   Custom);
     setOperationAction(ISD::FP_TO_SINT,         MVT::i64,   Custom);
+    setOperationAction(ISD::SHL_PARTS,          MVT::i64,   Custom);
+    setOperationAction(ISD::SRA_PARTS,          MVT::i64,   Custom);
+    setOperationAction(ISD::SRL_PARTS,          MVT::i64,   Custom);
   }
 
   if (!Subtarget.isGP64bit()) {
     setOperationAction(ISD::SHL_PARTS,          MVT::i32,   Custom);
     setOperationAction(ISD::SRA_PARTS,          MVT::i32,   Custom);
     setOperationAction(ISD::SRL_PARTS,          MVT::i32,   Custom);
   }
 
SDValue MipsTargetLowering::lowerATOMIC_FENCE(SDValue Op,
   SDLoc DL(Op);
   return DAG.getNode(MipsISD::Sync, DL, MVT::Other, Op.getOperand(0),
                      DAG.getConstant(SType, MVT::i32));
 }
 
 SDValue MipsTargetLowering::lowerShiftLeftParts(SDValue Op,
                                                 SelectionDAG &DAG) const {
   SDLoc DL(Op);
+  MVT VT = Subtarget.isGP64bit() ? MVT::i64 : MVT::i32;
+
   SDValue Lo = Op.getOperand(0), Hi = Op.getOperand(1);
   SDValue Shamt = Op.getOperand(2);
-
-  // if shamt < 32:
+  // if shamt < (VT.bits):
   //  lo = (shl lo, shamt)
   //  hi = (or (shl hi, shamt) (srl (srl lo, 1), ~shamt))
   // else:
   //  lo = 0
   //  hi = (shl lo, shamt[4:0])
   SDValue Not = DAG.getNode(ISD::XOR, DL, MVT::i32, Shamt,
                             DAG.getConstant(-1, MVT::i32));
-  SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, MVT::i32, Lo,
-                                      DAG.getConstant(1, MVT::i32));
-  SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, MVT::i32, ShiftRight1Lo,
-                                     Not);
-  SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, MVT::i32, Hi, Shamt);
-  SDValue Or = DAG.getNode(ISD::OR, DL, MVT::i32, ShiftLeftHi, ShiftRightLo);
-  SDValue ShiftLeftLo = DAG.getNode(ISD::SHL, DL, MVT::i32, Lo, Shamt);
+  SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo,
+                                      DAG.getConstant(1, VT));
+  SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, Not);
+  SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
+  SDValue Or = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
+  SDValue ShiftLeftLo = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
   SDValue Cond = DAG.getNode(ISD::AND, DL, MVT::i32, Shamt,
                              DAG.getConstant(0x20, MVT::i32));
-  Lo = DAG.getNode(ISD::SELECT, DL, MVT::i32, Cond,
-                   DAG.getConstant(0, MVT::i32), ShiftLeftLo);
-  Hi = DAG.getNode(ISD::SELECT, DL, MVT::i32, Cond, ShiftLeftLo, Or);
+  Lo = DAG.getNode(ISD::SELECT, DL, VT, Cond,
+                   DAG.getConstant(0, VT), ShiftLeftLo);
+  Hi = DAG.getNode(ISD::SELECT, DL, VT, Cond, ShiftLeftLo, Or);
 
   SDValue Ops[2] = {Lo, Hi};
   return DAG.getMergeValues(Ops, DL);
 }
 
 SDValue MipsTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
                                                  bool IsSRA) const {
   SDLoc DL(Op);
   SDValue Lo = Op.getOperand(0), Hi = Op.getOperand(1);
   SDValue Shamt = Op.getOperand(2);
+  MVT VT = Subtarget.isGP64bit() ? MVT::i64 : MVT::i32;
 
-  // if shamt < 32:
+  // if shamt < (VT.bits):
   //  lo = (or (shl (shl hi, 1), ~shamt) (srl lo, shamt))
   //  if isSRA:
   //    hi = (sra hi, shamt)
   //  else:
   //    hi = (srl hi, shamt)
   // else:
   //  if isSRA:
   //   lo = (sra hi, shamt[4:0])
   //   hi = (sra hi, 31)
   //  else:
   //   lo = (srl hi, shamt[4:0])
   //   hi = 0
   SDValue Not = DAG.getNode(ISD::XOR, DL, MVT::i32, Shamt,
                             DAG.getConstant(-1, MVT::i32));
-  SDValue ShiftLeft1Hi = DAG.getNode(ISD::SHL, DL, MVT::i32, Hi,
-                                     DAG.getConstant(1, MVT::i32));
-  SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, MVT::i32, ShiftLeft1Hi, Not);
-  SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, MVT::i32, Lo, Shamt);
-  SDValue Or = DAG.getNode(ISD::OR, DL, MVT::i32, ShiftLeftHi, ShiftRightLo);
-  SDValue ShiftRightHi = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, DL, MVT::i32,
-                                     Hi, Shamt);
+  SDValue ShiftLeft1Hi = DAG.getNode(ISD::SHL, DL, VT, Hi,
+                                     DAG.getConstant(1, VT));
+  SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, ShiftLeft1Hi, Not);
+  SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
+  SDValue Or = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
+  SDValue ShiftRightHi = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL,
+                                     DL, VT, Hi, Shamt);
   SDValue Cond = DAG.getNode(ISD::AND, DL, MVT::i32, Shamt,
                              DAG.getConstant(0x20, MVT::i32));
-  SDValue Shift31 = DAG.getNode(ISD::SRA, DL, MVT::i32, Hi,
-                                DAG.getConstant(31, MVT::i32));
-  Lo = DAG.getNode(ISD::SELECT, DL, MVT::i32, Cond, ShiftRightHi, Or);
-  Hi = DAG.getNode(ISD::SELECT, DL, MVT::i32, Cond,
-                   IsSRA ? Shift31 : DAG.getConstant(0, MVT::i32),
-                   ShiftRightHi);
+  SDValue Shift31 = DAG.getNode(ISD::SRA, DL, VT, Hi, DAG.getConstant(31, VT));
+  Lo = DAG.getNode(ISD::SELECT, DL, VT, Cond, ShiftRightHi, Or);
+  Hi = DAG.getNode(ISD::SELECT, DL, VT, Cond,
+                   IsSRA ? Shift31 : DAG.getConstant(0, VT), ShiftRightHi);
 
   SDValue Ops[2] = {Lo, Hi};
   return DAG.getMergeValues(Ops, DL);
 }
 
 static SDValue createLoadLR(unsigned Opc, SelectionDAG &DAG, LoadSDNode *LD,
                             SDValue Chain, SDValue Src, unsigned Offset) {
   SDValue Ptr = LD->getBasePtr();

llvm/trunk/lib/Target/Mips/MipsSEISelDAGToDAG.cpp

SDNode *MipsSEDAGToDAGISel::selectAddESubE(unsigned MOp, SDValue InFlag,
                                            SDValue CmpLHS, SDLoc DL,
                                            SDNode *Node) const {
   unsigned Opc = InFlag.getOpcode(); (void)Opc;
 
   assert(((Opc == ISD::ADDC || Opc == ISD::ADDE) ||
           (Opc == ISD::SUBC || Opc == ISD::SUBE)) &&
          "(ADD|SUB)E flag operand must come from (ADD|SUB)C/E insn");
 
+  unsigned SLTuOp = Mips::SLTu, ADDuOp = Mips::ADDu;
+  if (Subtarget->isGP64bit()) {
+    SLTuOp = Mips::SLTu64;
+    ADDuOp = Mips::DADDu;
+  }
+
   SDValue Ops[] = { CmpLHS, InFlag.getOperand(1) };
   SDValue LHS = Node->getOperand(0), RHS = Node->getOperand(1);
   EVT VT = LHS.getValueType();
 
-  SDNode *Carry = CurDAG->getMachineNode(Mips::SLTu, DL, VT, Ops);
-  SDNode *AddCarry = CurDAG->getMachineNode(Mips::ADDu, DL, VT,
+  SDNode *Carry = CurDAG->getMachineNode(SLTuOp, DL, VT, Ops);
+
+  if (Subtarget->isGP64bit()) {
+    // On 64-bit targets, sltu produces an i64 but our backend currently says
+    // that SLTu64 produces an i32. We need to fix this in the long run but for
+    // now, just make the DAG type-correct by asserting the upper bits are zero.
+    Carry = CurDAG->getMachineNode(Mips::SUBREG_TO_REG, DL, VT,
+                                   CurDAG->getTargetConstant(0, VT),
+                                   SDValue(Carry, 0),
+                                   CurDAG->getTargetConstant(Mips::sub_32, VT));
+  }
+
+  SDNode *AddCarry = CurDAG->getMachineNode(ADDuOp, DL, VT,
                                             SDValue(Carry, 0), RHS);
+
   return CurDAG->SelectNodeTo(Node, MOp, VT, MVT::Glue, LHS,
                               SDValue(AddCarry, 0));
 }
 
 /// Match frameindex
 bool MipsSEDAGToDAGISel::selectAddrFrameIndex(SDValue Addr, SDValue &Base,
                                               SDValue &Offset) const {
   if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
std::pair<bool, SDNode*> MipsSEDAGToDAGISel::selectNode(SDNode *Node) {
   ///
   SDNode *Result;
 
   switch(Opcode) {
   default: break;
 
   case ISD::SUBE: {
     SDValue InFlag = Node->getOperand(2);
-    Result = selectAddESubE(Mips::SUBu, InFlag, InFlag.getOperand(0), DL, Node);
+    unsigned Opc = Subtarget->isGP64bit() ? Mips::DSUBu : Mips::SUBu;
+    Result = selectAddESubE(Opc, InFlag, InFlag.getOperand(0), DL, Node);
     return std::make_pair(true, Result);
   }
 
   case ISD::ADDE: {
     if (Subtarget->hasDSP()) // Select DSP instructions, ADDSC and ADDWC.
       break;
     SDValue InFlag = Node->getOperand(2);
-    Result = selectAddESubE(Mips::ADDu, InFlag, InFlag.getValue(0), DL, Node);
+    unsigned Opc = Subtarget->isGP64bit() ? Mips::DADDu : Mips::ADDu;
+    Result = selectAddESubE(Opc, InFlag, InFlag.getValue(0), DL, Node);
     return std::make_pair(true, Result);
   }
 
   case ISD::ConstantFP: {
     ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Node);
     if (Node->getValueType(0) == MVT::f64 && CN->isExactlyValue(+0.0)) {
       if (Subtarget->isGP64bit()) {
         SDValue Zero = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL,

llvm/trunk/lib/Target/Mips/MipsSEISelLowering.cpp

MipsSETargetLowering::MipsSETargetLowering(const MipsTargetMachine &TM,
   setOperationAction(ISD::MULHU,              MVT::i32, Custom);
 
   if (Subtarget.hasCnMips())
     setOperationAction(ISD::MUL,              MVT::i64, Legal);
   else if (Subtarget.isGP64bit())
     setOperationAction(ISD::MUL,              MVT::i64, Custom);
 
   if (Subtarget.isGP64bit()) {
+    setOperationAction(ISD::SMUL_LOHI,        MVT::i64, Custom);
+    setOperationAction(ISD::UMUL_LOHI,        MVT::i64, Custom);
     setOperationAction(ISD::MULHS,            MVT::i64, Custom);
     setOperationAction(ISD::MULHU,            MVT::i64, Custom);
     setOperationAction(ISD::SDIVREM,          MVT::i64, Custom);
     setOperationAction(ISD::UDIVREM,          MVT::i64, Custom);
   }
 
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
   setOperationAction(ISD::INTRINSIC_W_CHAIN,  MVT::i64, Custom);
if (Subtarget.hasMips32r6()) {
     setCondCodeAction(ISD::SETOGT, MVT::f64, Expand);
     setCondCodeAction(ISD::SETUGE, MVT::f64, Expand);
     setCondCodeAction(ISD::SETUGT, MVT::f64, Expand);
   }
 
   if (Subtarget.hasMips64r6()) {
     // MIPS64r6 replaces the accumulator-based multiplies with a three register
     // instruction
+    setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
+    setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
     setOperationAction(ISD::MUL, MVT::i64, Legal);
     setOperationAction(ISD::MULHS, MVT::i64, Legal);
     setOperationAction(ISD::MULHU, MVT::i64, Legal);
 
     // MIPS32r6 replaces the accumulator-based division/remainder with separate
     // three register division and remainder instructions.
     setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
     setOperationAction(ISD::UDIVREM, MVT::i64, Expand);

llvm/trunk/test/CodeGen/Mips/llvm-ir/add.ll

; ALL-LABEL: add_i64:
   ; GP32:       addu    $[[T1:[0-9]+]], $[[T0]], $6
   ; GP32:       addu    $2, $4, $[[T1]]
 
   ; GP64:       daddu   $2, $4, $5
 
   %r = add i64 %a, %b
   ret i64 %r
 }
+
+define signext i128 @add_i128(i128 signext %a, i128 signext %b) {
+entry:
+; ALL-LABEL: add_i128:
+
+  ; GP32:       lw        $[[T0:[0-9]+]], 28($sp)
+  ; GP32:       addu      $[[T1:[0-9]+]], $7, $[[T0]]
+  ; GP32:       sltu      $[[T2:[0-9]+]], $[[T1]], $[[T0]]
+  ; GP32:       lw        $[[T3:[0-9]+]], 24($sp)
+  ; GP32:       addu      $[[T4:[0-9]+]], $[[T2]], $[[T3]]
+  ; GP32:       addu      $[[T5:[0-9]+]], $6, $[[T4]]
+  ; GP32:       sltu      $[[T6:[0-9]+]], $[[T5]], $[[T3]]
+  ; GP32:       lw        $[[T7:[0-9]+]], 20($sp)
+  ; GP32:       addu      $[[T8:[0-9]+]], $[[T6]], $[[T7]]
+  ; GP32:       lw        $[[T9:[0-9]+]], 16($sp)
+  ; GP32:       addu      $3, $5, $[[T8]]
+  ; GP32:       sltu      $[[T10:[0-9]+]], $3, $[[T7]]
+  ; GP32:       addu      $[[T11:[0-9]+]], $[[T10]], $[[T9]]
+  ; GP32:       addu      $2, $4, $[[T11]]
+  ; GP32:       move      $4, $[[T5]]
+  ; GP32:       move      $5, $[[T1]]
+
+  ; GP64:       daddu     $3, $5, $7
+  ; GP64:       sltu      $[[T0:[0-9]+]], $3, $7
+  ; GP64:       daddu     $[[T1:[0-9]+]], $[[T0]], $6
+  ; GP64:       daddu     $2, $4, $[[T1]]
+
+  %r = add i128 %a, %b
+  ret i128 %r
+}

llvm/trunk/test/CodeGen/Mips/llvm-ir/and.ll

; ALL-LABEL: and_i64:
   ; GP32:         and     $2, $4, $6
   ; GP32:         and     $3, $5, $7
 
   ; GP64:         and     $2, $4, $5
 
   %r = and i64 %a, %b
   ret i64 %r
 }
+
+define signext i128 @and_i128(i128 signext %a, i128 signext %b) {
+entry:
+; ALL-LABEL: and_i128:
+
+  ; GP32:         lw      $[[T0:[0-9]+]], 24($sp)
+  ; GP32:         lw      $[[T1:[0-9]+]], 20($sp)
+  ; GP32:         lw      $[[T2:[0-9]+]], 16($sp)
+  ; GP32:         and     $2, $4, $[[T2]]
+  ; GP32:         and     $3, $5, $[[T1]]
+  ; GP32:         and     $4, $6, $[[T0]]
+  ; GP32:         lw      $[[T3:[0-9]+]], 28($sp)
+  ; GP32:         and     $5, $7, $[[T3]]
+
+  ; GP64:         and     $2, $4, $6
+  ; GP64:         and     $3, $5, $7
+
+  %r = and i128 %a, %b
+  ret i128 %r
+}

llvm/trunk/test/CodeGen/Mips/llvm-ir/ashr.ll

; ALL-LABEL: ashr_i64:
 
   ; FIXME: The sll instruction below is redundant.
   ; GP64:       sll       $[[T0:[0-9]+]], $5, 0
   ; GP64:       dsrav     $2, $4, $[[T0]]
 
   %r = ashr i64 %a, %b
   ret i64 %r
 }
+
+define signext i128 @ashr_i128(i128 signext %a, i128 signext %b) {
+entry:
+; ALL-LABEL: ashr_i128:
+
+  ; GP32:           lw        $25, %call16(__ashrti3)($gp)
+
+  ; M3:             sll       $[[T0:[0-9]+]], $7, 0
+  ; M3:             dsrav     $[[T1:[0-9]+]], $4, $[[T0]]
+  ; M3:             andi      $[[T2:[0-9]+]], $[[T0]], 32
+  ; M3:             bnez      $[[T3:[0-9]+]], $[[BB0:BB[0-9_]+]]
+  ; M3:             move      $3, $[[T1]]
+  ; M3:             dsrlv     $[[T4:[0-9]+]], $5, $[[T0]]
+  ; M3:             dsll      $[[T5:[0-9]+]], $4, 1
+  ; M3:             not       $[[T6:[0-9]+]], $[[T0]]
+  ; M3:             dsllv     $[[T7:[0-9]+]], $[[T5]], $[[T6]]
+  ; M3:             or        $3, $[[T7]], $[[T4]]
+  ; M3:             $[[BB0]]:
+  ; M3:             beqz      $[[T3]], $[[BB1:BB[0-9_]+]]
+  ; M3:             nop
+  ; M3:             dsra      $2, $4, 31
+  ; M3:             $[[BB1]]:
+  ; M3:             jr        $ra
+  ; M3:             nop
+
+  ; GP64-NOT-R6:    sll       $[[T0:[0-9]+]], $7, 0
+  ; GP64-NOT-R6:    dsrlv     $[[T1:[0-9]+]], $5, $[[T0]]
+  ; GP64-NOT-R6:    dsll      $[[T2:[0-9]+]], $4, 1
+  ; GP64-NOT-R6:    not       $[[T3:[0-9]+]], $[[T0]]
+  ; GP64-NOT-R6:    dsllv     $[[T4:[0-9]+]], $[[T2]], $[[T3]]
+  ; GP64-NOT-R6:    or        $3, $[[T4]], $[[T1]]
+  ; GP64-NOT-R6:    dsrav     $2, $4, $[[T0]]
+  ; GP64-NOT-R6:    andi      $[[T5:[0-9]+]], $[[T0]], 32
+
+  ; GP64-NOT-R6:    movn      $3, $2, $[[T5]]
+  ; GP64-NOT-R6:    dsra      $[[T6:[0-9]+]], $4, 31
+  ; GP64-NOT-R6:    jr        $ra
+  ; GP64-NOT-R6:    movn      $2, $[[T6]], $[[T5]]
+
+  ; 64R6:           sll       $[[T0:[0-9]+]], $7, 0
+  ; 64R6:           dsrav     $[[T1:[0-9]+]], $4, $[[T0]]
+  ; 64R6:           andi      $[[T2:[0-9]+]], $[[T0]], 32
+  ; 64R6:           sll       $[[T3:[0-9]+]], $[[T2]], 0
+  ; 64R6:           seleqz    $[[T4:[0-9]+]], $[[T1]], $[[T3]]
+  ; 64R6:           dsra      $[[T5:[0-9]+]], $4, 31
+  ; 64R6:           selnez    $[[T6:[0-9]+]], $[[T5]], $[[T3]]
+  ; 64R6:           or        $2, $[[T6]], $[[T4]]
+  ; 64R6:           dsrlv     $[[T7:[0-9]+]], $5, $[[T0]]
+  ; 64R6:           dsll      $[[T8:[0-9]+]], $4, 1
+  ; 64R6:           not       $[[T9:[0-9]+]], $[[T0]]
+  ; 64R6:           dsllv     $[[T10:[0-9]+]], $[[T8]], $[[T9]]
+  ; 64R6:           or        $[[T11:[0-9]+]], $[[T10]], $[[T7]]
+  ; 64R6:           seleqz    $[[T12:[0-9]+]], $[[T11]], $[[T3]]
+  ; 64R6:           selnez    $[[T13:[0-9]+]], $[[T1]], $[[T3]]
+  ; 64R6:           jr        $ra
+  ; 64R6:           or        $3, $[[T13]], $[[T12]]
+
+  %r = ashr i128 %a, %b
+  ret i128 %r
+}

llvm/trunk/test/CodeGen/Mips/llvm-ir/lshr.ll

; ALL-LABEL: lshr_i64:
   ; 32R6:       seleqz    $2, $[[T7]], $[[T5]]
 
   ; GP64:         sll     $[[T0:[0-9]+]], $5, 0
   ; GP64:         dsrlv   $2, $4, $[[T0]]
 
   %r = lshr i64 %a, %b
   ret i64 %r
 }
+
+define signext i128 @lshr_i128(i128 signext %a, i128 signext %b) {
+entry:
+; ALL-LABEL: lshr_i128:
+
+  ; GP32:         lw      $25, %call16(__lshrti3)($gp)
+
+  ; M3:             sll       $[[T0:[0-9]+]], $7, 0
+  ; M3:             dsrlv     $[[T1:[0-9]+]], $4, $[[T0]]
+  ; M3:             andi      $[[T2:[0-9]+]], $[[T0]], 32
+  ; M3:             bnez      $[[T3:[0-9]+]], $[[BB0:BB[0-9_]+]]
+  ; M3:             move      $3, $[[T1]]
+  ; M3:             dsrlv     $[[T4:[0-9]+]], $5, $[[T0]]
+  ; M3:             dsll      $[[T5:[0-9]+]], $4, 1
+  ; M3:             not       $[[T6:[0-9]+]], $[[T0]]
+  ; M3:             dsllv     $[[T7:[0-9]+]], $[[T5]], $[[T6]]
+  ; M3:             or        $3, $[[T7]], $[[T4]]
+  ; M3:             $[[BB0]]:
+  ; M3:             bnez      $[[T3]], $[[BB1:BB[0-9_]+]]
+  ; M3:             daddiu    $2, $zero, 0
+  ; M3:             move      $2, $[[T1]]
+  ; M3:             $[[BB1]]:
+  ; M3:             jr        $ra
+  ; M3:             nop
+
+  ; GP64-NOT-R6:    sll       $[[T0:[0-9]+]], $7, 0
+  ; GP64-NOT-R6:    dsrlv     $[[T1:[0-9]+]], $5, $[[T0]]
+  ; GP64-NOT-R6:    dsll      $[[T2:[0-9]+]], $4, 1
+  ; GP64-NOT-R6:    not       $[[T3:[0-9]+]], $[[T0]]
+  ; GP64-NOT-R6:    dsllv     $[[T4:[0-9]+]], $[[T2]], $[[T3]]
+  ; GP64-NOT-R6:    or        $3, $[[T4]], $[[T1]]
+  ; GP64-NOT-R6:    dsrlv     $2, $4, $[[T0]]
+  ; GP64-NOT-R6:    andi      $[[T5:[0-9]+]], $[[T0]], 32
+  ; GP64-NOT-R6:    movn      $3, $2, $[[T5]]
+  ; GP64-NOT-R6:    jr        $ra
+  ; GP64-NOT-R6:    movn      $2, $zero, $1
+
+  ; 64R6:           sll       $[[T0:[0-9]+]], $7, 0
+  ; 64R6:           dsrlv     $[[T1:[0-9]+]], $5, $[[T0]]
+  ; 64R6:           dsll      $[[T2:[0-9]+]], $4, 1
+  ; 64R6:           not       $[[T3:[0-9]+]], $[[T0]]
+  ; 64R6:           dsllv     $[[T4:[0-9]+]], $[[T2]], $[[T3]]
+  ; 64R6:           or        $[[T5:[0-9]+]], $[[T4]], $[[T1]]
+  ; 64R6:           andi      $[[T6:[0-9]+]], $[[T0]], 32
+  ; 64R6:           sll       $[[T7:[0-9]+]], $[[T6]], 0
+  ; 64R6:           seleqz    $[[T8:[0-9]+]], $[[T5]], $[[T7]]
+  ; 64R6:           dsrlv     $[[T9:[0-9]+]], $4, $[[T0]]
+  ; 64R6:           selnez    $[[T10:[0-9]+]], $[[T9]], $[[T7]]
+  ; 64R6:           or        $3, $[[T10]], $[[T8]]
+  ; 64R6:           jr        $ra
+  ; 64R6:           seleqz    $2, $[[T0]], $[[T7]]
+
+  %r = lshr i128 %a, %b
+  ret i128 %r
+}

llvm/trunk/test/CodeGen/Mips/llvm-ir/mul.ll

-; RUN: llc < %s -march=mips -mcpu=mips2 | FileCheck %s \
-; RUN:    -check-prefix=ALL -check-prefix=M2
-; RUN: llc < %s -march=mips -mcpu=mips32 | FileCheck %s \
-; RUN:    -check-prefix=ALL -check-prefix=32R1-R2 -check-prefix=32R1
-; RUN: llc < %s -march=mips -mcpu=mips32r2 | FileCheck %s \
-; RUN:    -check-prefix=ALL -check-prefix=32R1-R2 -check-prefix=32R2
-; RUN: llc < %s -march=mips -mcpu=mips32r6 | FileCheck %s \
-; RUN:    -check-prefix=ALL -check-prefix=32R6
-; RUN: llc < %s -march=mips64 -mcpu=mips4 | FileCheck %s \
-; RUN:    -check-prefix=ALL -check-prefix=M4
-; RUN: llc < %s -march=mips64 -mcpu=mips64 | FileCheck %s \
-; RUN:    -check-prefix=ALL -check-prefix=64R1-R2
-; RUN: llc < %s -march=mips64 -mcpu=mips64r2 | FileCheck %s \
-; RUN:    -check-prefix=ALL -check-prefix=64R1-R2
-; RUN: llc < %s -march=mips64 -mcpu=mips64r6 | FileCheck %s \
-; RUN:     -check-prefix=ALL -check-prefix=64R6
+; RUN: llc < %s -march=mips -mcpu=mips2 | FileCheck %s -check-prefix=ALL \
+; RUN:    -check-prefix=M2 -check-prefix=GP32
+; RUN: llc < %s -march=mips -mcpu=mips32 | FileCheck %s -check-prefix=ALL \
+; RUN:    -check-prefix=32R1-R2 -check-prefix=GP32
+; RUN: llc < %s -march=mips -mcpu=mips32r2 | FileCheck %s -check-prefix=ALL \
+; RUN:    -check-prefix=32R1-R2 -check-prefix=32R2 -check-prefix=GP32
+; RUN: llc < %s -march=mips -mcpu=mips32r6 | FileCheck %s -check-prefix=ALL \
+; RUN:    -check-prefix=32R6 -check-prefix=GP32
+; RUN: llc < %s -march=mips64 -mcpu=mips4 | FileCheck %s -check-prefix=ALL \
+; RUN:    -check-prefix=M4 -check-prefix=GP64-NOT-R6
+; RUN: llc < %s -march=mips64 -mcpu=mips64 | FileCheck %s -check-prefix=ALL \
+; RUN:    -check-prefix=64R1-R2 -check-prefix=GP64-NOT-R6
+; RUN: llc < %s -march=mips64 -mcpu=mips64r2 | FileCheck %s -check-prefix=ALL \
+; RUN:    -check-prefix=64R1-R2 -check-prefix=GP64 -check-prefix=GP64-NOT-R6
+; RUN: llc < %s -march=mips64 -mcpu=mips64r6 | FileCheck %s -check-prefix=ALL \
+; RUN:    -check-prefix=64R6
 
 define signext i1 @mul_i1(i1 signext %a, i1 signext %b) {
 entry:
 ; ALL-LABEL: mul_i1:
 
   ; M2:         mult    $4, $5
   ; M2:         mflo    $[[T0:[0-9]+]]
   ; M2:         sll     $[[T0]], $[[T0]], 31
; ALL-LABEL: mul_i64:
   ; 64R1-R2:    dmult   $4, $5
   ; 64R1-R2:    mflo    $2
 
   ; 64R6:       dmul    $2, $4, $5
 
   %r = mul i64 %a, %b
   ret i64 %r
 }
+
+define signext i128 @mul_i128(i128 signext %a, i128 signext %b) {
+entry:
+; ALL-LABEL: mul_i128:
+
+  ; GP32:           lw      $25, %call16(__multi3)($gp)
+
+  ; GP64-NOT-R6:    dmult   $4, $7
+  ; GP64-NOT-R6:    mflo    $[[T0:[0-9]+]]
+  ; GP64-NOT-R6:    dmult   $5, $6
+  ; GP64-NOT-R6:    mflo    $[[T1:[0-9]+]]
+  ; GP64-NOT-R6:    dmultu  $5, $7
+  ; GP64-NOT-R6:    mflo    $3
+  ; GP64-NOT-R6:    mfhi    $[[T2:[0-9]+]]
+  ; GP64-NOT-R6:    daddu   $[[T3:[0-9]+]], $[[T2]], $[[T1]]
+  ; GP64-NOT-R6:    daddu   $2, $[[T3:[0-9]+]], $[[T0]]
+
+  ; 64R6:           dmul    $[[T0:[0-9]+]], $5, $6
+  ; 64R6:           dmuhu   $[[T1:[0-9]+]], $5, $7
+  ; 64R6:           daddu   $[[T2:[0-9]+]], $[[T1]], $[[T0]]
+  ; 64R6:           dmul    $[[T3:[0-9]+]], $4, $7
+  ; 64R6:           daddu   $2, $[[T2]], $[[T3]]
+  ; 64R6:           dmul    $3, $5, $7
+
+  %r = mul i128 %a, %b
+  ret i128 %r
+}

llvm/trunk/test/CodeGen/Mips/llvm-ir/or.ll

; ALL-LABEL: or_i64:
   ; GP32:         or     $2, $4, $6
   ; GP32:         or     $3, $5, $7
 
   ; GP64:         or     $2, $4, $5
 
   %r = or i64 %a, %b
   ret i64 %r
 }
+
+define signext i128 @or_i128(i128 signext %a, i128 signext %b) {
+entry:
+; ALL-LABEL: or_i128:
+
+  ; GP32:         lw     $[[T0:[0-9]+]], 24($sp)
+  ; GP32:         lw     $[[T1:[0-9]+]], 20($sp)
+  ; GP32:         lw     $[[T2:[0-9]+]], 16($sp)
+  ; GP32:         or     $2, $4, $[[T2]]
+  ; GP32:         or     $3, $5, $[[T1]]
+  ; GP32:         or     $4, $6, $[[T0]]
+  ; GP32:         lw     $[[T3:[0-9]+]], 28($sp)
+  ; GP32:         or     $5, $7, $[[T3]]
+
+  ; GP64:         or     $2, $4, $6
+  ; GP64:         or     $3, $5, $7
+
+  %r = or i128 %a, %b
+  ret i128 %r
+}

llvm/trunk/test/CodeGen/Mips/llvm-ir/sdiv.ll

; ALL-LABEL: sdiv_i64:
   ; GP64-NOT-R6:  mflo    $2
 
   ; 64R6:         ddiv    $2, $4, $5
   ; 64R6:         teq     $5, $zero, 7
 
   %r = sdiv i64 %a, %b
   ret i64 %r
 }
+
+define signext i128 @sdiv_i128(i128 signext %a, i128 signext %b) {
+entry:
+  ; ALL-LABEL: sdiv_i128:
+
+    ; GP32:         lw      $25, %call16(__divti3)($gp)
+
+    ; GP64-NOT-R6:  ld      $25, %call16(__divti3)($gp)
+    ; 64R6:         ld      $25, %call16(__divti3)($gp)
+
+    %r = sdiv i128 %a, %b
+    ret i128 %r
+}

llvm/trunk/test/CodeGen/Mips/llvm-ir/shl.ll

; ALL-LABEL: shl_i64:
   ; 32R6:       seleqz    $3, $[[T7]], $[[T5]]
 
   ; GP64:       sll       $[[T0:[0-9]+]], $5, 0
   ; GP64:       dsllv     $2, $4, $1
 
   %r = shl i64 %a, %b
   ret i64 %r
 }
+
+define signext i128 @shl_i128(i128 signext %a, i128 signext %b) {
+entry:
+; ALL-LABEL: shl_i128:
+
+  ; GP32:           lw        $25, %call16(__ashlti3)($gp)
+
+  ; M3:             sll       $[[T0:[0-9]+]], $7, 0
+  ; M3:             dsllv     $[[T1:[0-9]+]], $5, $[[T0]]
+  ; M3:             andi      $[[T2:[0-9]+]], $[[T0]], 32
+  ; M3:             bnez      $[[T3:[0-9]+]], $[[BB0:BB[0-9_]+]]
+  ; M3:             move      $2, $[[T1]]
+  ; M3:             dsllv     $[[T4:[0-9]+]], $4, $[[T0]]
+  ; M3:             dsrl      $[[T5:[0-9]+]], $5, 1
+  ; M3:             not       $[[T6:[0-9]+]], $[[T0]]
+  ; M3:             dsrlv     $[[T7:[0-9]+]], $[[T5]], $[[T6]]
+  ; M3:             or        $2, $[[T4]], $[[T7]]
+  ; M3:             $[[BB0]]:
+  ; M3:             bnez      $[[T3]], $[[BB1:BB[0-9_]+]]
+  ; M3:             daddiu    $3, $zero, 0
+  ; M3:             move      $3, $[[T1]]
+  ; M3:             $[[BB1]]:
+  ; M3:             jr        $ra
+  ; M3:             nop
+
+  ; GP64-NOT-R6:    sll       $[[T0:[0-9]+]], $7, 0
+  ; GP64-NOT-R6:    dsllv     $[[T1:[0-9]+]], $4, $[[T0]]
+  ; GP64-NOT-R6:    dsrl      $[[T2:[0-9]+]], $5, 1
+  ; GP64-NOT-R6:    not       $[[T3:[0-9]+]], $[[T0]]
+  ; GP64-NOT-R6:    dsrlv     $[[T4:[0-9]+]], $[[T2]], $[[T3]]
+  ; GP64-NOT-R6:    or        $2, $[[T1]], $[[T4]]
+  ; GP64-NOT-R6:    dsllv     $3, $5, $[[T0]]
+  ; GP64-NOT-R6:    andi      $[[T5:[0-9]+]], $[[T0]], 32
+  ; GP64-NOT-R6:    movn      $2, $3, $[[T5]]
+  ; GP64-NOT-R6:    jr        $ra
+  ; GP64-NOT-R6:    movn      $3, $zero, $1
+
+  ; 64R6:           sll       $[[T0:[0-9]+]], $7, 0
+  ; 64R6:           dsllv     $[[T1:[0-9]+]], $4, $[[T0]]
+  ; 64R6:           dsrl      $[[T2:[0-9]+]], $5, 1
+  ; 64R6:           not       $[[T3:[0-9]+]], $[[T0]]
+  ; 64R6:           dsrlv     $[[T4:[0-9]+]], $[[T2]], $[[T3]]
+  ; 64R6:           or        $[[T5:[0-9]+]], $[[T1]], $[[T4]]
+  ; 64R6:           andi      $[[T6:[0-9]+]], $[[T0]], 32
+  ; 64R6:           sll       $[[T7:[0-9]+]], $[[T6]], 0
+  ; 64R6:           seleqz    $[[T8:[0-9]+]], $[[T5]], $[[T7]]
+  ; 64R6:           dsllv     $[[T9:[0-9]+]], $5, $[[T0]]
+  ; 64R6:           selnez    $[[T10:[0-9]+]], $[[T9]], $[[T7]]
+  ; 64R6:           or        $2, $[[T10]], $[[T8]]
+  ; 64R6:           jr        $ra
+  ; 64R6:           seleqz    $3, $[[T0]], $[[T7]]
+
+  %r = shl i128 %a, %b
+  ret i128 %r
+}

llvm/trunk/test/CodeGen/Mips/llvm-ir/srem.ll

; ALL-LABEL: srem_i64:
   ; GP64-NOT-R6:  mfhi    $2
 
   ; 64R6:         dmod    $2, $4, $5
   ; 64R6:         teq     $5, $zero, 7
 
   %r = srem i64 %a, %b
   ret i64 %r
 }
+
+define signext i128 @srem_i128(i128 signext %a, i128 signext %b) {
+entry:
+; ALL-LABEL: srem_i128:
+
+  ; GP32:         lw      $25, %call16(__modti3)($gp)
+
+  ; GP64-NOT-R6:  ld      $25, %call16(__modti3)($gp)
+  ; 64-R6:        ld      $25, %call16(__modti3)($gp)
+
+  %r = srem i128 %a, %b
+  ret i128 %r
+}

llvm/trunk/test/CodeGen/Mips/llvm-ir/sub.ll

; ALL-LABEL: sub_i64:
   ; GP32:           addu    $[[T1:[0-9]+]], $[[T0]], $6
   ; GP32:           subu    $2, $4, $[[T1]]
 
   ; GP64:           dsubu   $2, $4, $5
 
   %r = sub i64 %a, %b
   ret i64 %r
 }
+
+define signext i128 @sub_i128(i128 signext %a, i128 signext %b) {
+entry:
+; ALL-LABEL: sub_i128:
+
+  ; GP32:       lw        $[[T0:[0-9]+]], 20($sp)
+  ; GP32:       sltu      $[[T1:[0-9]+]], $5, $[[T0]]
+  ; GP32:       lw        $[[T2:[0-9]+]], 16($sp)
+  ; GP32:       addu      $[[T3:[0-9]+]], $[[T1]], $[[T2]]
+  ; GP32:       lw        $[[T4:[0-9]+]], 24($sp)
+  ; GP32:       lw        $[[T5:[0-9]+]], 28($sp)
+  ; GP32:       subu      $[[T6:[0-9]+]], $7, $[[T5]]
+  ; GP32:       subu      $2, $4, $[[T3]]
+  ; GP32:       sltu      $[[T8:[0-9]+]], $6, $[[T4]]
+  ; GP32:       addu      $[[T9:[0-9]+]], $[[T8]], $[[T0]]
+  ; GP32:       subu      $3, $5, $[[T9]]
+  ; GP32:       sltu      $[[T10:[0-9]+]], $7, $[[T5]]
+  ; GP32:       addu      $[[T11:[0-9]+]], $[[T10]], $[[T4]]
+  ; GP32:       subu      $4, $6, $[[T11]]
+  ; GP32:       move      $5, $[[T6]]
+
+  ; GP64:       dsubu     $3, $5, $7
+  ; GP64:       sltu      $[[T0:[0-9]+]], $5, $7
+  ; GP64:       daddu     $[[T1:[0-9]+]], $[[T0]], $6
+  ; GP64:       dsubu     $2, $4, $[[T1]]
+
+  %r = sub i128 %a, %b
+  ret i128 %r
+}

llvm/trunk/test/CodeGen/Mips/llvm-ir/udiv.ll

; ALL-LABEL: udiv_i64:
   ; GP64-NOT-R6:  mflo    $2
 
   ; 64R6:         ddivu   $2, $4, $5
   ; 64R6:         teq     $5, $zero, 7
 
   %r = udiv i64 %a, %b
   ret i64 %r
 }
+
+define signext i128 @udiv_i128(i128 signext %a, i128 signext %b) {
+entry:
+; ALL-LABEL: udiv_i128:
+
+  ; GP32:         lw      $25, %call16(__udivti3)($gp)
+
+  ; GP64-NOT-R6:  ld      $25, %call16(__udivti3)($gp)
+  ; 64-R6:        ld      $25, %call16(__udivti3)($gp)
+
+  %r = udiv i128 %a, %b
+  ret i128 %r
+}

llvm/trunk/test/CodeGen/Mips/llvm-ir/urem.ll

; ALL-LABEL: urem_i64:
   ; GP64-NOT-R6:  mfhi    $2
 
   ; 64R6:         dmodu   $2, $4, $5
   ; 64R6:         teq     $5, $zero, 7
 
   %r = urem i64 %a, %b
   ret i64 %r
 }
+
+define signext i128 @urem_i128(i128 signext %a, i128 signext %b) {
+entry:
+  ; ALL-LABEL: urem_i128:
+
+    ; GP32:         lw      $25, %call16(__umodti3)($gp)
+
+    ; GP64-NOT-R6:  ld      $25, %call16(__umodti3)($gp)
+    ; 64-R6:        ld      $25, %call16(__umodti3)($gp)
+
+    %r = urem i128 %a, %b
+    ret i128 %r
+}

llvm/trunk/test/CodeGen/Mips/llvm-ir/xor.ll

; ALL-LABEL: xor_i64:
   ; GP32:         xor     $2, $4, $6
   ; GP32:         xor     $3, $5, $7
 
   ; GP64:         xor     $2, $4, $5
 
   %r = xor i64 %a, %b
   ret i64 %r
 }
+
+define signext i128 @xor_i128(i128 signext %a, i128 signext %b) {
+entry:
+; ALL-LABEL: xor_i128:
+
+  ; GP32:         lw      $[[T0:[0-9]+]], 24($sp)
+  ; GP32:         lw      $[[T1:[0-9]+]], 20($sp)
+  ; GP32:         lw      $[[T2:[0-9]+]], 16($sp)
+  ; GP32:         xor     $2, $4, $[[T2]]
+  ; GP32:         xor     $3, $5, $[[T1]]
+  ; GP32:         xor     $4, $6, $[[T0]]
+  ; GP32:         lw      $[[T3:[0-9]+]], 28($sp)
+  ; GP32:         xor     $5, $7, $[[T3]]
+
+  ; GP64:         xor     $2, $4, $6
+  ; GP64:         xor     $3, $5, $7
+
+  %r = xor i128 %a, %b
+  ret i128 %r
+}