Index: lib/Target/X86/X86ISelDAGToDAG.cpp =================================================================== --- lib/Target/X86/X86ISelDAGToDAG.cpp +++ lib/Target/X86/X86ISelDAGToDAG.cpp @@ -2412,11 +2412,14 @@ } case ISD::SDIVREM: - case ISD::UDIVREM: { + case ISD::UDIVREM: + case X86ISD::SDIVREM8_SEXT_HREG: + case X86ISD::UDIVREM8_ZEXT_HREG: { SDValue N0 = Node->getOperand(0); SDValue N1 = Node->getOperand(1); - bool isSigned = Opcode == ISD::SDIVREM; + bool isSigned = (Opcode == ISD::SDIVREM || + Opcode == X86ISD::SDIVREM8_SEXT_HREG); if (!isSigned) { switch (NVT.SimpleTy) { default: llvm_unreachable("Unsupported VT!"); @@ -2532,33 +2535,43 @@ SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, N1, InFlag), 0); } - // Prevent use of AH in a REX instruction by referencing AX instead. - // Shift it down 8 bits. + // Prevent use of AH in a REX instruction by explicitly copying it to + // an ABCD_L register. // // The current assumption of the register allocator is that isel - // won't generate explicit references to the GPR8_NOREX registers. If + // won't generate explicit references to the GR8_ABCD_H registers. If // the allocator and/or the backend get enhanced to be more robust in // that regard, this can be, and should be, removed. - if (HiReg == X86::AH && Subtarget->is64Bit() && - !SDValue(Node, 1).use_empty()) { - SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, - X86::AX, MVT::i16, InFlag); - InFlag = Result.getValue(2); + if (HiReg == X86::AH && !SDValue(Node, 1).use_empty()) { + SDValue AHCopy = CurDAG->getRegister(X86::AH, MVT::i8); + unsigned AHExtOpcode = + isSigned ? X86::MOVSX32_NOREXrr8 : X86::MOVZX32_NOREXrr8; - // If we also need AL (the quotient), get it by extracting a subreg from - // Result. The fast register allocator does not like multiple CopyFromReg - // nodes using aliasing registers. - if (!SDValue(Node, 0).use_empty()) - ReplaceUses(SDValue(Node, 0), - CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); + SDNode *RNode = CurDAG->getMachineNode(AHExtOpcode, dl, MVT::i32, + MVT::Glue, AHCopy, InFlag); + SDValue Result(RNode, 0); + InFlag = SDValue(RNode, 1); - // Shift AX right by 8 bits instead of using AH. - Result = SDValue(CurDAG->getMachineNode(X86::SHR16ri, dl, MVT::i16, - Result, - CurDAG->getTargetConstant(8, MVT::i8)), - 0); - ReplaceUses(SDValue(Node, 1), - CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); + if (Opcode == X86ISD::UDIVREM8_ZEXT_HREG || + Opcode == X86ISD::SDIVREM8_SEXT_HREG) { + if (Node->getValueType(1) == MVT::i64) { + // It's not possible to directly movsx AH to a 64bit register, because + // the latter needs the REX prefix, but the former can't have it. + assert(Opcode != X86ISD::SDIVREM8_SEXT_HREG && + "Unexpected i64 sext of h-register"); + Result = + SDValue(CurDAG->getMachineNode( + TargetOpcode::SUBREG_TO_REG, dl, MVT::i64, + CurDAG->getTargetConstant(0, MVT::i64), Result, + CurDAG->getTargetConstant(X86::sub_32bit, MVT::i32)), + 0); + } + } else { + Result = + CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result); + } + ReplaceUses(SDValue(Node, 1), Result); + DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); } // Copy the division (low) result, if it is needed. if (!SDValue(Node, 0).use_empty()) { Index: lib/Target/X86/X86ISelLowering.h =================================================================== --- lib/Target/X86/X86ISelLowering.h +++ lib/Target/X86/X86ISelLowering.h @@ -304,6 +304,10 @@ // 8-bit SMUL/UMUL - AX, FLAGS = smul8/umul8 AL, RHS SMUL8, UMUL8, + // 8-bit divrem that zero-extend the high result (AH). + UDIVREM8_ZEXT_HREG, + SDIVREM8_SEXT_HREG, + // MUL_IMM - X86 specific multiply by immediate. MUL_IMM, Index: lib/Target/X86/X86ISelLowering.cpp =================================================================== --- lib/Target/X86/X86ISelLowering.cpp +++ lib/Target/X86/X86ISelLowering.cpp @@ -19069,6 +19069,8 @@ case X86ISD::SBB: return "X86ISD::SBB"; case X86ISD::SMUL: return "X86ISD::SMUL"; case X86ISD::UMUL: return "X86ISD::UMUL"; + case X86ISD::SDIVREM8_SEXT_HREG: return "X86ISD::SDIVREM8_SEXT_HREG"; + case X86ISD::UDIVREM8_ZEXT_HREG: return "X86ISD::UDIVREM8_ZEXT_HREG"; case X86ISD::INC: return "X86ISD::INC"; case X86ISD::DEC: return "X86ISD::DEC"; case X86ISD::OR: return "X86ISD::OR"; @@ -24267,13 +24269,29 @@ static SDValue PerformSExtCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) { + SDValue N0 = N->getOperand(0); + EVT VT = N->getValueType(0); + + // (i8,i32 sext (sdivrem (i8 x, i8 y)) -> + // (i8,i32 (sdivrem_sext_hreg (i8 x, i8 y) + // This exposes the sext to the sdivrem lowering, so that it directly extends + // from AH (which we otherwise need to do contortions to access). + if (N0.getOpcode() == ISD::SDIVREM && N0.getResNo() == 1 && + N0.getValueType() == MVT::i8 && VT == MVT::i32) { + SDLoc dl(N); + SDVTList NodeTys = DAG.getVTList(MVT::i8, VT); + SDValue R = DAG.getNode(X86ISD::SDIVREM8_SEXT_HREG, dl, NodeTys, + N0.getOperand(0), N0.getOperand(1)); + DAG.ReplaceAllUsesOfValueWith(N0.getValue(0), R.getValue(0)); + return R.getValue(1); + } + if (!DCI.isBeforeLegalizeOps()) return SDValue(); if (!Subtarget->hasFp256()) return SDValue(); - EVT VT = N->getValueType(0); if (VT.isVector() && VT.getSizeInBits() == 256) { SDValue R = WidenMaskArithmetic(N, DAG, DCI, Subtarget); if (R.getNode()) @@ -24366,6 +24384,20 @@ return R; } + // (i8,i32 zext (udivrem (i8 x, i8 y)) -> + // (i8,i32 (udivrem_zext_hreg (i8 x, i8 y) + // This exposes the zext to the udivrem lowering, so that it directly extends + // from AH (which we otherwise need to do contortions to access). + if (N0.getOpcode() == ISD::UDIVREM && + N0.getResNo() == 1 && N0.getValueType() == MVT::i8 && + (VT == MVT::i32 || VT == MVT::i64)) { + SDVTList NodeTys = DAG.getVTList(MVT::i8, VT); + SDValue R = DAG.getNode(X86ISD::UDIVREM8_ZEXT_HREG, dl, NodeTys, + N0.getOperand(0), N0.getOperand(1)); + DAG.ReplaceAllUsesOfValueWith(N0.getValue(0), R.getValue(0)); + return R.getValue(1); + } + return SDValue(); } Index: lib/Target/X86/X86InstrExtension.td =================================================================== --- lib/Target/X86/X86InstrExtension.td +++ lib/Target/X86/X86InstrExtension.td @@ -97,13 +97,23 @@ let neverHasSideEffects = 1, isCodeGenOnly = 1 in { def MOVZX32_NOREXrr8 : I<0xB6, MRMSrcReg, (outs GR32_NOREX:$dst), (ins GR8_NOREX:$src), - "movz{bl|x}\t{$src, $dst|$dst, $src}", + "movz{bl|x}\t{$src, $dst|$dst, $src} # NOREX", [], IIC_MOVZX>, TB, Sched<[WriteALU]>; let mayLoad = 1 in def MOVZX32_NOREXrm8 : I<0xB6, MRMSrcMem, (outs GR32_NOREX:$dst), (ins i8mem_NOREX:$src), - "movz{bl|x}\t{$src, $dst|$dst, $src}", + "movz{bl|x}\t{$src, $dst|$dst, $src} # NOREX", [], IIC_MOVZX>, TB, Sched<[WriteALULd]>; + +def MOVSX32_NOREXrr8 : I<0xBE, MRMSrcReg, + (outs GR32_NOREX:$dst), (ins GR8_NOREX:$src), + "movs{bl|x}\t{$src, $dst|$dst, $src} # NOREX", + [], IIC_MOVSX>, TB, Sched<[WriteALU]>; +let mayLoad = 1 in +def MOVSX32_NOREXrm8 : I<0xBE, MRMSrcMem, + (outs GR32_NOREX:$dst), (ins i8mem_NOREX:$src), + "movs{bl|x}\t{$src, $dst|$dst, $src} # NOREX", + [], IIC_MOVSX>, TB, Sched<[WriteALULd]>; } // MOVSX64rr8 always has a REX prefix and it has an 8-bit register Index: test/CodeGen/X86/divrem8_ext.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/divrem8_ext.ll @@ -0,0 +1,106 @@ +; RUN: llc < %s | FileCheck %s +target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.10.0" + +define zeroext i8 @test_udivrem_zext_ah(i8 %x, i8 %y) { +; CHECK-LABEL: test_udivrem_zext_ah +; CHECK: movq _z@GOTPCREL(%rip), [[REG_ZPTR:%[a-z0-9]+]] +; CHECK: movzbl %dil, %eax +; CHECK: divb %sil +; CHECK: movzbl %ah, [[REG_REM:%[a-z0-9]+]] +; CHECK: movb %al, ([[REG_ZPTR]]) +; CHECK: movl [[REG_REM]], %eax +; CHECK: retq + %div = udiv i8 %x, %y + store i8 %div, i8* @z + %1 = urem i8 %x, %y + ret i8 %1 +} + +define zeroext i8 @test_urem_zext_ah(i8 %x, i8 %y) { +; CHECK-LABEL: test_urem_zext_ah +; CHECK: movzbl %dil, %eax +; CHECK: divb %sil +; CHECK: movzbl %ah, %eax +; CHECK: retq + %1 = urem i8 %x, %y + ret i8 %1 +} + +define i8 @test_urem_noext_ah(i8 %x, i8 %y) { +; CHECK-LABEL: test_urem_noext_ah +; CHECK: movzbl %dil, %eax +; CHECK: divb %sil +; CHECK: movzbl %ah, %eax +; CHECK: addb %sil, %al +; CHECK: retq + %1 = urem i8 %x, %y + %2 = add i8 %1, %y + ret i8 %2 +} + +define i64 @test_urem_zext64_ah(i8 %x, i8 %y) { +; CHECK-LABEL: test_urem_zext64_ah +; CHECK: movzbl %dil, %eax +; CHECK: divb %sil +; CHECK: movzbl %ah, %eax +; CHECK: retq + %1 = urem i8 %x, %y + %2 = zext i8 %1 to i64 + ret i64 %2 +} + +define signext i8 @test_sdivrem_sext_ah(i8 %x, i8 %y) { +; CHECK-LABEL: test_sdivrem_sext_ah +; CHECK: movq _z@GOTPCREL(%rip), [[REG_ZPTR:%[a-z0-9]+]] +; CHECK: movb %dil, %al +; CHECK: cbtw +; CHECK: idivb %sil +; CHECK: movsbl %ah, [[REG_REM:%[a-z0-9]+]] +; CHECK: movb %al, ([[REG_ZPTR]]) +; CHECK: movl [[REG_REM]], %eax +; CHECK: retq + %div = sdiv i8 %x, %y + store i8 %div, i8* @z + %1 = srem i8 %x, %y + ret i8 %1 +} + +define signext i8 @test_srem_sext_ah(i8 %x, i8 %y) { +; CHECK-LABEL: test_srem_sext_ah +; CHECK: movb %dil, %al +; CHECK: cbtw +; CHECK: idivb %sil +; CHECK: movsbl %ah, %eax +; CHECK: retq + %1 = srem i8 %x, %y + ret i8 %1 +} + +define i8 @test_srem_noext_ah(i8 %x, i8 %y) { +; CHECK-LABEL: test_srem_noext_ah +; CHECK: movb %dil, %al +; CHECK: cbtw +; CHECK: idivb %sil +; CHECK: movsbl %ah, %eax +; CHECK: addb %sil, %al +; CHECK: retq + %1 = srem i8 %x, %y + %2 = add i8 %1, %y + ret i8 %2 +} + +define i64 @test_srem_sext64_ah(i8 %x, i8 %y) { +; CHECK-LABEL: test_srem_sext64_ah +; CHECK: movb %dil, %al +; CHECK: cbtw +; CHECK: idivb %sil +; CHECK: movsbl %ah, %eax +; CHECK: movsbq %al, %rax +; CHECK: retq + %1 = srem i8 %x, %y + %2 = sext i8 %1 to i64 + ret i64 %2 +} + +@z = external global i8