Index: docs/LangRef.rst =================================================================== --- docs/LangRef.rst +++ docs/LangRef.rst @@ -11880,6 +11880,105 @@ .. _int_overflow: +'``llvm.fshl.*``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +This is an overloaded intrinsic. You can use ``llvm.fshl`` on any +integer bit width or any vector of integer elements. Not all targets +support all bit widths or vector types, however. + +:: + + declare i8 @llvm.fshl.i8 (i8 %a, i8 %b, i8 %c) + declare i67 @llvm.fshl.i67(i67 %a, i67 %b, i67 %c) + declare <2 x i32> @llvm.fshl.v2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) + +Overview: +""""""""" + +The '``llvm.fshl``' family of intrinsic functions performs a funnel shift left: +two values are concatenated, the combined value is shifted left, and the most +significant bits are extracted to produce a result that is the same size as the +original arguments. If the 2 values are identical, this is equivalent to a +rotate left operation. For vector types, the operation occurs for each element +of the vector. + +Arguments: +"""""""""" + +The first two arguments are the values to be concatenated. The third +argument is the shift amount. The arguments may be any integer type or a +vector with integer element type. All arguments and the return value must +have the same type. + +Semantics: """""""""" + +'``llvm.fshl``' extracts a string of bits from the concatenation of two values. +The extracted value is the same size as either of the inputs. The shift amount +is treated as an unsigned value modulo the size of the values. + +Example: +"""""""" + +.. code-block:: text + + %r = call i8 @llvm.fshl.i8(i8 %x, i8 %y, i8 %z) ; %r = i8: extract((concat(%x, %y) << %z), 0, 7) + %r = call i8 @llvm.fshl.i8(i8 255, i8 0, i8 15) ; %r = i8: 128 (0b10000000) + %r = call i8 @llvm.fshl.i8(i8 15, i8 15, i8 3) ; %r = i8: 120 (0b01111000) + +'``llvm.fshr.*``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +This is an overloaded intrinsic. You can use ``llvm.fshr`` on any +integer bit width or any vector of integer elements. Not all targets +support all bit widths or vector types, however. + +:: + + declare i8 @llvm.fshr.i8 (i8 %a, i8 %b, i8 %c) + declare i67 @llvm.fshr.i67(i67 %a, i67 %b, i67 %c) + declare <2 x i32> @llvm.fshr.v2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) + +Overview: +""""""""" + +The '``llvm.fshr``' family of intrinsic functions performs a funnel shift right: +two values are concatenated, the combined value is shifted right, and the least +significant bits are extracted to produce a result that is the same size as the +original arguments. If the 2 values are identical, this is equivalent to a +rotate right operation. For vector types, the operation occurs for each element +of the vector. + +Arguments: +"""""""""" + +The first two arguments are the values to be concatenated. The third +argument is the shift amount. The arguments may be any integer type or a +vector with integer element type. All arguments and the return value must +have the same type. + +Semantics: +"""""""""" + +'``llvm.fshr``' extracts a string of bits from the concatenation of two values. +The extracted value is the same size as either of the inputs. The shift amount +is treated as an unsigned value modulo the size of the values. + +Example: +"""""""" + +.. code-block:: text + + %r = call i8 @llvm.fshr.i8(i8 %x, i8 %y, i8 %z) ; %r = i8: extract((concat(%x, %y) >> %z), 0, 7) + %r = call i8 @llvm.fshr.i8(i8 255, i8 0, i8 15) ; %r = i8: 128 (0b00000001) + %r = call i8 @llvm.fshr.i8(i8 15, i8 15, i8 3) ; %r = i8: 120 (0b11100001) + Arithmetic with Overflow Intrinsics ----------------------------------- Index: include/llvm/IR/Intrinsics.td =================================================================== --- include/llvm/IR/Intrinsics.td +++ include/llvm/IR/Intrinsics.td @@ -577,6 +577,10 @@ def int_ctlz : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_i1_ty]>; def int_cttz : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_i1_ty]>; def int_bitreverse : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>]>; + def int_fshl : Intrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>; + def int_fshr : Intrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>; } //===------------------------ Debugger Intrinsics -------------------------===// Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -5673,6 +5673,28 @@ setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg)); return nullptr; } + case Intrinsic::fshl: + case Intrinsic::fshr: { + bool IsFSHL = Intrinsic == Intrinsic::fshl; + SDValue X = getValue(I.getArgOperand(0)); + SDValue Y = getValue(I.getArgOperand(1)); + SDValue Z = getValue(I.getArgOperand(2)); + EVT VT = X.getValueType(); + + // TODO: When X == Y, this is rotate. Create the node directly if legal. + + // Get the shift amount and inverse shift amount, modulo the bit-width. + SDValue BitWidthC = DAG.getConstant(VT.getScalarSizeInBits(), sdl, VT); + SDValue ShAmt = DAG.getNode(ISD::UREM, sdl, VT, Z, BitWidthC); + SDValue InvShAmt = DAG.getNode(ISD::SUB, sdl, VT, BitWidthC, ShAmt); + + // fshl: (X << Z') | (Y >> (BitWidth - Z')) + // fshr: (X << (BitWidth - Z')) | (Y >> Z') + SDValue ShX = DAG.getNode(ISD::SHL, sdl, VT, X, IsFSHL ? ShAmt : InvShAmt); + SDValue ShY = DAG.getNode(ISD::SRL, sdl, VT, Y, IsFSHL ? InvShAmt : ShAmt); + setValue(&I, DAG.getNode(ISD::OR, sdl, VT, ShX, ShY)); + return nullptr; + } case Intrinsic::stacksave: { SDValue Op = getRoot(); Res = DAG.getNode( Index: test/CodeGen/AArch64/funnel-shift.ll =================================================================== --- test/CodeGen/AArch64/funnel-shift.ll +++ test/CodeGen/AArch64/funnel-shift.ll @@ -0,0 +1,329 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s + +declare i8 @llvm.fshl.i8(i8, i8, i8) +declare i16 @llvm.fshl.i16(i16, i16, i16) +declare i32 @llvm.fshl.i32(i32, i32, i32) +declare i64 @llvm.fshl.i64(i64, i64, i64) +declare <4 x i32> @llvm.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>) + +declare i8 @llvm.fshr.i8(i8, i8, i8) +declare i16 @llvm.fshr.i16(i16, i16, i16) +declare i32 @llvm.fshr.i32(i32, i32, i32) +declare i64 @llvm.fshr.i64(i64, i64, i64) +declare <4 x i32> @llvm.fshr.v4i32(<4 x i32>, <4 x i32>, <4 x i32>) + +; General case - all operands can be variables. + +define i32 @fshl_i32(i32 %x, i32 %y, i32 %z) { +; CHECK-LABEL: fshl_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: and w9, w2, #0x1f +; CHECK-NEXT: neg w9, w9 +; CHECK-NEXT: lsl w8, w0, w2 +; CHECK-NEXT: lsr w9, w1, w9 +; CHECK-NEXT: orr w0, w8, w9 +; CHECK-NEXT: ret + %f = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %z) + ret i32 %f +} + +; Verify that weird types are minimally supported. +declare i37 @llvm.fshl.i37(i37, i37, i37) +define i37 @fshl_i37(i37 %x, i37 %y, i37 %z) { +; CHECK-LABEL: fshl_i37: +; CHECK: // %bb.0: +; CHECK-NEXT: mov x10, #31883 +; CHECK-NEXT: movk x10, #3542, lsl #16 +; CHECK-NEXT: movk x10, #51366, lsl #32 +; CHECK-NEXT: and x9, x2, #0x1fffffffff +; CHECK-NEXT: movk x10, #56679, lsl #48 +; CHECK-NEXT: umulh x10, x9, x10 +; CHECK-NEXT: mov w11, #37 +; CHECK-NEXT: lsr x10, x10, #5 +; CHECK-NEXT: msub x9, x10, x11, x9 +; CHECK-NEXT: and x8, x1, #0x1fffffffff +; CHECK-NEXT: lsl x10, x0, x9 +; CHECK-NEXT: sub x9, x11, x9 +; CHECK-NEXT: lsr x8, x8, x9 +; CHECK-NEXT: orr x0, x10, x8 +; CHECK-NEXT: ret + %f = call i37 @llvm.fshl.i37(i37 %x, i37 %y, i37 %z) + ret i37 %f +} + +; extract(concat(0b1110000, 0b1111111) << 2) = 0b1000011 + +declare i7 @llvm.fshl.i7(i7, i7, i7) +define i7 @fshl_i7_const_fold() { +; CHECK-LABEL: fshl_i7_const_fold: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w0, #67 +; CHECK-NEXT: ret + %f = call i7 @llvm.fshl.i7(i7 112, i7 127, i7 2) + ret i7 %f +} + +; With constant shift amount, this is 'extr'. + +define i32 @fshl_i32_const_shift(i32 %x, i32 %y) { +; CHECK-LABEL: fshl_i32_const_shift: +; CHECK: // %bb.0: +; CHECK-NEXT: extr w0, w0, w1, #23 +; CHECK-NEXT: ret + %f = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 9) + ret i32 %f +} + +; Check modulo math on shift amount. + +define i32 @fshl_i32_const_overshift(i32 %x, i32 %y) { +; CHECK-LABEL: fshl_i32_const_overshift: +; CHECK: // %bb.0: +; CHECK-NEXT: extr w0, w0, w1, #23 +; CHECK-NEXT: ret + %f = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 41) + ret i32 %f +} + +; 64-bit should also work. + +define i64 @fshl_i64_const_overshift(i64 %x, i64 %y) { +; CHECK-LABEL: fshl_i64_const_overshift: +; CHECK: // %bb.0: +; CHECK-NEXT: extr x0, x0, x1, #23 +; CHECK-NEXT: ret + %f = call i64 @llvm.fshl.i64(i64 %x, i64 %y, i64 105) + ret i64 %f +} + +; This should work without any node-specific logic. + +define i8 @fshl_i8_const_fold() { +; CHECK-LABEL: fshl_i8_const_fold: +; CHECK: // %bb.0: +; CHECK-NEXT: orr w0, wzr, #0x80 +; CHECK-NEXT: ret + %f = call i8 @llvm.fshl.i8(i8 255, i8 0, i8 7) + ret i8 %f +} + +; When first 2 operands match, it's a rotate. + +define i8 @fshl_i8_rotl_const_shift(i8 %x) { +; CHECK-LABEL: fshl_i8_rotl_const_shift: +; CHECK: // %bb.0: +; CHECK-NEXT: ubfx w8, w0, #5, #3 +; CHECK-NEXT: bfi w8, w0, #3, #29 +; CHECK-NEXT: mov w0, w8 +; CHECK-NEXT: ret + %f = call i8 @llvm.fshl.i8(i8 %x, i8 %x, i8 3) + ret i8 %f +} + +; When first 2 operands match, it's a rotate (by variable amount). + +define i16 @fshl_i16_rotl(i16 %x, i16 %z) { +; CHECK-LABEL: fshl_i16_rotl: +; CHECK: // %bb.0: +; CHECK-NEXT: and w9, w1, #0xf +; CHECK-NEXT: orr w10, wzr, #0x10 +; CHECK-NEXT: and w8, w0, #0xffff +; CHECK-NEXT: lsl w11, w0, w9 +; CHECK-NEXT: sub w9, w10, w9 +; CHECK-NEXT: lsr w8, w8, w9 +; CHECK-NEXT: orr w0, w11, w8 +; CHECK-NEXT: ret + %f = call i16 @llvm.fshl.i16(i16 %x, i16 %x, i16 %z) + ret i16 %f +} + +; Vector rotate. + +define <4 x i32> @fshl_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) { +; CHECK-LABEL: fshl_v4i32: +; CHECK: // %bb.0: +; CHECK-NEXT: movi v1.4s, #31 +; CHECK-NEXT: movi v3.4s, #32 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: ushl v2.4s, v0.4s, v1.4s +; CHECK-NEXT: sub v1.4s, v3.4s, v1.4s +; CHECK-NEXT: neg v1.4s, v1.4s +; CHECK-NEXT: ushl v0.4s, v0.4s, v1.4s +; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b +; CHECK-NEXT: ret + %f = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> %z) + ret <4 x i32> %f +} + +; Vector rotate by constant splat amount. + +define <4 x i32> @fshl_v4i32_rotl_const_shift(<4 x i32> %x) { +; CHECK-LABEL: fshl_v4i32_rotl_const_shift: +; CHECK: // %bb.0: +; CHECK-NEXT: ushr v1.4s, v0.4s, #29 +; CHECK-NEXT: shl v0.4s, v0.4s, #3 +; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b +; CHECK-NEXT: ret + %f = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> ) + ret <4 x i32> %f +} + +; Repeat everything for funnel shift right. + +; General case - all operands can be variables. + +define i32 @fshr_i32(i32 %x, i32 %y, i32 %z) { +; CHECK-LABEL: fshr_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: and w9, w2, #0x1f +; CHECK-NEXT: neg w9, w9 +; CHECK-NEXT: lsr w8, w1, w2 +; CHECK-NEXT: lsl w9, w0, w9 +; CHECK-NEXT: orr w0, w9, w8 +; CHECK-NEXT: ret + %f = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %z) + ret i32 %f +} + +; Verify that weird types are minimally supported. +declare i37 @llvm.fshr.i37(i37, i37, i37) +define i37 @fshr_i37(i37 %x, i37 %y, i37 %z) { +; CHECK-LABEL: fshr_i37: +; CHECK: // %bb.0: +; CHECK-NEXT: mov x10, #31883 +; CHECK-NEXT: movk x10, #3542, lsl #16 +; CHECK-NEXT: movk x10, #51366, lsl #32 +; CHECK-NEXT: and x9, x2, #0x1fffffffff +; CHECK-NEXT: movk x10, #56679, lsl #48 +; CHECK-NEXT: umulh x10, x9, x10 +; CHECK-NEXT: mov w11, #37 +; CHECK-NEXT: lsr x10, x10, #5 +; CHECK-NEXT: and x8, x1, #0x1fffffffff +; CHECK-NEXT: msub x9, x10, x11, x9 +; CHECK-NEXT: lsr x8, x8, x9 +; CHECK-NEXT: sub x9, x11, x9 +; CHECK-NEXT: lsl x9, x0, x9 +; CHECK-NEXT: orr x0, x9, x8 +; CHECK-NEXT: ret + %f = call i37 @llvm.fshr.i37(i37 %x, i37 %y, i37 %z) + ret i37 %f +} + +; extract(concat(0b1110000, 0b1111111) >> 2) = 0b0011111 + +declare i7 @llvm.fshr.i7(i7, i7, i7) +define i7 @fshr_i7_const_fold() { +; CHECK-LABEL: fshr_i7_const_fold: +; CHECK: // %bb.0: +; CHECK-NEXT: orr w0, wzr, #0x1f +; CHECK-NEXT: ret + %f = call i7 @llvm.fshr.i7(i7 112, i7 127, i7 2) + ret i7 %f +} + +; With constant shift amount, this is 'extr'. + +define i32 @fshr_i32_const_shift(i32 %x, i32 %y) { +; CHECK-LABEL: fshr_i32_const_shift: +; CHECK: // %bb.0: +; CHECK-NEXT: extr w0, w0, w1, #9 +; CHECK-NEXT: ret + %f = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 9) + ret i32 %f +} + +; Check modulo math on shift amount. 41-32=9. + +define i32 @fshr_i32_const_overshift(i32 %x, i32 %y) { +; CHECK-LABEL: fshr_i32_const_overshift: +; CHECK: // %bb.0: +; CHECK-NEXT: extr w0, w0, w1, #9 +; CHECK-NEXT: ret + %f = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 41) + ret i32 %f +} + +; 64-bit should also work. 105-64 = 41. + +define i64 @fshr_i64_const_overshift(i64 %x, i64 %y) { +; CHECK-LABEL: fshr_i64_const_overshift: +; CHECK: // %bb.0: +; CHECK-NEXT: extr x0, x0, x1, #41 +; CHECK-NEXT: ret + %f = call i64 @llvm.fshr.i64(i64 %x, i64 %y, i64 105) + ret i64 %f +} + +; This should work without any node-specific logic. + +define i8 @fshr_i8_const_fold() { +; CHECK-LABEL: fshr_i8_const_fold: +; CHECK: // %bb.0: +; CHECK-NEXT: orr w0, wzr, #0xfe +; CHECK-NEXT: ret + %f = call i8 @llvm.fshr.i8(i8 255, i8 0, i8 7) + ret i8 %f +} + +; When first 2 operands match, it's a rotate. + +define i8 @fshr_i8_rotl_const_shift(i8 %x) { +; CHECK-LABEL: fshr_i8_rotl_const_shift: +; CHECK: // %bb.0: +; CHECK-NEXT: ubfx w8, w0, #3, #5 +; CHECK-NEXT: bfi w8, w0, #5, #27 +; CHECK-NEXT: mov w0, w8 +; CHECK-NEXT: ret + %f = call i8 @llvm.fshr.i8(i8 %x, i8 %x, i8 3) + ret i8 %f +} + +; When first 2 operands match, it's a rotate (by variable amount). + +define i16 @fshr_i16_rotl(i16 %x, i16 %z) { +; CHECK-LABEL: fshr_i16_rotl: +; CHECK: // %bb.0: +; CHECK-NEXT: and w8, w0, #0xffff +; CHECK-NEXT: and w9, w1, #0xf +; CHECK-NEXT: orr w10, wzr, #0x10 +; CHECK-NEXT: lsr w8, w8, w9 +; CHECK-NEXT: sub w9, w10, w9 +; CHECK-NEXT: lsl w9, w0, w9 +; CHECK-NEXT: orr w0, w9, w8 +; CHECK-NEXT: ret + %f = call i16 @llvm.fshr.i16(i16 %x, i16 %x, i16 %z) + ret i16 %f +} + +; Vector rotate. + +define <4 x i32> @fshr_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) { +; CHECK-LABEL: fshr_v4i32: +; CHECK: // %bb.0: +; CHECK-NEXT: movi v1.4s, #31 +; CHECK-NEXT: movi v3.4s, #32 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: neg v2.4s, v1.4s +; CHECK-NEXT: sub v1.4s, v3.4s, v1.4s +; CHECK-NEXT: ushl v2.4s, v0.4s, v2.4s +; CHECK-NEXT: ushl v0.4s, v0.4s, v1.4s +; CHECK-NEXT: orr v0.16b, v0.16b, v2.16b +; CHECK-NEXT: ret + %f = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> %z) + ret <4 x i32> %f +} + +; Vector rotate by constant splat amount. + +define <4 x i32> @fshr_v4i32_rotl_const_shift(<4 x i32> %x) { +; CHECK-LABEL: fshr_v4i32_rotl_const_shift: +; CHECK: // %bb.0: +; CHECK-NEXT: ushr v1.4s, v0.4s, #3 +; CHECK-NEXT: shl v0.4s, v0.4s, #29 +; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b +; CHECK-NEXT: ret + %f = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> ) + ret <4 x i32> %f +} + Index: test/CodeGen/PowerPC/funnel-shift.ll =================================================================== --- test/CodeGen/PowerPC/funnel-shift.ll +++ test/CodeGen/PowerPC/funnel-shift.ll @@ -0,0 +1,355 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=powerpc64le-- | FileCheck %s + +declare i8 @llvm.fshl.i8(i8, i8, i8) +declare i16 @llvm.fshl.i16(i16, i16, i16) +declare i32 @llvm.fshl.i32(i32, i32, i32) +declare i64 @llvm.fshl.i64(i64, i64, i64) +declare <4 x i32> @llvm.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>) + +declare i8 @llvm.fshr.i8(i8, i8, i8) +declare i16 @llvm.fshr.i16(i16, i16, i16) +declare i32 @llvm.fshr.i32(i32, i32, i32) +declare i64 @llvm.fshr.i64(i64, i64, i64) +declare <4 x i32> @llvm.fshr.v4i32(<4 x i32>, <4 x i32>, <4 x i32>) + +; General case - all operands can be variables. + +define i32 @fshl_i32(i32 %x, i32 %y, i32 %z) { +; CHECK-LABEL: fshl_i32: +; CHECK: # %bb.0: +; CHECK-NEXT: rlwinm 5, 5, 0, 27, 31 +; CHECK-NEXT: subfic 6, 5, 32 +; CHECK-NEXT: slw 3, 3, 5 +; CHECK-NEXT: srw 4, 4, 6 +; CHECK-NEXT: or 3, 3, 4 +; CHECK-NEXT: blr + %f = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %z) + ret i32 %f +} + +; Verify that weird types are minimally supported. +declare i37 @llvm.fshl.i37(i37, i37, i37) +define i37 @fshl_i37(i37 %x, i37 %y, i37 %z) { +; CHECK-LABEL: fshl_i37: +; CHECK: # %bb.0: +; CHECK-NEXT: lis 6, -8857 +; CHECK-NEXT: clrldi 5, 5, 27 +; CHECK-NEXT: clrldi 4, 4, 27 +; CHECK-NEXT: ori 6, 6, 51366 +; CHECK-NEXT: sldi 6, 6, 32 +; CHECK-NEXT: oris 6, 6, 3542 +; CHECK-NEXT: ori 6, 6, 31883 +; CHECK-NEXT: mulhdu 6, 5, 6 +; CHECK-NEXT: rldicl 6, 6, 59, 5 +; CHECK-NEXT: mulli 6, 6, 37 +; CHECK-NEXT: sub 5, 5, 6 +; CHECK-NEXT: subfic 6, 5, 37 +; CHECK-NEXT: sld 3, 3, 5 +; CHECK-NEXT: srd 4, 4, 6 +; CHECK-NEXT: or 3, 3, 4 +; CHECK-NEXT: blr + %f = call i37 @llvm.fshl.i37(i37 %x, i37 %y, i37 %z) + ret i37 %f +} + +; extract(concat(0b1110000, 0b1111111) << 2) = 0b1000011 + +declare i7 @llvm.fshl.i7(i7, i7, i7) +define i7 @fshl_i7_const_fold() { +; CHECK-LABEL: fshl_i7_const_fold: +; CHECK: # %bb.0: +; CHECK-NEXT: li 3, 67 +; CHECK-NEXT: blr + %f = call i7 @llvm.fshl.i7(i7 112, i7 127, i7 2) + ret i7 %f +} + +; With constant shift amount, this is rotate + insert (missing extended mnemonics). + +define i32 @fshl_i32_const_shift(i32 %x, i32 %y) { +; CHECK-LABEL: fshl_i32_const_shift: +; CHECK: # %bb.0: +; CHECK-NEXT: rlwinm 4, 4, 9, 0, 31 +; CHECK-NEXT: rlwimi 4, 3, 9, 0, 22 +; CHECK-NEXT: mr 3, 4 +; CHECK-NEXT: blr + %f = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 9) + ret i32 %f +} + +; Check modulo math on shift amount. + +define i32 @fshl_i32_const_overshift(i32 %x, i32 %y) { +; CHECK-LABEL: fshl_i32_const_overshift: +; CHECK: # %bb.0: +; CHECK-NEXT: rlwinm 4, 4, 9, 0, 31 +; CHECK-NEXT: rlwimi 4, 3, 9, 0, 22 +; CHECK-NEXT: mr 3, 4 +; CHECK-NEXT: blr + %f = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 41) + ret i32 %f +} + +; 64-bit should also work. + +define i64 @fshl_i64_const_overshift(i64 %x, i64 %y) { +; CHECK-LABEL: fshl_i64_const_overshift: +; CHECK: # %bb.0: +; CHECK-NEXT: rotldi 4, 4, 41 +; CHECK-NEXT: rldimi 4, 3, 41, 0 +; CHECK-NEXT: mr 3, 4 +; CHECK-NEXT: blr + %f = call i64 @llvm.fshl.i64(i64 %x, i64 %y, i64 105) + ret i64 %f +} + +; This should work without any node-specific logic. + +define i8 @fshl_i8_const_fold() { +; CHECK-LABEL: fshl_i8_const_fold: +; CHECK: # %bb.0: +; CHECK-NEXT: li 3, 128 +; CHECK-NEXT: blr + %f = call i8 @llvm.fshl.i8(i8 255, i8 0, i8 7) + ret i8 %f +} + +; When first 2 operands match, it's a rotate. + +define i8 @fshl_i8_rotl_const_shift(i8 %x) { +; CHECK-LABEL: fshl_i8_rotl_const_shift: +; CHECK: # %bb.0: +; CHECK-NEXT: rlwinm 4, 3, 27, 0, 31 +; CHECK-NEXT: rlwimi 4, 3, 3, 0, 28 +; CHECK-NEXT: mr 3, 4 +; CHECK-NEXT: blr + %f = call i8 @llvm.fshl.i8(i8 %x, i8 %x, i8 3) + ret i8 %f +} + +; When first 2 operands match, it's a rotate (by variable amount). + +define i16 @fshl_i16_rotl(i16 %x, i16 %z) { +; CHECK-LABEL: fshl_i16_rotl: +; CHECK: # %bb.0: +; CHECK-NEXT: rlwinm 4, 4, 0, 28, 31 +; CHECK-NEXT: clrlwi 5, 3, 16 +; CHECK-NEXT: subfic 6, 4, 16 +; CHECK-NEXT: slw 3, 3, 4 +; CHECK-NEXT: srw 4, 5, 6 +; CHECK-NEXT: or 3, 3, 4 +; CHECK-NEXT: blr + %f = call i16 @llvm.fshl.i16(i16 %x, i16 %x, i16 %z) + ret i16 %f +} + +; Vector rotate. + +define <4 x i32> @fshl_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) { +; CHECK-LABEL: fshl_v4i32: +; CHECK: # %bb.0: +; CHECK-NEXT: vspltisw 3, -16 +; CHECK-NEXT: vspltisw 5, 15 +; CHECK-NEXT: addis 3, 2, .LCPI9_0@toc@ha +; CHECK-NEXT: addi 3, 3, .LCPI9_0@toc@l +; CHECK-NEXT: vsubuwm 3, 5, 3 +; CHECK-NEXT: lvx 5, 0, 3 +; CHECK-NEXT: xxland 35, 36, 35 +; CHECK-NEXT: vslw 4, 2, 4 +; CHECK-NEXT: vsubuwm 3, 5, 3 +; CHECK-NEXT: vsrw 2, 2, 3 +; CHECK-NEXT: xxlor 34, 36, 34 +; CHECK-NEXT: blr + %f = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> %z) + ret <4 x i32> %f +} + +; Vector rotate by constant splat amount. + +define <4 x i32> @fshl_v4i32_rotl_const_shift(<4 x i32> %x) { +; CHECK-LABEL: fshl_v4i32_rotl_const_shift: +; CHECK: # %bb.0: +; CHECK-NEXT: vspltisw 3, -16 +; CHECK-NEXT: vspltisw 4, 13 +; CHECK-NEXT: vspltisw 5, 3 +; CHECK-NEXT: vsubuwm 3, 4, 3 +; CHECK-NEXT: vslw 4, 2, 5 +; CHECK-NEXT: vsrw 2, 2, 3 +; CHECK-NEXT: xxlor 34, 36, 34 +; CHECK-NEXT: blr + %f = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> ) + ret <4 x i32> %f +} + +; Repeat everything for funnel shift right. + +; General case - all operands can be variables. + +define i32 @fshr_i32(i32 %x, i32 %y, i32 %z) { +; CHECK-LABEL: fshr_i32: +; CHECK: # %bb.0: +; CHECK-NEXT: rlwinm 5, 5, 0, 27, 31 +; CHECK-NEXT: subfic 6, 5, 32 +; CHECK-NEXT: srw 4, 4, 5 +; CHECK-NEXT: slw 3, 3, 6 +; CHECK-NEXT: or 3, 3, 4 +; CHECK-NEXT: blr + %f = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %z) + ret i32 %f +} + +; Verify that weird types are minimally supported. +declare i37 @llvm.fshr.i37(i37, i37, i37) +define i37 @fshr_i37(i37 %x, i37 %y, i37 %z) { +; CHECK-LABEL: fshr_i37: +; CHECK: # %bb.0: +; CHECK-NEXT: lis 6, -8857 +; CHECK-NEXT: clrldi 5, 5, 27 +; CHECK-NEXT: clrldi 4, 4, 27 +; CHECK-NEXT: ori 6, 6, 51366 +; CHECK-NEXT: sldi 6, 6, 32 +; CHECK-NEXT: oris 6, 6, 3542 +; CHECK-NEXT: ori 6, 6, 31883 +; CHECK-NEXT: mulhdu 6, 5, 6 +; CHECK-NEXT: rldicl 6, 6, 59, 5 +; CHECK-NEXT: mulli 6, 6, 37 +; CHECK-NEXT: sub 5, 5, 6 +; CHECK-NEXT: subfic 6, 5, 37 +; CHECK-NEXT: srd 4, 4, 5 +; CHECK-NEXT: sld 3, 3, 6 +; CHECK-NEXT: or 3, 3, 4 +; CHECK-NEXT: blr + %f = call i37 @llvm.fshr.i37(i37 %x, i37 %y, i37 %z) + ret i37 %f +} + +; extract(concat(0b1110000, 0b1111111) >> 2) = 0b0011111 + +declare i7 @llvm.fshr.i7(i7, i7, i7) +define i7 @fshr_i7_const_fold() { +; CHECK-LABEL: fshr_i7_const_fold: +; CHECK: # %bb.0: +; CHECK-NEXT: li 3, 31 +; CHECK-NEXT: blr + %f = call i7 @llvm.fshr.i7(i7 112, i7 127, i7 2) + ret i7 %f +} + +; With constant shift amount, this is rotate + insert (missing extended mnemonics). + +define i32 @fshr_i32_const_shift(i32 %x, i32 %y) { +; CHECK-LABEL: fshr_i32_const_shift: +; CHECK: # %bb.0: +; CHECK-NEXT: rlwinm 4, 4, 23, 0, 31 +; CHECK-NEXT: rlwimi 4, 3, 23, 0, 8 +; CHECK-NEXT: mr 3, 4 +; CHECK-NEXT: blr + %f = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 9) + ret i32 %f +} + +; Check modulo math on shift amount. 41-32=9. + +define i32 @fshr_i32_const_overshift(i32 %x, i32 %y) { +; CHECK-LABEL: fshr_i32_const_overshift: +; CHECK: # %bb.0: +; CHECK-NEXT: rlwinm 4, 4, 23, 0, 31 +; CHECK-NEXT: rlwimi 4, 3, 23, 0, 8 +; CHECK-NEXT: mr 3, 4 +; CHECK-NEXT: blr + %f = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 41) + ret i32 %f +} + +; 64-bit should also work. 105-64 = 41. + +define i64 @fshr_i64_const_overshift(i64 %x, i64 %y) { +; CHECK-LABEL: fshr_i64_const_overshift: +; CHECK: # %bb.0: +; CHECK-NEXT: rotldi 4, 4, 23 +; CHECK-NEXT: rldimi 4, 3, 23, 0 +; CHECK-NEXT: mr 3, 4 +; CHECK-NEXT: blr + %f = call i64 @llvm.fshr.i64(i64 %x, i64 %y, i64 105) + ret i64 %f +} + +; This should work without any node-specific logic. + +define i8 @fshr_i8_const_fold() { +; CHECK-LABEL: fshr_i8_const_fold: +; CHECK: # %bb.0: +; CHECK-NEXT: li 3, 254 +; CHECK-NEXT: blr + %f = call i8 @llvm.fshr.i8(i8 255, i8 0, i8 7) + ret i8 %f +} + +; When first 2 operands match, it's a rotate. + +define i8 @fshr_i8_rotl_const_shift(i8 %x) { +; CHECK-LABEL: fshr_i8_rotl_const_shift: +; CHECK: # %bb.0: +; CHECK-NEXT: rlwinm 4, 3, 29, 0, 31 +; CHECK-NEXT: rlwimi 4, 3, 5, 0, 26 +; CHECK-NEXT: mr 3, 4 +; CHECK-NEXT: blr + %f = call i8 @llvm.fshr.i8(i8 %x, i8 %x, i8 3) + ret i8 %f +} + +; When first 2 operands match, it's a rotate (by variable amount). + +define i16 @fshr_i16_rotl(i16 %x, i16 %z) { +; CHECK-LABEL: fshr_i16_rotl: +; CHECK: # %bb.0: +; CHECK-NEXT: rlwinm 4, 4, 0, 28, 31 +; CHECK-NEXT: clrlwi 5, 3, 16 +; CHECK-NEXT: subfic 6, 4, 16 +; CHECK-NEXT: srw 4, 5, 4 +; CHECK-NEXT: slw 3, 3, 6 +; CHECK-NEXT: or 3, 3, 4 +; CHECK-NEXT: blr + %f = call i16 @llvm.fshr.i16(i16 %x, i16 %x, i16 %z) + ret i16 %f +} + +; Vector rotate. + +define <4 x i32> @fshr_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) { +; CHECK-LABEL: fshr_v4i32: +; CHECK: # %bb.0: +; CHECK-NEXT: vspltisw 3, -16 +; CHECK-NEXT: vspltisw 5, 15 +; CHECK-NEXT: addis 3, 2, .LCPI20_0@toc@ha +; CHECK-NEXT: addi 3, 3, .LCPI20_0@toc@l +; CHECK-NEXT: vsubuwm 3, 5, 3 +; CHECK-NEXT: lvx 5, 0, 3 +; CHECK-NEXT: xxland 35, 36, 35 +; CHECK-NEXT: vsrw 4, 2, 4 +; CHECK-NEXT: vsubuwm 3, 5, 3 +; CHECK-NEXT: vslw 2, 2, 3 +; CHECK-NEXT: xxlor 34, 34, 36 +; CHECK-NEXT: blr + %f = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> %z) + ret <4 x i32> %f +} + +; Vector rotate by constant splat amount. + +define <4 x i32> @fshr_v4i32_rotl_const_shift(<4 x i32> %x) { +; CHECK-LABEL: fshr_v4i32_rotl_const_shift: +; CHECK: # %bb.0: +; CHECK-NEXT: vspltisw 3, -16 +; CHECK-NEXT: vspltisw 4, 13 +; CHECK-NEXT: vspltisw 5, 3 +; CHECK-NEXT: vsubuwm 3, 4, 3 +; CHECK-NEXT: vsrw 4, 2, 5 +; CHECK-NEXT: vslw 2, 2, 3 +; CHECK-NEXT: xxlor 34, 34, 36 +; CHECK-NEXT: blr + %f = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> ) + ret <4 x i32> %f +} + Index: test/CodeGen/X86/funnel-shift.ll =================================================================== --- test/CodeGen/X86/funnel-shift.ll +++ test/CodeGen/X86/funnel-shift.ll @@ -0,0 +1,386 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s --check-prefixes=ANY,X64 +; RUN: llc < %s -mtriple=x86_64-- -mattr=avx2 | FileCheck %s --check-prefixes=ANY,AVX2 + +declare i8 @llvm.fshl.i8(i8, i8, i8) +declare i16 @llvm.fshl.i16(i16, i16, i16) +declare i32 @llvm.fshl.i32(i32, i32, i32) +declare i64 @llvm.fshl.i64(i64, i64, i64) +declare <4 x i32> @llvm.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>) + +declare i8 @llvm.fshr.i8(i8, i8, i8) +declare i16 @llvm.fshr.i16(i16, i16, i16) +declare i32 @llvm.fshr.i32(i32, i32, i32) +declare i64 @llvm.fshr.i64(i64, i64, i64) +declare <4 x i32> @llvm.fshr.v4i32(<4 x i32>, <4 x i32>, <4 x i32>) + +; General case - all operands can be variables - x86 has shld. + +define i32 @fshl_i32(i32 %x, i32 %y, i32 %z) { +; ANY-LABEL: fshl_i32: +; ANY: # %bb.0: +; ANY-NEXT: andl $31, %edx +; ANY-NEXT: movl %edx, %ecx +; ANY-NEXT: shldl %cl, %esi, %edi +; ANY-NEXT: movl %edi, %eax +; ANY-NEXT: retq + %f = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %z) + ret i32 %f +} + +; Verify that weird types are minimally supported. +declare i37 @llvm.fshl.i37(i37, i37, i37) +define i37 @fshl_i37(i37 %x, i37 %y, i37 %z) { +; ANY-LABEL: fshl_i37: +; ANY: # %bb.0: +; ANY-NEXT: movq %rdx, %r8 +; ANY-NEXT: movabsq $137438953471, %rax # imm = 0x1FFFFFFFFF +; ANY-NEXT: andq %rax, %rsi +; ANY-NEXT: andq %rax, %r8 +; ANY-NEXT: movabsq $-2492803253203993461, %rcx # imm = 0xDD67C8A60DD67C8B +; ANY-NEXT: movq %r8, %rax +; ANY-NEXT: mulq %rcx +; ANY-NEXT: shrq $5, %rdx +; ANY-NEXT: imulq $37, %rdx, %rax +; ANY-NEXT: subq %rax, %r8 +; ANY-NEXT: movl %r8d, %ecx +; ANY-NEXT: shlq %cl, %rdi +; ANY-NEXT: movl $37, %ecx +; ANY-NEXT: subl %r8d, %ecx +; ANY-NEXT: # kill: def $cl killed $cl killed $ecx +; ANY-NEXT: shrq %cl, %rsi +; ANY-NEXT: orq %rdi, %rsi +; ANY-NEXT: movq %rsi, %rax +; ANY-NEXT: retq + %f = call i37 @llvm.fshl.i37(i37 %x, i37 %y, i37 %z) + ret i37 %f +} + +; extract(concat(0b1110000, 0b1111111) << 2) = 0b1000011 + +declare i7 @llvm.fshl.i7(i7, i7, i7) +define i7 @fshl_i7_const_fold() { +; ANY-LABEL: fshl_i7_const_fold: +; ANY: # %bb.0: +; ANY-NEXT: movb $67, %al +; ANY-NEXT: retq + %f = call i7 @llvm.fshl.i7(i7 112, i7 127, i7 2) + ret i7 %f +} + +; With constant shift amount, this is 'shld' with constant operand. + +define i32 @fshl_i32_const_shift(i32 %x, i32 %y) { +; ANY-LABEL: fshl_i32_const_shift: +; ANY: # %bb.0: +; ANY-NEXT: shldl $9, %esi, %edi +; ANY-NEXT: movl %edi, %eax +; ANY-NEXT: retq + %f = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 9) + ret i32 %f +} + +; Check modulo math on shift amount. + +define i32 @fshl_i32_const_overshift(i32 %x, i32 %y) { +; ANY-LABEL: fshl_i32_const_overshift: +; ANY: # %bb.0: +; ANY-NEXT: shldl $9, %esi, %edi +; ANY-NEXT: movl %edi, %eax +; ANY-NEXT: retq + %f = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 41) + ret i32 %f +} + +; 64-bit should also work. + +define i64 @fshl_i64_const_overshift(i64 %x, i64 %y) { +; ANY-LABEL: fshl_i64_const_overshift: +; ANY: # %bb.0: +; ANY-NEXT: shldq $41, %rsi, %rdi +; ANY-NEXT: movq %rdi, %rax +; ANY-NEXT: retq + %f = call i64 @llvm.fshl.i64(i64 %x, i64 %y, i64 105) + ret i64 %f +} + +; This should work without any node-specific logic. + +define i8 @fshl_i8_const_fold() { +; ANY-LABEL: fshl_i8_const_fold: +; ANY: # %bb.0: +; ANY-NEXT: movb $-128, %al +; ANY-NEXT: retq + %f = call i8 @llvm.fshl.i8(i8 255, i8 0, i8 7) + ret i8 %f +} + +; When first 2 operands match, it's a rotate. + +define i8 @fshl_i8_rotl_const_shift(i8 %x) { +; ANY-LABEL: fshl_i8_rotl_const_shift: +; ANY: # %bb.0: +; ANY-NEXT: rolb $3, %dil +; ANY-NEXT: movl %edi, %eax +; ANY-NEXT: retq + %f = call i8 @llvm.fshl.i8(i8 %x, i8 %x, i8 3) + ret i8 %f +} + +; When first 2 operands match, it's a rotate (by variable amount). + +define i16 @fshl_i16_rotl(i16 %x, i16 %z) { +; ANY-LABEL: fshl_i16_rotl: +; ANY: # %bb.0: +; ANY-NEXT: movl %esi, %ecx +; ANY-NEXT: rolw %cl, %di +; ANY-NEXT: movl %edi, %eax +; ANY-NEXT: retq + %f = call i16 @llvm.fshl.i16(i16 %x, i16 %x, i16 %z) + ret i16 %f +} + +; Vector rotate. + +define <4 x i32> @fshl_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) { +; X64-LABEL: fshl_v4i32: +; X64: # %bb.0: +; X64-NEXT: pand {{.*}}(%rip), %xmm2 +; X64-NEXT: pslld $23, %xmm2 +; X64-NEXT: paddd {{.*}}(%rip), %xmm2 +; X64-NEXT: cvttps2dq %xmm2, %xmm1 +; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] +; X64-NEXT: pmuludq %xmm1, %xmm0 +; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3] +; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; X64-NEXT: pmuludq %xmm2, %xmm1 +; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3] +; X64-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] +; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; X64-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; X64-NEXT: por %xmm3, %xmm0 +; X64-NEXT: retq +; +; AVX2-LABEL: fshl_v4i32: +; AVX2: # %bb.0: +; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [31,31,31,31] +; AVX2-NEXT: vpand %xmm1, %xmm2, %xmm1 +; AVX2-NEXT: vpsllvd %xmm1, %xmm0, %xmm2 +; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [32,32,32,32] +; AVX2-NEXT: vpsubd %xmm1, %xmm3, %xmm1 +; AVX2-NEXT: vpsrlvd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm0, %xmm2, %xmm0 +; AVX2-NEXT: retq + %f = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> %z) + ret <4 x i32> %f +} + +; Vector rotate by constant splat amount. + +define <4 x i32> @fshl_v4i32_rotl_const_shift(<4 x i32> %x) { +; X64-LABEL: fshl_v4i32_rotl_const_shift: +; X64: # %bb.0: +; X64-NEXT: movdqa %xmm0, %xmm1 +; X64-NEXT: psrld $29, %xmm1 +; X64-NEXT: pslld $3, %xmm0 +; X64-NEXT: por %xmm1, %xmm0 +; X64-NEXT: retq +; +; AVX2-LABEL: fshl_v4i32_rotl_const_shift: +; AVX2: # %bb.0: +; AVX2-NEXT: vpsrld $29, %xmm0, %xmm1 +; AVX2-NEXT: vpslld $3, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq + %f = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> ) + ret <4 x i32> %f +} + +; Repeat everything for funnel shift right. + +; General case - all operands can be variables - x86 has 'shrd'. + +define i32 @fshr_i32(i32 %x, i32 %y, i32 %z) { +; ANY-LABEL: fshr_i32: +; ANY: # %bb.0: +; ANY-NEXT: andl $31, %edx +; ANY-NEXT: movl %edx, %ecx +; ANY-NEXT: shrdl %cl, %edi, %esi +; ANY-NEXT: movl %esi, %eax +; ANY-NEXT: retq + %f = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %z) + ret i32 %f +} + +; Verify that weird types are minimally supported. +declare i37 @llvm.fshr.i37(i37, i37, i37) +define i37 @fshr_i37(i37 %x, i37 %y, i37 %z) { +; ANY-LABEL: fshr_i37: +; ANY: # %bb.0: +; ANY-NEXT: movq %rdx, %r8 +; ANY-NEXT: movabsq $137438953471, %rax # imm = 0x1FFFFFFFFF +; ANY-NEXT: andq %rax, %rsi +; ANY-NEXT: andq %rax, %r8 +; ANY-NEXT: movabsq $-2492803253203993461, %rcx # imm = 0xDD67C8A60DD67C8B +; ANY-NEXT: movq %r8, %rax +; ANY-NEXT: mulq %rcx +; ANY-NEXT: shrq $5, %rdx +; ANY-NEXT: imulq $37, %rdx, %rax +; ANY-NEXT: subq %rax, %r8 +; ANY-NEXT: movl %r8d, %ecx +; ANY-NEXT: shrq %cl, %rsi +; ANY-NEXT: movl $37, %ecx +; ANY-NEXT: subl %r8d, %ecx +; ANY-NEXT: # kill: def $cl killed $cl killed $ecx +; ANY-NEXT: shlq %cl, %rdi +; ANY-NEXT: orq %rsi, %rdi +; ANY-NEXT: movq %rdi, %rax +; ANY-NEXT: retq + %f = call i37 @llvm.fshr.i37(i37 %x, i37 %y, i37 %z) + ret i37 %f +} + +; extract(concat(0b1110000, 0b1111111) >> 2) = 0b0011111 + +declare i7 @llvm.fshr.i7(i7, i7, i7) +define i7 @fshr_i7_const_fold() { +; ANY-LABEL: fshr_i7_const_fold: +; ANY: # %bb.0: +; ANY-NEXT: movb $31, %al +; ANY-NEXT: retq + %f = call i7 @llvm.fshr.i7(i7 112, i7 127, i7 2) + ret i7 %f +} + +; With constant shift amount, this is 'shrd' or 'shld'. + +define i32 @fshr_i32_const_shift(i32 %x, i32 %y) { +; ANY-LABEL: fshr_i32_const_shift: +; ANY: # %bb.0: +; ANY-NEXT: shldl $23, %esi, %edi +; ANY-NEXT: movl %edi, %eax +; ANY-NEXT: retq + %f = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 9) + ret i32 %f +} + +; Check modulo math on shift amount. 41-32=9, but right-shift became left, so 32-9=23. + +define i32 @fshr_i32_const_overshift(i32 %x, i32 %y) { +; ANY-LABEL: fshr_i32_const_overshift: +; ANY: # %bb.0: +; ANY-NEXT: shldl $23, %esi, %edi +; ANY-NEXT: movl %edi, %eax +; ANY-NEXT: retq + %f = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 41) + ret i32 %f +} + +; 64-bit should also work. 105-64 = 41, but right-shift became left, so 64-41=23. + +define i64 @fshr_i64_const_overshift(i64 %x, i64 %y) { +; ANY-LABEL: fshr_i64_const_overshift: +; ANY: # %bb.0: +; ANY-NEXT: shldq $23, %rsi, %rdi +; ANY-NEXT: movq %rdi, %rax +; ANY-NEXT: retq + %f = call i64 @llvm.fshr.i64(i64 %x, i64 %y, i64 105) + ret i64 %f +} + +; This should work without any node-specific logic. + +define i8 @fshr_i8_const_fold() { +; ANY-LABEL: fshr_i8_const_fold: +; ANY: # %bb.0: +; ANY-NEXT: movb $-2, %al +; ANY-NEXT: retq + %f = call i8 @llvm.fshr.i8(i8 255, i8 0, i8 7) + ret i8 %f +} + +; When first 2 operands match, it's a rotate. + +define i8 @fshr_i8_rotl_const_shift(i8 %x) { +; ANY-LABEL: fshr_i8_rotl_const_shift: +; ANY: # %bb.0: +; ANY-NEXT: rolb $5, %dil +; ANY-NEXT: movl %edi, %eax +; ANY-NEXT: retq + %f = call i8 @llvm.fshr.i8(i8 %x, i8 %x, i8 3) + ret i8 %f +} + +; When first 2 operands match, it's a rotate (by variable amount). + +define i16 @fshr_i16_rotl(i16 %x, i16 %z) { +; ANY-LABEL: fshr_i16_rotl: +; ANY: # %bb.0: +; ANY-NEXT: movl %esi, %ecx +; ANY-NEXT: rorw %cl, %di +; ANY-NEXT: movl %edi, %eax +; ANY-NEXT: retq + %f = call i16 @llvm.fshr.i16(i16 %x, i16 %x, i16 %z) + ret i16 %f +} + +; Vector rotate. + +define <4 x i32> @fshr_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) { +; X64-LABEL: fshr_v4i32: +; X64: # %bb.0: +; X64-NEXT: pand {{.*}}(%rip), %xmm2 +; X64-NEXT: movdqa {{.*#+}} xmm1 = [32,32,32,32] +; X64-NEXT: psubd %xmm2, %xmm1 +; X64-NEXT: pslld $23, %xmm1 +; X64-NEXT: paddd {{.*}}(%rip), %xmm1 +; X64-NEXT: cvttps2dq %xmm1, %xmm1 +; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] +; X64-NEXT: pmuludq %xmm1, %xmm0 +; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3] +; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; X64-NEXT: pmuludq %xmm2, %xmm1 +; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3] +; X64-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] +; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; X64-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; X64-NEXT: por %xmm3, %xmm0 +; X64-NEXT: retq +; +; AVX2-LABEL: fshr_v4i32: +; AVX2: # %bb.0: +; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [31,31,31,31] +; AVX2-NEXT: vpand %xmm1, %xmm2, %xmm1 +; AVX2-NEXT: vpsrlvd %xmm1, %xmm0, %xmm2 +; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [32,32,32,32] +; AVX2-NEXT: vpsubd %xmm1, %xmm3, %xmm1 +; AVX2-NEXT: vpsllvd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: retq + %f = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> %z) + ret <4 x i32> %f +} + +; Vector rotate by constant splat amount. + +define <4 x i32> @fshr_v4i32_rotl_const_shift(<4 x i32> %x) { +; X64-LABEL: fshr_v4i32_rotl_const_shift: +; X64: # %bb.0: +; X64-NEXT: movdqa %xmm0, %xmm1 +; X64-NEXT: psrld $3, %xmm1 +; X64-NEXT: pslld $29, %xmm0 +; X64-NEXT: por %xmm1, %xmm0 +; X64-NEXT: retq +; +; AVX2-LABEL: fshr_v4i32_rotl_const_shift: +; AVX2: # %bb.0: +; AVX2-NEXT: vpsrld $3, %xmm0, %xmm1 +; AVX2-NEXT: vpslld $29, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq + %f = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> ) + ret <4 x i32> %f +} +