[WebAssembly] Saturating arithmetic intrinsics

[tlively]

Summary

Depends on D52805.

[tlively]

• Add newline for readability.

[aheejin]

lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

987

Newline here? :3

[tlively]

• Another newline

[tlively]

• Rebase

[aheejin]

test/CodeGen/WebAssembly/simd-arith.ll

6

Do we need both the slow and fast versions for NO-SIMD128 too? Aren't they the same anyway?

[tlively]

• Remove unused FileCheck labels

[aheejin]

test/CodeGen/WebAssembly/simd-arith.ll

6

Oh what I meant is... do we need to test both with and without fastisel for NO-SIMD128? With NO-SIMD128 we are just testing the pre-existing scalar expansion behavior and checking the SIMD instructions do not kick in.

[tlively]

test/CodeGen/WebAssembly/simd-arith.ll

6

Sounds like fast isel is responsible for complying with architecture features itself, so we still need this test.

include/llvm/IR/IntrinsicsWebAssembly.td

def int_wasm_atomic_notify:
   Intrinsic<[llvm_i32_ty], [LLVMPointerType<llvm_i32_ty>, llvm_i32_ty],
             [IntrInaccessibleMemOnly, NoCapture<0>, IntrHasSideEffects], "",
             [SDNPMemOperand]>;
 
 //===----------------------------------------------------------------------===//
 // SIMD intrinsics
 //===----------------------------------------------------------------------===//
 
+def int_wasm_add_saturate_signed :
+  Intrinsic<[llvm_anyvector_ty],
+            [LLVMMatchType<0>, LLVMMatchType<0>],
+            [IntrNoMem, IntrSpeculatable]>;
+def int_wasm_add_saturate_unsigned :
+  Intrinsic<[llvm_anyvector_ty],
+            [LLVMMatchType<0>, LLVMMatchType<0>],
+            [IntrNoMem, IntrSpeculatable]>;
+def int_wasm_sub_saturate_signed :
+  Intrinsic<[llvm_anyvector_ty],
+            [LLVMMatchType<0>, LLVMMatchType<0>],
+            [IntrNoMem, IntrSpeculatable]>;
+def int_wasm_sub_saturate_unsigned :
+  Intrinsic<[llvm_anyvector_ty],
+            [LLVMMatchType<0>, LLVMMatchType<0>],
+            [IntrNoMem, IntrSpeculatable]>;
+def int_wasm_bitselect :
+  Intrinsic<[llvm_anyvector_ty],
+            [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
+            [IntrNoMem, IntrSpeculatable]>;
 def int_wasm_anytrue :
   Intrinsic<[llvm_i32_ty],
             [llvm_anyvector_ty],
             [IntrNoMem, IntrSpeculatable]>;
 def int_wasm_alltrue :
   Intrinsic<[llvm_i32_ty],
             [llvm_anyvector_ty],
             [IntrNoMem, IntrSpeculatable]>;
 
 } // TargetPrefix = "wasm"

lib/Target/WebAssembly/WebAssemblyISD.def

 HANDLE_NODETYPE(RETURN)
 HANDLE_NODETYPE(ARGUMENT)
 HANDLE_NODETYPE(Wrapper)
 HANDLE_NODETYPE(BR_IF)
 HANDLE_NODETYPE(BR_TABLE)
 HANDLE_NODETYPE(SHUFFLE)
 HANDLE_NODETYPE(ANYTRUE)
 HANDLE_NODETYPE(ALLTRUE)
+HANDLE_NODETYPE(BITSELECT)
+HANDLE_NODETYPE(ADD_SAT_S)
+HANDLE_NODETYPE(ADD_SAT_U)
+HANDLE_NODETYPE(SUB_SAT_S)
+HANDLE_NODETYPE(SUB_SAT_U)
 
 // add memory opcodes starting at ISD::FIRST_TARGET_MEMORY_OPCODE here...

lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

 SDValue
 WebAssemblyTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                                                    SelectionDAG &DAG) const {
   unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   SDLoc DL(Op);
   switch (IntNo) {
   default:
     return {}; // Don't custom lower most intrinsics.
+
+  case Intrinsic::wasm_add_saturate_signed:
+  case Intrinsic::wasm_add_saturate_unsigned:
+  case Intrinsic::wasm_sub_saturate_signed:
+  case Intrinsic::wasm_sub_saturate_unsigned: {
+    unsigned OpCode;
+    switch (IntNo) {
+    case Intrinsic::wasm_add_saturate_signed:
+      OpCode = WebAssemblyISD::ADD_SAT_S;
+      break;
+    case Intrinsic::wasm_add_saturate_unsigned:
+      OpCode = WebAssemblyISD::ADD_SAT_U;
+      break;
+    case Intrinsic::wasm_sub_saturate_signed:
+      OpCode = WebAssemblyISD::SUB_SAT_S;
+      break;
+    case Intrinsic::wasm_sub_saturate_unsigned:
+      OpCode = WebAssemblyISD::SUB_SAT_U;
+      break;
+    default:
+      llvm_unreachable("unexpected intrinsic id");
+      break;
+    }
+    return DAG.getNode(OpCode, DL, Op.getValueType(), Op.getOperand(1),
+                       Op.getOperand(2));
+  }
+
+  case Intrinsic::wasm_bitselect:
+    return DAG.getNode(WebAssemblyISD::BITSELECT, DL, Op.getValueType(),
+                       Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
+
   case Intrinsic::wasm_anytrue:
   case Intrinsic::wasm_alltrue: {
     unsigned OpCode = IntNo == Intrinsic::wasm_anytrue
                           ? WebAssemblyISD::ANYTRUE
                           : WebAssemblyISD::ALLTRUE;
     return DAG.getNode(OpCode, DL, Op.getValueType(), Op.getOperand(1));
   }
+
   case Intrinsic::wasm_lsda:
     // TODO For now, just return 0 not to crash
     return DAG.getConstant(0, DL, Op.getValueType());
   }
 }
 
 SDValue
 WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,

lib/Target/WebAssembly/WebAssemblyInstrSIMD.td

 // constrained immediate argument types
 foreach SIZE = [8, 16] in
 def ImmI#SIZE : ImmLeaf<i32, "return (Imm & ((1UL << "#SIZE#") - 1)) == Imm;">;
 foreach SIZE = [2, 4, 8, 16, 32] in
 def LaneIdx#SIZE : ImmLeaf<i32, "return 0 <= Imm && Imm < "#SIZE#";">;
 
 // Custom nodes for custom operations
 def wasm_shuffle_t : SDTypeProfile<1, 18, []>;
+def wasm_saturate_t : SDTypeProfile<1, 2,
+  [SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>]
+>;
+def wasm_bitselect_t : SDTypeProfile<1, 3,
+  [SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisSameAs<0, 3>]
+>;
 def wasm_reduce_t : SDTypeProfile<1, 1, [SDTCisVT<0, i32>, SDTCisVec<1>]>;
 def wasm_shuffle : SDNode<"WebAssemblyISD::SHUFFLE", wasm_shuffle_t>;
+def wasm_add_sat_s : SDNode<"WebAssemblyISD::ADD_SAT_S", wasm_saturate_t>;
+def wasm_add_sat_u : SDNode<"WebAssemblyISD::ADD_SAT_U", wasm_saturate_t>;
+def wasm_sub_sat_s : SDNode<"WebAssemblyISD::SUB_SAT_S", wasm_saturate_t>;
+def wasm_sub_sat_u : SDNode<"WebAssemblyISD::SUB_SAT_U", wasm_saturate_t>;
+def wasm_bitselect : SDNode<"WebAssemblyISD::BITSELECT", wasm_bitselect_t>;
 def wasm_anytrue : SDNode<"WebAssemblyISD::ANYTRUE", wasm_reduce_t>;
 def wasm_alltrue : SDNode<"WebAssemblyISD::ALLTRUE", wasm_reduce_t>;
 
 multiclass ConstVec<ValueType vec_t, dag ops, dag pat, string args> {
   let isMoveImm = 1, isReMaterializable = 1 in
   defm CONST_V128_#vec_t : SIMD_I<(outs V128:$dst), ops, (outs), ops,
                                   [(set V128:$dst, (vec_t pat))],
                                   "v128.const\t$dst, "#args,

 multiclass SIMDBinaryInt<SDNode node, string name, bits<32> baseInst> {
   defm "" : SIMDBinaryIntNoI64x2<node, name, baseInst>;
   defm "" : SIMDBinary<v2i64, "i64x2", node, name, !add(baseInst, 3)>;
 }
 multiclass SIMDBinaryFP<SDNode node, string name, bits<32> baseInst> {
   defm "" : SIMDBinary<v4f32, "f32x4", node, name, baseInst>;
   defm "" : SIMDBinary<v2f64, "f64x2", node, name, !add(baseInst, 1)>;
 }
+multiclass SIMDBinarySat<SDNode node, string name, bits<32> baseInst> {
+  defm "" : SIMDBinary<v16i8, "i8x16", node, name, baseInst>;
+  defm "" : SIMDBinary<v8i16, "i16x8", node, name, !add(baseInst, 2)>;
+}
 multiclass SIMDShift<ValueType vec_t, string vec, SDNode node, dag shift_vec,
                      string name, bits<32> simdop> {
   defm _#vec_t : SIMD_I<(outs V128:$dst), (ins V128:$vec, I32:$x),
                         (outs), (ins),
                         [(set (vec_t V128:$dst),
                           (node V128:$vec, (vec_t shift_vec)))],
                         vec#"."#name#"\t$dst, $vec, $x", vec#"."#name, simdop>;
 }
defm NOT_#vec_t : SIMD_I<(outs V128:$dst), (ins V128:$vec),
                              (vec_t V128:$dst),
                              (vec_t (xor
                                (vec_t V128:$vec),
                                (vec_t (splat_pat (lane_t -1)))
                              ))
                            )],
                            "v128.not\t$dst, $vec", "v128.not", 63>;
 }
+multiclass Bitselect<ValueType vec_t> {
+  defm BITSELECT_#vec_t :
+    SIMD_I<(outs V128:$dst), (ins V128:$v1, V128:$v2, V128:$c), (outs), (ins),
+           [(set (vec_t V128:$dst),
+             (vec_t (wasm_bitselect
+               (vec_t V128:$c), (vec_t V128:$v1), (vec_t V128:$v2)
+             ))
+           )],
+           "v128.bitselect\t$dst, $v1, $v2, $c", "v128.bitselect", 64>;
+}
 multiclass SIMDReduceVec<ValueType vec_t, string vec, string name, SDNode op,
                       bits<32> simdop> {
   defm _#vec_t : SIMD_I<(outs I32:$dst), (ins V128:$vec), (outs), (ins),
                        [(set I32:$dst, (i32 (op (vec_t V128:$vec))))],
                        vec#"."#name#"\t$dst, $vec", vec#"."#name, simdop>;
 }
 multiclass SIMDReduce<string name, SDNode op, bits<32> baseInst> {
   defm "" : SIMDReduceVec<v16i8, "i8x16", name, op, baseInst>;
SIMD_I<(outs V128:$dst),
            "$m8, $m9, $mA, $mB, $mC, $mD, $mE, $mF",
          23>;
 
 let isCommutable = 1 in {
 defm ADD : SIMDBinaryInt<add, "add", 24>;
 defm ADD : SIMDBinaryFP<fadd, "add", 133>;
 defm MUL : SIMDBinaryIntNoI64x2<mul, "mul", 32>;
 defm MUL : SIMDBinaryFP<fmul, "mul", 139>;
+defm ADD_SAT_S : SIMDBinarySat<wasm_add_sat_s, "add_saturate_s", 40>;
+defm ADD_SAT_U : SIMDBinarySat<wasm_add_sat_u, "add_saturate_u", 41>;
 } // isCommutable = 1
 
 defm SUB : SIMDBinaryInt<sub, "sub", 28>;
 defm SUB : SIMDBinaryFP<fsub, "sub", 135>;
+defm SUB_SAT_S : SIMDBinarySat<wasm_sub_sat_s, "sub_saturate_s", 44>;
+defm SUB_SAT_U : SIMDBinarySat<wasm_sub_sat_u, "sub_saturate_u", 45>;
 defm DIV : SIMDBinaryFP<fdiv, "div", 137>;
 
 defm "" : SIMDNegInt<v16i8, "i8x16", splat16, i32, 36>;
 defm "" : SIMDNegInt<v8i16, "i16x8", splat8, i32, 37>;
 defm "" : SIMDNegInt<v4i32, "i32x4", splat4, i32, 38>;
 defm "" : SIMDNegInt<v2i64, "i64x2", splat2, i64, 39>;
 defm "" : SIMDNegFP<v4f32, "f32x4", splat4, f32, 125>;
 defm "" : SIMDNegFP<v2f64, "f64x2", splat2, f64, 126>;
 
 defm SHL : SIMDShiftInt<shl, "shl", 48, 0>;
 defm SHR_S : SIMDShiftInt<sra, "shr_s", 52, 1>;
 defm SHR_U : SIMDShiftInt<srl, "shr_u", 53, 1>;
 
 let isCommutable = 1 in {
 defm AND : SIMDBitwise<and, "and", 60>;
 defm OR : SIMDBitwise<or, "or", 61>;
 defm XOR : SIMDBitwise<xor, "xor", 62>;
 } // isCommutable = 1
 
 defm "" : SIMDNot<v16i8, splat16, i32>;
 defm "" : SIMDNot<v8i16, splat8, i32>;
 defm "" : SIMDNot<v4i32, splat4, i32>;
 defm "" : SIMDNot<v2i64, splat2, i64>;
 
+foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in
+defm "" : Bitselect<vec_t>;
+
 defm ANYTRUE : SIMDReduce<"any_true", wasm_anytrue, 65>;
 defm ALLTRUE : SIMDReduce<"all_true", wasm_alltrue, 69>;
 
 let isCommutable = 1 in {
 defm EQ : SIMDConditionInt<"eq", SETEQ, 73>;
 defm EQ : SIMDConditionFP<"eq", SETOEQ, 77>;
 defm NE : SIMDConditionInt<"ne", SETNE, 79>;
 defm NE : SIMDConditionFP<"ne", SETUNE, 83>;

 def : StorePatGlobalAddr<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
 def : StorePatExternalSym<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
 def : StorePatOffsetOnly<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
 def : StorePatGlobalAddrOffOnly<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
 def : StorePatExternSymOffOnly<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
 
 }
 
+// Bitselect is equivalent to (c & v1) | (~c & v2)
+foreach vec_t = [v16i8, v8i16, v4i32, v2i64] in
+  def : Pat<(vec_t (or (and (vec_t V128:$c), (vec_t V128:$v1)),
+              (and (vnot V128:$c), (vec_t V128:$v2)))),
+            (!cast<Instruction>("BITSELECT_"#vec_t)
+              V128:$v1, V128:$v2, V128:$c)>;
+
 // Lower float comparisons that don't care about NaN to standard
 // WebAssembly float comparisons. These instructions are generated in
 // the target-independent expansion of unordered comparisons and
 // ordered ne.
 def : Pat<(v4i32 (seteq (v4f32 V128:$lhs), (v4f32 V128:$rhs))),
           (v4i32 (EQ_v4f32 (v4f32 V128:$lhs), (v4f32 V128:$rhs)))>;
 def : Pat<(v4i32 (setne (v4f32 V128:$lhs), (v4f32 V128:$rhs))),
           (v4i32 (NE_v4f32 (v4f32 V128:$lhs), (v4f32 V128:$rhs)))>;

test/CodeGen/WebAssembly/simd-arith.ll

-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -wasm-enable-unimplemented-simd -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -wasm-enable-unimplemented-simd -mattr=+simd128 -fast-isel | FileCheck %s --check-prefixes CHECK,SIMD128
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128-VM
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 -fast-isel | FileCheck %s --check-prefixes CHECK,SIMD128-VM
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=-simd128 | FileCheck %s --check-prefixes CHECK,NO-SIMD128
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=-simd128 -fast-isel | FileCheck %s --check-prefixes CHECK,NO-SIMD128
+; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -wasm-enable-unimplemented-simd -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128,SIMD128-SLOW
+; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -wasm-enable-unimplemented-simd -mattr=+simd128 -fast-isel | FileCheck %s --check-prefixes CHECK,SIMD128,SIMD128-FAST
+; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128-VM,SIMD128-VM-SLOW
+; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 -fast-isel | FileCheck %s --check-prefixes CHECK,SIMD128-VM,SIMD128-VM-FAST
+; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=-simd128 | FileCheck %s --check-prefixes CHECK,NO-SIMD128,NO-SIMD128-SLOW
+; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=-simd128 -fast-isel | FileCheck %s --check-prefixes CHECK,NO-SIMD128,NO-SIMD128-FAST
 
 ; Test that basic SIMD128 arithmetic operations assemble as expected.
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
 ; ==============================================================================
 ; 16 x i8

 define <16 x i8> @not_v16i8(<16 x i8> %x) {
   %a = xor <16 x i8> %x, <i8 -1, i8 -1, i8 -1, i8 -1,
                           i8 -1, i8 -1, i8 -1, i8 -1,
                           i8 -1, i8 -1, i8 -1, i8 -1,
                           i8 -1, i8 -1, i8 -1, i8 -1>
   ret <16 x i8> %a
 }
 
+; CHECK-LABEL: bitselect_v16i8:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .param v128, v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-SLOW-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $1, $2, $0{{$}}
+; SIMD128-SLOW-NEXT: return $pop[[R]]{{$}}
+; SIMD128-FAST-NEXT: v128.and
+; SIMD128-FAST-NEXT: v128.not
+; SIMD128-FAST-NEXT: v128.and
+; SIMD128-FAST-NEXT: v128.or
+; SIMD128-FAST-NEXT: return
+define <16 x i8> @bitselect_v16i8(<16 x i8> %c, <16 x i8> %v1, <16 x i8> %v2) {
+  %masked_v1 = and <16 x i8> %c, %v1
+  %inv_mask = xor <16 x i8> %c,
+    <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1,
+     i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %masked_v2 = and <16 x i8> %inv_mask, %v2
+  %a = or <16 x i8> %masked_v1, %masked_v2
+  ret <16 x i8> %a
+}
+
 ; ==============================================================================
 ; 8 x i16
 ; ==============================================================================
 ; CHECK-LABEL: add_v8i16:
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .param v128, v128{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: i16x8.add $push[[R:[0-9]+]]=, $0, $1{{$}}

 ; SIMD128-NEXT: v128.not $push[[R:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @not_v8i16(<8 x i16> %x) {
   %a = xor <8 x i16> %x, <i16 -1, i16 -1, i16 -1, i16 -1,
                           i16 -1, i16 -1, i16 -1, i16 -1>
   ret <8 x i16> %a
 }
 
+; CHECK-LABEL: bitselect_v8i16:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .param v128, v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-SLOW-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $1, $2, $0{{$}}
+; SIMD128-SLOW-NEXT: return $pop[[R]]{{$}}
+; SIMD128-FAST-NEXT: v128.and
+; SIMD128-FAST-NEXT: v128.not
+; SIMD128-FAST-NEXT: v128.and
+; SIMD128-FAST-NEXT: v128.or
+; SIMD128-FAST-NEXT: return
+define <8 x i16> @bitselect_v8i16(<8 x i16> %c, <8 x i16> %v1, <8 x i16> %v2) {
+  %masked_v1 = and <8 x i16> %v1, %c
+  %inv_mask = xor <8 x i16>
+    <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>,
+    %c
+  %masked_v2 = and <8 x i16> %v2, %inv_mask
+  %a = or <8 x i16> %masked_v1, %masked_v2
+  ret <8 x i16> %a
+}
+
 ; ==============================================================================
 ; 4 x i32
 ; ==============================================================================
 ; CHECK-LABEL: add_v4i32:
 ; NO-SIMD128-NOT: i32x4
 ; SIMD128-NEXT: .param v128, v128{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: i32x4.add $push[[R:[0-9]+]]=, $0, $1{{$}}

 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: v128.not $push[[R:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @not_v4i32(<4 x i32> %x) {
   %a = xor <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
   ret <4 x i32> %a
 }
 
+; CHECK-LABEL: bitselect_v4i32:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .param v128, v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-SLOW-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $1, $2, $0{{$}}
+; SIMD128-SLOW-NEXT: return $pop[[R]]{{$}}
+; SIMD128-FAST-NEXT: v128.not
+; SIMD128-FAST-NEXT: v128.and
+; SIMD128-FAST-NEXT: v128.and
+; SIMD128-FAST-NEXT: v128.or
+; SIMD128-FAST-NEXT: return
+define <4 x i32> @bitselect_v4i32(<4 x i32> %c, <4 x i32> %v1, <4 x i32> %v2) {
+  %masked_v1 = and <4 x i32> %c, %v1
+  %inv_mask = xor <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, %c
+  %masked_v2 = and <4 x i32> %inv_mask, %v2
+  %a = or <4 x i32> %masked_v2, %masked_v1
+  ret <4 x i32> %a
+}
+
 ; ==============================================================================
 ; 2 x i64
 ; ==============================================================================
 ; CHECK-LABEL: add_v2i64:
 ; NO-SIMD128-NOT: i64x2
 ; SIMD128-VM-NOT: i64x2
 ; SIMD128-NEXT: .param v128, v128{{$}}
 ; SIMD128-NEXT: .result v128{{$}}

 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: v128.not $push[[R:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @not_v2i64(<2 x i64> %x) {
   %a = xor <2 x i64> %x, <i64 -1, i64 -1>
   ret <2 x i64> %a
 }
 
+; CHECK-LABEL: bitselect_v2i64:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .param v128, v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-SLOW-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $1, $2, $0{{$}}
+; SIMD128-SLOW-NEXT: return $pop[[R]]{{$}}
+; SIMD128-FAST-NEXT: v128.not
+; SIMD128-FAST-NEXT: v128.and
+; SIMD128-FAST-NEXT: v128.and
+; SIMD128-FAST-NEXT: v128.or
+; SIMD128-FAST-NEXT: return
+define <2 x i64> @bitselect_v2i64(<2 x i64> %c, <2 x i64> %v1, <2 x i64> %v2) {
+  %masked_v1 = and <2 x i64> %v1, %c
+  %inv_mask = xor <2 x i64> <i64 -1, i64 -1>, %c
+  %masked_v2 = and <2 x i64> %v2, %inv_mask
+  %a = or <2 x i64> %masked_v2, %masked_v1
+  ret <2 x i64> %a
+}
+
 ; ==============================================================================
 ; 4 x float
 ; ==============================================================================
 ; CHECK-LABEL: neg_v4f32:
 ; NO-SIMD128-NOT: f32x4
 ; SIMD128-NEXT: .param v128{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: f32x4.neg $push[[R:[0-9]+]]=, $0{{$}}

 ; SIMD128-NEXT: f64x2.abs $push[[R:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 declare <2 x double> @llvm.fabs.v2f64(<2 x double>) nounwind readnone
 define <2 x double> @abs_v2f64(<2 x double> %x) {
   %a = call <2 x double> @llvm.fabs.v2f64(<2 x double> %x)
   ret <2 x double> %a
 }
 
-
 ; CHECK-LABEL: add_v2f64:
 ; NO-SIMD128-NOT: f64x2
 ; SIMD128-VM-NOT: f62x2
 ; SIMD128-NEXT: .param v128, v128{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: f64x2.add $push[[R:[0-9]+]]=, $0, $1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @add_v2f64(<2 x double> %x, <2 x double> %y) {

test/CodeGen/WebAssembly/simd-intrinsics.ll

 ; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -wasm-enable-unimplemented-simd -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
 ; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -wasm-enable-unimplemented-simd -mattr=+simd128 -fast-isel | FileCheck %s --check-prefixes CHECK,SIMD128
 
 ; Test that SIMD128 intrinsics lower as expected. These intrinsics are
 ; only expected to lower successfully if the simd128 attribute is
 ; enabled and legal types are used.
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
 ; ==============================================================================
 ; 16 x i8
 ; ==============================================================================
+; CHECK-LABEL: add_sat_s_v16i8:
+; SIMD128-NEXT: .param v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: i8x16.add_saturate_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <16 x i8> @llvm.wasm.add.saturate.signed.v16i8(<16 x i8>, <16 x i8>)
+define <16 x i8> @add_sat_s_v16i8(<16 x i8> %x, <16 x i8> %y) {
+  %a = call <16 x i8> @llvm.wasm.add.saturate.signed.v16i8(
+    <16 x i8> %x, <16 x i8> %y
+  )
+  ret <16 x i8> %a
+}
+
+; CHECK-LABEL: add_sat_u_v16i8:
+; SIMD128-NEXT: .param v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: i8x16.add_saturate_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <16 x i8> @llvm.wasm.add.saturate.unsigned.v16i8(<16 x i8>, <16 x i8>)
+define <16 x i8> @add_sat_u_v16i8(<16 x i8> %x, <16 x i8> %y) {
+  %a = call <16 x i8> @llvm.wasm.add.saturate.unsigned.v16i8(
+    <16 x i8> %x, <16 x i8> %y
+  )
+  ret <16 x i8> %a
+}
+
+; CHECK-LABEL: sub_sat_s_v16i8:
+; SIMD128-NEXT: .param v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: i8x16.sub_saturate_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <16 x i8> @llvm.wasm.sub.saturate.signed.v16i8(<16 x i8>, <16 x i8>)
+define <16 x i8> @sub_sat_s_v16i8(<16 x i8> %x, <16 x i8> %y) {
+  %a = call <16 x i8> @llvm.wasm.sub.saturate.signed.v16i8(
+    <16 x i8> %x, <16 x i8> %y
+  )
+  ret <16 x i8> %a
+}
+
+; CHECK-LABEL: sub_sat_u_v16i8:
+; SIMD128-NEXT: .param v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: i8x16.sub_saturate_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <16 x i8> @llvm.wasm.sub.saturate.unsigned.v16i8(<16 x i8>, <16 x i8>)
+define <16 x i8> @sub_sat_u_v16i8(<16 x i8> %x, <16 x i8> %y) {
+  %a = call <16 x i8> @llvm.wasm.sub.saturate.unsigned.v16i8(
+    <16 x i8> %x, <16 x i8> %y
+  )
+  ret <16 x i8> %a
+}
+
 ; CHECK-LABEL: any_v16i8:
 ; SIMD128-NEXT: .param v128{{$}}
 ; SIMD128-NEXT: .result i32{{$}}
 ; SIMD128-NEXT: i8x16.any_true $push[[R:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.anytrue.v16i8(<16 x i8>)
 define i32 @any_v16i8(<16 x i8> %x) {
   %a = call i32 @llvm.wasm.anytrue.v16i8(<16 x i8> %x)
   ret i32 %a
 }
 
 ; CHECK-LABEL: all_v16i8:
 ; SIMD128-NEXT: .param v128{{$}}
 ; SIMD128-NEXT: .result i32{{$}}
 ; SIMD128-NEXT: i8x16.all_true $push[[R:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.alltrue.v16i8(<16 x i8>)
 define i32 @all_v16i8(<16 x i8> %x) {
   %a = call i32 @llvm.wasm.alltrue.v16i8(<16 x i8> %x)
   ret i32 %a
 }
 
+; CHECK-LABEL: bitselect_v16i8:
+; SIMD128-NEXT: .param v128, v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $1, $2, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <16 x i8> @llvm.wasm.bitselect.v16i8(<16 x i8>, <16 x i8>, <16 x i8>)
+define <16 x i8> @bitselect_v16i8(<16 x i8> %c, <16 x i8> %v1, <16 x i8> %v2) {
+  %a = call <16 x i8> @llvm.wasm.bitselect.v16i8(
+    <16 x i8> %c, <16 x i8> %v1, <16 x i8> %v2
+  )
+  ret <16 x i8> %a
+}
+
 ; ==============================================================================
 ; 8 x i16
 ; ==============================================================================
+; CHECK-LABEL: add_sat_s_v8i16:
+; SIMD128-NEXT: .param v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: i16x8.add_saturate_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.add.saturate.signed.v8i16(<8 x i16>, <8 x i16>)
+define <8 x i16> @add_sat_s_v8i16(<8 x i16> %x, <8 x i16> %y) {
+  %a = call <8 x i16> @llvm.wasm.add.saturate.signed.v8i16(
+    <8 x i16> %x, <8 x i16> %y
+  )
+  ret <8 x i16> %a
+}
+
+; CHECK-LABEL: add_sat_u_v8i16:
+; SIMD128-NEXT: .param v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: i16x8.add_saturate_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.add.saturate.unsigned.v8i16(<8 x i16>, <8 x i16>)
+define <8 x i16> @add_sat_u_v8i16(<8 x i16> %x, <8 x i16> %y) {
+  %a = call <8 x i16> @llvm.wasm.add.saturate.unsigned.v8i16(
+    <8 x i16> %x, <8 x i16> %y
+  )
+  ret <8 x i16> %a
+}
+
+; CHECK-LABEL: sub_sat_s_v8i16:
+; SIMD128-NEXT: .param v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: i16x8.sub_saturate_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.sub.saturate.signed.v8i16(<8 x i16>, <8 x i16>)
+define <8 x i16> @sub_sat_s_v8i16(<8 x i16> %x, <8 x i16> %y) {
+  %a = call <8 x i16> @llvm.wasm.sub.saturate.signed.v8i16(
+    <8 x i16> %x, <8 x i16> %y
+  )
+  ret <8 x i16> %a
+}
+
+; CHECK-LABEL: sub_sat_u_v8i16:
+; SIMD128-NEXT: .param v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: i16x8.sub_saturate_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.sub.saturate.unsigned.v8i16(<8 x i16>, <8 x i16>)
+define <8 x i16> @sub_sat_u_v8i16(<8 x i16> %x, <8 x i16> %y) {
+  %a = call <8 x i16> @llvm.wasm.sub.saturate.unsigned.v8i16(
+    <8 x i16> %x, <8 x i16> %y
+  )
+  ret <8 x i16> %a
+}
+
 ; CHECK-LABEL: any_v8i16:
 ; SIMD128-NEXT: .param v128{{$}}
 ; SIMD128-NEXT: .result i32{{$}}
 ; SIMD128-NEXT: i16x8.any_true $push[[R:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.anytrue.v8i16(<8 x i16>)
 define i32 @any_v8i16(<8 x i16> %x) {
   %a = call i32 @llvm.wasm.anytrue.v8i16(<8 x i16> %x)
   ret i32 %a
 }
 
 ; CHECK-LABEL: all_v8i16:
 ; SIMD128-NEXT: .param v128{{$}}
 ; SIMD128-NEXT: .result i32{{$}}
 ; SIMD128-NEXT: i16x8.all_true $push[[R:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.alltrue.v8i16(<8 x i16>)
 define i32 @all_v8i16(<8 x i16> %x) {
   %a = call i32 @llvm.wasm.alltrue.v8i16(<8 x i16> %x)
   ret i32 %a
 }
 
+; CHECK-LABEL: bitselect_v8i16:
+; SIMD128-NEXT: .param v128, v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $1, $2, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.bitselect.v8i16(<8 x i16>, <8 x i16>, <8 x i16>)
+define <8 x i16> @bitselect_v8i16(<8 x i16> %c, <8 x i16> %v1, <8 x i16> %v2) {
+  %a = call <8 x i16> @llvm.wasm.bitselect.v8i16(
+    <8 x i16> %c, <8 x i16> %v1, <8 x i16> %v2
+  )
+  ret <8 x i16> %a
+}
+
 ; ==============================================================================
 ; 4 x i32
 ; ==============================================================================
 ; CHECK-LABEL: any_v4i32:
 ; SIMD128-NEXT: .param v128{{$}}
 ; SIMD128-NEXT: .result i32{{$}}
 ; SIMD128-NEXT: i32x4.any_true $push[[R:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}

 ; SIMD128-NEXT: i32x4.all_true $push[[R:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.alltrue.v4i32(<4 x i32>)
 define i32 @all_v4i32(<4 x i32> %x) {
   %a = call i32 @llvm.wasm.alltrue.v4i32(<4 x i32> %x)
   ret i32 %a
 }
 
+; CHECK-LABEL: bitselect_v4i32:
+; SIMD128-NEXT: .param v128, v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $1, $2, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <4 x i32> @llvm.wasm.bitselect.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
+define <4 x i32> @bitselect_v4i32(<4 x i32> %c, <4 x i32> %v1, <4 x i32> %v2) {
+  %a = call <4 x i32> @llvm.wasm.bitselect.v4i32(
+    <4 x i32> %c, <4 x i32> %v1, <4 x i32> %v2
+  )
+  ret <4 x i32> %a
+}
+
 ; ==============================================================================
 ; 2 x i64
 ; ==============================================================================
 ; CHECK-LABEL: any_v2i64:
 ; SIMD128-NEXT: .param v128{{$}}
 ; SIMD128-NEXT: .result i32{{$}}
 ; SIMD128-NEXT: i64x2.any_true $push[[R:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.anytrue.v2i64(<2 x i64>)
 define i32 @any_v2i64(<2 x i64> %x) {
   %a = call i32 @llvm.wasm.anytrue.v2i64(<2 x i64> %x)
   ret i32 %a
 }
 
 ; CHECK-LABEL: all_v2i64:
 ; SIMD128-NEXT: .param v128{{$}}
 ; SIMD128-NEXT: .result i32{{$}}
 ; SIMD128-NEXT: i64x2.all_true $push[[R:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.alltrue.v2i64(<2 x i64>)
 define i32 @all_v2i64(<2 x i64> %x) {
   %a = call i32 @llvm.wasm.alltrue.v2i64(<2 x i64> %x)
   ret i32 %a
 }
+
+; CHECK-LABEL: bitselect_v2i64:
+; SIMD128-NEXT: .param v128, v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $1, $2, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <2 x i64> @llvm.wasm.bitselect.v2i64(<2 x i64>, <2 x i64>, <2 x i64>)
+define <2 x i64> @bitselect_v2i64(<2 x i64> %c, <2 x i64> %v1, <2 x i64> %v2) {
+  %a = call <2 x i64> @llvm.wasm.bitselect.v2i64(
+    <2 x i64> %c, <2 x i64> %v1, <2 x i64> %v2
+  )
+  ret <2 x i64> %a
+}
+
+; ==============================================================================
+; 4 x f32
+; ==============================================================================
+; CHECK-LABEL: bitselect_v4f32:
+; SIMD128-NEXT: .param v128, v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $1, $2, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <4 x float> @llvm.wasm.bitselect.v4f32(<4 x float>, <4 x float>, <4 x float>)
+define <4 x float> @bitselect_v4f32(<4 x float> %c, <4 x float> %v1, <4 x float> %v2) {
+  %a = call <4 x float> @llvm.wasm.bitselect.v4f32(
+    <4 x float> %c, <4 x float> %v1, <4 x float> %v2
+  )
+  ret <4 x float> %a
+}
+
+; ==============================================================================
+; 2 x f64
+; ==============================================================================
+; CHECK-LABEL: bitselect_v2f64:
+; SIMD128-NEXT: .param v128, v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $1, $2, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <2 x double> @llvm.wasm.bitselect.v2f64(<2 x double>, <2 x double>, <2 x double>)
+define <2 x double> @bitselect_v2f64(<2 x double> %c, <2 x double> %v1, <2 x double> %v2) {
+  %a = call <2 x double> @llvm.wasm.bitselect.v2f64(
+    <2 x double> %c, <2 x double> %v1, <2 x double> %v2
+  )
+  ret <2 x double> %a
+}

test/MC/WebAssembly/simd-encodings.s

# RUN: llvm-mc -show-encoding -triple=wasm32-unkown-unknown -mattr=+sign-ext,+simd128 < %s | FileCheck %s
     i16x8.neg
 
     # CHECK: i32x4.neg # encoding: [0xfd,0x26]
     i32x4.neg
 
     # CHECK: i64x2.neg # encoding: [0xfd,0x27]
     i64x2.neg
 
+    # CHECK: i8x16.add_saturate_s # encoding: [0xfd,0x28]
+    i8x16.add_saturate_s
+
+    # CHECK: i8x16.add_saturate_u # encoding: [0xfd,0x29]
+    i8x16.add_saturate_u
+
+    # CHECK: i16x8.add_saturate_s # encoding: [0xfd,0x2a]
+    i16x8.add_saturate_s
+
+    # CHECK: i16x8.add_saturate_u # encoding: [0xfd,0x2b]
+    i16x8.add_saturate_u
+
+    # CHECK: i8x16.sub_saturate_s # encoding: [0xfd,0x2c]
+    i8x16.sub_saturate_s
+
+    # CHECK: i8x16.sub_saturate_u # encoding: [0xfd,0x2d]
+    i8x16.sub_saturate_u
+
+    # CHECK: i16x8.sub_saturate_s # encoding: [0xfd,0x2e]
+    i16x8.sub_saturate_s
+
+    # CHECK: i16x8.sub_saturate_u # encoding: [0xfd,0x2f]
+    i16x8.sub_saturate_u
+
     # CHECK: i8x16.shl # encoding: [0xfd,0x30]
     i8x16.shl
 
     # CHECK: i16x8.shl # encoding: [0xfd,0x31]
     i16x8.shl
 
     # CHECK: i32x4.shl # encoding: [0xfd,0x32]
     i32x4.shl
# RUN: llvm-mc -show-encoding -triple=wasm32-unkown-unknown -mattr=+sign-ext,+simd128 < %s | FileCheck %s
     v128.or
 
     # CHECK: v128.xor # encoding: [0xfd,0x3e]
     v128.xor
 
     # CHECK: v128.not # encoding: [0xfd,0x3f]
     v128.not
 
+    # CHECK: v128.bitselect # encoding: [0xfd,0x40]
+    v128.bitselect
+
     # CHECK: i8x16.any_true # encoding: [0xfd,0x41]
     i8x16.any_true
 
     # CHECK: i16x8.any_true # encoding: [0xfd,0x42]
     i16x8.any_true
 
     # CHECK: i32x4.any_true # encoding: [0xfd,0x43]
     i32x4.any_true