[WebAssembly] Handle undefined lane indices in SIMD patterns

[tlively]

Summary

Undefined indices in shuffles can be used when not all lanes of the output vector will be used. This happens for example in the expansion of vector reduce operations. Regardless, undefs are legal as lane indices in IR and should be supported.

[aheejin]

Could you give an example of what kind of programs would generate undefs in the IR and why we need this? And also I guess it's good to include that in the CL and the commit description.

[tlively]

• Rebase and update summary in commit message

[tlively]

In D53057#1261175, @aheejin wrote:

Could you give an example of what kind of programs would generate undefs in the IR and why we need this? And also I guess it's good to include that in the CL and the commit description.

Done. See new description.

[aheejin]

lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

1032

Why -1? Is it how undef is represented?

lib/Target/WebAssembly/WebAssemblyInstrSIMD.td

194

Are these unnecessary sequence of masking by and also removed in non-undef cases?

202

Here the same. Are these sext_inreg also removed in non-undef cases?

232

Move this pattern to extract_lane category above

test/CodeGen/WebAssembly/simd.ll

133

i32 undef? For all other replace_undef_ functions too.

[aheejin]

lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

1032

I think MaskVal sounds confusing. It sounds like it is a mask value (such as 0xff). Can we change it to something else, like, just Val or something?

[tlively]

lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

1032

ByteIndex seemed like a nice descriptive name.

1032

Yes, that seems to be how undef is represented.

lib/Target/WebAssembly/WebAssemblyInstrSIMD.td

194

Yes, see multiclass ExtractLaneExtended, which uses the corresponding patterns in multiclass ExtractPat.

202

Yes, same as above.

[tlively]

• Address comments

[aheejin]

LGTM with a nit.

lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

1032

Then one-line comment stating that here -1 means undef would be helpful I think.

lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
   SDValue Ops[18];
   size_t OpIdx = 0;
   Ops[OpIdx++] = Op.getOperand(0);
   Ops[OpIdx++] = Op.getOperand(1);
 
   // Expand mask indices to byte indices and materialize them as operands
   for (size_t I = 0, Lanes = Mask.size(); I < Lanes; ++I) {
     for (size_t J = 0; J < LaneBytes; ++J) {
-      Ops[OpIdx++] =
-          DAG.getConstant((uint64_t)Mask[I] * LaneBytes + J, DL, MVT::i32);
+      // Lower undefs to zero
+      uint64_t MaskVal = Mask[I] == -1 ? 0 : (uint64_t)Mask[I] * LaneBytes + J;
+      Ops[OpIdx++] = DAG.getConstant(MaskVal, DL, MVT::i32);
     }
   }
 
   return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, MVT::v16i8, Ops);
 }
 
 //===----------------------------------------------------------------------===//
 //                          WebAssembly Optimization Hooks
 //===----------------------------------------------------------------------===//

lib/Target/WebAssembly/WebAssemblyInstrSIMD.td

 defm "" : ExtractLane<v2f64, "f64x2", LaneIdx2, F64, 16>;
 
 // Follow convention of making implicit expansions unsigned
 def : Pat<(i32 (vector_extract (v16i8 V128:$vec), (i32 LaneIdx16:$idx))),
           (EXTRACT_LANE_v16i8_u V128:$vec, (i32 LaneIdx16:$idx))>;
 def : Pat<(i32 (vector_extract (v8i16 V128:$vec), (i32 LaneIdx8:$idx))),
           (EXTRACT_LANE_v8i16_u V128:$vec, (i32 LaneIdx8:$idx))>;
 
+// Lower undef lane indices to zero
+def : Pat<(and (i32 (vector_extract (v16i8 V128:$vec), undef)), (i32 0xff)),
+          (EXTRACT_LANE_v16i8_u V128:$vec, 0)>;
+def : Pat<(and (i32 (vector_extract (v8i16 V128:$vec), undef)), (i32 0xffff)),
+          (EXTRACT_LANE_v8i16_u V128:$vec, 0)>;
+def : Pat<(i32 (vector_extract (v16i8 V128:$vec), undef)),
+          (EXTRACT_LANE_v16i8_u V128:$vec, 0)>;
+def : Pat<(i32 (vector_extract (v8i16 V128:$vec), undef)),
+          (EXTRACT_LANE_v8i16_u V128:$vec, 0)>;
+def : Pat<(sext_inreg (i32 (vector_extract (v16i8 V128:$vec), undef)), i8),
+          (EXTRACT_LANE_v16i8_s V128:$vec, 0)>;
+def : Pat<(sext_inreg (i32 (vector_extract (v8i16 V128:$vec), undef)), i16),
+          (EXTRACT_LANE_v8i16_s V128:$vec, 0)>;
+def : Pat<(vector_extract (v4i32 V128:$vec), undef),
+          (EXTRACT_LANE_v4i32 V128:$vec, 0)>;
+def : Pat<(vector_extract (v2i64 V128:$vec), undef),
+          (EXTRACT_LANE_v2i64 V128:$vec, 0)>;
+def : Pat<(vector_extract (v4f32 V128:$vec), undef),
+          (EXTRACT_LANE_v4f32 V128:$vec, 0)>;
+
 // Replace lane value: replace_lane
 multiclass ReplaceLane<ValueType vec_t, string vec, ImmLeaf imm_t,
                        WebAssemblyRegClass reg_t, ValueType lane_t,
                        bits<32> simdop> {
   defm REPLACE_LANE_#vec_t :
       SIMD_I<(outs V128:$dst), (ins V128:$vec, vec_i8imm_op:$idx, reg_t:$x),
              (outs), (ins vec_i8imm_op:$idx),
              [(set V128:$dst, (vector_insert
                (vec_t V128:$vec), (lane_t reg_t:$x), (i32 imm_t:$idx)))],
              vec#".replace_lane\t$dst, $vec, $idx, $x",
              vec#".replace_lane\t$idx", simdop>;
 }
 
 defm "" : ReplaceLane<v16i8, "i8x16", LaneIdx16, I32, i32, 17>;
 defm "" : ReplaceLane<v8i16, "i16x8", LaneIdx8, I32, i32, 18>;
 defm "" : ReplaceLane<v4i32, "i32x4", LaneIdx4, I32, i32, 19>;
 defm "" : ReplaceLane<v2i64, "i64x2", LaneIdx2, I64, i64, 20>;
 defm "" : ReplaceLane<v4f32, "f32x4", LaneIdx4, F32, f32, 21>;
 defm "" : ReplaceLane<v2f64, "f64x2", LaneIdx2, F64, f64, 22>;
 
+// Lower undef lane indices to zero
+def : Pat<(vector_extract (v2f64 V128:$vec), undef),
+          (EXTRACT_LANE_v2f64 V128:$vec, 0)>;
+def : Pat<(vector_insert (v16i8 V128:$vec), I32:$x, undef),
+          (REPLACE_LANE_v16i8 V128:$vec, 0, I32:$x)>;
+def : Pat<(vector_insert (v8i16 V128:$vec), I32:$x, undef),
+          (REPLACE_LANE_v8i16 V128:$vec, 0, I32:$x)>;
+def : Pat<(vector_insert (v4i32 V128:$vec), I32:$x, undef),
+          (REPLACE_LANE_v4i32 V128:$vec, 0, I32:$x)>;
+def : Pat<(vector_insert (v2i64 V128:$vec), I64:$x, undef),
+          (REPLACE_LANE_v2i64 V128:$vec, 0, I64:$x)>;
+def : Pat<(vector_insert (v4f32 V128:$vec), F32:$x, undef),
+          (REPLACE_LANE_v4f32 V128:$vec, 0, F32:$x)>;
+def : Pat<(vector_insert (v2f64 V128:$vec), F64:$x, undef),
+          (REPLACE_LANE_v2f64 V128:$vec, 0, F64:$x)>;
+
 // Arbitrary other BUILD_VECTOR patterns
 def : Pat<(v16i8 (build_vector
             (i32 I32:$x0), (i32 I32:$x1), (i32 I32:$x2), (i32 I32:$x3),
             (i32 I32:$x4), (i32 I32:$x5), (i32 I32:$x6), (i32 I32:$x7),
             (i32 I32:$x8), (i32 I32:$x9), (i32 I32:$x10), (i32 I32:$x11),
             (i32 I32:$x12), (i32 I32:$x13), (i32 I32:$x14), (i32 I32:$x15)
           )),
           (v16i8 (REPLACE_LANE_v16i8

test/CodeGen/WebAssembly/simd.ll

 ; SIMD128-NEXT: i8x16.extract_lane_s $push[[R:[0-9]+]]=, $0, 13{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define i32 @extract_v16i8_s(<16 x i8> %v) {
   %elem = extractelement <16 x i8> %v, i8 13
   %a = sext i8 %elem to i32
   ret i32 %a
 }
 
+; CHECK-LABEL: extract_undef_v16i8_s:
+; NO-SIMD128-NOT: i8x16
+; SIMD128-NEXT: .param v128{{$}}
+; SIMD128-NEXT: .result i32{{$}}
+; SIMD128-NEXT: i8x16.extract_lane_s $push[[R:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define i32 @extract_undef_v16i8_s(<16 x i8> %v) {
+  %elem = extractelement <16 x i8> %v, i8 undef
+  %a = sext i8 %elem to i32
+  ret i32 %a
+}
+
 ; CHECK-LABEL: extract_v16i8_u:
 ; NO-SIMD128-NOT: i8x16
 ; SIMD128-NEXT: .param v128{{$}}
 ; SIMD128-NEXT: .result i32{{$}}
 ; SIMD128-NEXT: i8x16.extract_lane_u $push[[R:[0-9]+]]=, $0, 13{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define i32 @extract_v16i8_u(<16 x i8> %v) {
   %elem = extractelement <16 x i8> %v, i8 13
   %a = zext i8 %elem to i32
   ret i32 %a
 }
 
+; CHECK-LABEL: extract_undef_v16i8_u:
+; NO-SIMD128-NOT: i8x16
+; SIMD128-NEXT: .param v128{{$}}
+; SIMD128-NEXT: .result i32{{$}}
+; SIMD128-NEXT: i8x16.extract_lane_u $push[[R:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define i32 @extract_undef_v16i8_u(<16 x i8> %v) {
+  %elem = extractelement <16 x i8> %v, i8 undef
+  %a = zext i8 %elem to i32
+  ret i32 %a
+}
+
 ; CHECK-LABEL: extract_v16i8:
 ; NO-SIMD128-NOT: i8x16
 ; SIMD128-NEXT: .param v128{{$}}
 ; SIMD128-NEXT: .result i32{{$}}
 ; SIMD128-NEXT: i8x16.extract_lane_u $push[[R:[0-9]+]]=, $0, 13{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define i8 @extract_v16i8(<16 x i8> %v) {
   %elem = extractelement <16 x i8> %v, i8 13
   ret i8 %elem
 }
 
+; CHECK-LABEL: extract_undef_v16i8:
+; NO-SIMD128-NOT: i8x16
+; SIMD128-NEXT: .param v128{{$}}
+; SIMD128-NEXT: .result i32{{$}}
+; SIMD128-NEXT: i8x16.extract_lane_u $push[[R:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define i8 @extract_undef_v16i8(<16 x i8> %v) {
+  %elem = extractelement <16 x i8> %v, i8 undef
+  ret i8 %elem
+}
+
 ; CHECK-LABEL: replace_v16i8:
 ; NO-SIMD128-NOT: i8x16
 ; SIMD128-NEXT: .param v128, i32{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: i8x16.replace_lane $push[[R:[0-9]+]]=, $0, 11, $1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @replace_v16i8(<16 x i8> %v, i8 %x) {
   %res = insertelement <16 x i8> %v, i8 %x, i32 11
   ret <16 x i8> %res
 }
 
+; CHECK-LABEL: replace_undef_v16i8:
+; NO-SIMD128-NOT: i8x16
+; SIMD128-NEXT: .param v128, i32{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: i8x16.replace_lane $push[[R:[0-9]+]]=, $0, 0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @replace_undef_v16i8(<16 x i8> %v, i8 %x) {
+  %res = insertelement <16 x i8> %v, i8 %x, i32 0
+  ret <16 x i8> %res
+}
+
 ; CHECK-LABEL: shuffle_v16i8:
 ; NO-SIMD128-NOT: v8x16
 ; SIMD128-NEXT: .param v128, v128{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: v8x16.shuffle $push[[R:[0-9]+]]=, $0, $1,
 ; SIMD128-SAME: 0, 17, 2, 19, 4, 21, 6, 23, 8, 25, 10, 27, 12, 29, 14, 31{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @shuffle_v16i8(<16 x i8> %x, <16 x i8> %y) {
   %res = shufflevector <16 x i8> %x, <16 x i8> %y,
     <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 4, i32 21, i32 6, i32 23,
                 i32 8, i32 25, i32 10, i32 27, i32 12, i32 29, i32 14, i32 31>
   ret <16 x i8> %res
 }
 
+; CHECK-LABEL: shuffle_undef_v16i8:
+; NO-SIMD128-NOT: v8x16
+; SIMD128-NEXT: .param v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: v8x16.shuffle $push[[R:[0-9]+]]=, $0, $0,
+; SIMD128-SAME: 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @shuffle_undef_v16i8(<16 x i8> %x, <16 x i8> %y) {
+  %res = shufflevector <16 x i8> %x, <16 x i8> %y,
+    <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef,
+                i32 undef, i32 undef, i32 undef, i32 undef,
+                i32 undef, i32 undef, i32 undef, i32 undef,
+                i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <16 x i8> %res
+}
+
 ; CHECK-LABEL: build_v16i8:
 ; NO-SIMD128-NOT: i8x16
 ; SIMD128-NEXT: .param i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: i8x16.splat $push[[L0:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: i8x16.replace_lane $push[[L1:[0-9]+]]=, $pop[[L0]], 1, $1{{$}}
 ; SIMD128-NEXT: i8x16.replace_lane $push[[L2:[0-9]+]]=, $pop[[L1]], 2, $2{{$}}
 ; SIMD128-NEXT: i8x16.replace_lane $push[[L3:[0-9]+]]=, $pop[[L2]], 3, $3{{$}}

 ; SIMD128-NEXT: i16x8.extract_lane_s $push[[R:[0-9]+]]=, $0, 5{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define i32 @extract_v8i16_s(<8 x i16> %v) {
   %elem = extractelement <8 x i16> %v, i16 5
   %a = sext i16 %elem to i32
   ret i32 %a
 }
 
+; CHECK-LABEL: extract_undef_v8i16_s:
+; NO-SIMD128-NOT: i16x8
+; SIMD128-NEXT: .param v128{{$}}
+; SIMD128-NEXT: .result i32{{$}}
+; SIMD128-NEXT: i16x8.extract_lane_s $push[[R:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define i32 @extract_undef_v8i16_s(<8 x i16> %v) {
+  %elem = extractelement <8 x i16> %v, i16 undef
+  %a = sext i16 %elem to i32
+  ret i32 %a
+}
+
 ; CHECK-LABEL: extract_v8i16_u:
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .param v128{{$}}
 ; SIMD128-NEXT: .result i32{{$}}
 ; SIMD128-NEXT: i16x8.extract_lane_u $push[[R:[0-9]+]]=, $0, 5{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define i32 @extract_v8i16_u(<8 x i16> %v) {
   %elem = extractelement <8 x i16> %v, i16 5
   %a = zext i16 %elem to i32
   ret i32 %a
 }
 
+; CHECK-LABEL: extract_undef_v8i16_u:
+; NO-SIMD128-NOT: i16x8
+; SIMD128-NEXT: .param v128{{$}}
+; SIMD128-NEXT: .result i32{{$}}
+; SIMD128-NEXT: i16x8.extract_lane_u $push[[R:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define i32 @extract_undef_v8i16_u(<8 x i16> %v) {
+  %elem = extractelement <8 x i16> %v, i16 undef
+  %a = zext i16 %elem to i32
+  ret i32 %a
+}
+
 ; CHECK-LABEL: extract_v8i16:
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .param v128{{$}}
 ; SIMD128-NEXT: .result i32{{$}}
 ; SIMD128-NEXT: i16x8.extract_lane_u $push[[R:[0-9]+]]=, $0, 5{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define i16 @extract_v8i16(<8 x i16> %v) {
   %elem = extractelement <8 x i16> %v, i16 5
   ret i16 %elem
 }
 
+; CHECK-LABEL: extract_undef_v8i16:
+; NO-SIMD128-NOT: i16x8
+; SIMD128-NEXT: .param v128{{$}}
+; SIMD128-NEXT: .result i32{{$}}
+; SIMD128-NEXT: i16x8.extract_lane_u $push[[R:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define i16 @extract_undef_v8i16(<8 x i16> %v) {
+  %elem = extractelement <8 x i16> %v, i16 undef
+  ret i16 %elem
+}
+
 ; CHECK-LABEL: replace_v8i16:
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .param v128, i32{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: i16x8.replace_lane $push[[R:[0-9]+]]=, $0, 7, $1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @replace_v8i16(<8 x i16> %v, i16 %x) {
   %res = insertelement <8 x i16> %v, i16 %x, i32 7
   ret <8 x i16> %res
 }
 
+; CHECK-LABEL: replace_undef_v8i16:
+; NO-SIMD128-NOT: i16x8
+; SIMD128-NEXT: .param v128, i32{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: i16x8.replace_lane $push[[R:[0-9]+]]=, $0, 0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @replace_undef_v8i16(<8 x i16> %v, i16 %x) {
+  %res = insertelement <8 x i16> %v, i16 %x, i32 0
+  ret <8 x i16> %res
+}
+
 ; CHECK-LABEL: shuffle_v8i16:
 ; NO-SIMD128-NOT: v8x16
 ; SIMD128-NEXT: .param v128, v128{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: v8x16.shuffle $push[[R:[0-9]+]]=, $0, $1,
 ; SIMD128-SAME: 0, 1, 18, 19, 4, 5, 22, 23, 8, 9, 26, 27, 12, 13, 30, 31{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @shuffle_v8i16(<8 x i16> %x, <8 x i16> %y) {
   %res = shufflevector <8 x i16> %x, <8 x i16> %y,
     <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
   ret <8 x i16> %res
 }
 
+; CHECK-LABEL: shuffle_undef_v8i16:
+; NO-SIMD128-NOT: v8x16
+; SIMD128-NEXT: .param v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: v8x16.shuffle $push[[R:[0-9]+]]=, $0, $0,
+; SIMD128-SAME: 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @shuffle_undef_v8i16(<8 x i16> %x, <8 x i16> %y) {
+  %res = shufflevector <8 x i16> %x, <8 x i16> %y,
+    <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef,
+               i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i16> %res
+}
+
 ; CHECK-LABEL: build_v8i16:
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .param i32, i32, i32, i32, i32, i32, i32, i32{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: i16x8.splat $push[[L0:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: i16x8.replace_lane $push[[L1:[0-9]+]]=, $pop[[L0]], 1, $1{{$}}
 ; SIMD128-NEXT: i16x8.replace_lane $push[[L2:[0-9]+]]=, $pop[[L1]], 2, $2{{$}}
 ; SIMD128-NEXT: i16x8.replace_lane $push[[L3:[0-9]+]]=, $pop[[L2]], 3, $3{{$}}

 ; SIMD128-NEXT: .result i32{{$}}
 ; SIMD128-NEXT: i32x4.extract_lane $push[[R:[0-9]+]]=, $0, 3{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define i32 @extract_v4i32(<4 x i32> %v) {
   %elem = extractelement <4 x i32> %v, i32 3
   ret i32 %elem
 }
 
+; CHECK-LABEL: extract_undef_v4i32:
+; NO-SIMD128-NOT: i32x4
+; SIMD128-NEXT: .param v128{{$}}
+; SIMD128-NEXT: .result i32{{$}}
+; SIMD128-NEXT: i32x4.extract_lane $push[[R:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define i32 @extract_undef_v4i32(<4 x i32> %v) {
+  %elem = extractelement <4 x i32> %v, i32 undef
+  ret i32 %elem
+}
+
 ; CHECK-LABEL: replace_v4i32:
 ; NO-SIMD128-NOT: i32x4
 ; SIMD128-NEXT: .param v128, i32{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: i32x4.replace_lane $push[[R:[0-9]+]]=, $0, 2, $1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @replace_v4i32(<4 x i32> %v, i32 %x) {
   %res = insertelement <4 x i32> %v, i32 %x, i32 2
   ret <4 x i32> %res
 }
 
+; CHECK-LABEL: replace_undef_v4i32:
+; NO-SIMD128-NOT: i32x4
+; SIMD128-NEXT: .param v128, i32{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: i32x4.replace_lane $push[[R:[0-9]+]]=, $0, 0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @replace_undef_v4i32(<4 x i32> %v, i32 %x) {
+  %res = insertelement <4 x i32> %v, i32 %x, i32 0
+  ret <4 x i32> %res
+}
+
 ; CHECK-LABEL: shuffle_v4i32:
 ; NO-SIMD128-NOT: v8x16
 ; SIMD128-NEXT: .param v128, v128{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: v8x16.shuffle $push[[R:[0-9]+]]=, $0, $1,
 ; SIMD128-SAME: 0, 1, 2, 3, 20, 21, 22, 23, 8, 9, 10, 11, 28, 29, 30, 31{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @shuffle_v4i32(<4 x i32> %x, <4 x i32> %y) {
   %res = shufflevector <4 x i32> %x, <4 x i32> %y,
     <4 x i32> <i32 0, i32 5, i32 2, i32 7>
   ret <4 x i32> %res
 }
 
+; CHECK-LABEL: shuffle_undef_v4i32:
+; NO-SIMD128-NOT: v8x16
+; SIMD128-NEXT: .param v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: v8x16.shuffle $push[[R:[0-9]+]]=, $0, $0,
+; SIMD128-SAME: 4, 5, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @shuffle_undef_v4i32(<4 x i32> %x, <4 x i32> %y) {
+  %res = shufflevector <4 x i32> %x, <4 x i32> %y,
+    <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  ret <4 x i32> %res
+}
+
 ; CHECK-LABEL: build_v4i32:
 ; NO-SIMD128-NOT: i32x4
 ; SIMD128-NEXT: .param i32, i32, i32, i32{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: i32x4.splat $push[[L0:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: i32x4.replace_lane $push[[L1:[0-9]+]]=, $pop[[L0]], 1, $1{{$}}
 ; SIMD128-NEXT: i32x4.replace_lane $push[[L2:[0-9]+]]=, $pop[[L1]], 2, $2{{$}}
 ; SIMD128-NEXT: i32x4.replace_lane $push[[R:[0-9]+]]=, $pop[[L2]], 3, $3{{$}}

 ; SIMD128-NEXT: .result i64{{$}}
 ; SIMD128-NEXT: i64x2.extract_lane $push[[R:[0-9]+]]=, $0, 1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define i64 @extract_v2i64(<2 x i64> %v) {
   %elem = extractelement <2 x i64> %v, i64 1
   ret i64 %elem
 }
 
+; CHECK-LABEL: extract_undef_v2i64:
+; NO-SIMD128-NOT: i64x2
+; SIMD128-VM-NOT: i64x2
+; SIMD128-NEXT: .param v128{{$}}
+; SIMD128-NEXT: .result i64{{$}}
+; SIMD128-NEXT: i64x2.extract_lane $push[[R:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define i64 @extract_undef_v2i64(<2 x i64> %v) {
+  %elem = extractelement <2 x i64> %v, i64 undef
+  ret i64 %elem
+}
+
 ; CHECK-LABEL: replace_v2i64:
 ; NO-SIMD128-NOT: i64x2
 ; SIMD128-VM-NOT: i64x2
 ; SIMD128-NEXT: .param v128, i64{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: i64x2.replace_lane $push[[R:[0-9]+]]=, $0, 0, $1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @replace_v2i64(<2 x i64> %v, i64 %x) {
   %res = insertelement <2 x i64> %v, i64 %x, i32 0
   ret <2 x i64> %res
 }
 
+; CHECK-LABEL: replace_undef_v2i64:
+; NO-SIMD128-NOT: i64x2
+; SIMD128-VM-NOT: i64x2
+; SIMD128-NEXT: .param v128, i64{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: i64x2.replace_lane $push[[R:[0-9]+]]=, $0, 0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @replace_undef_v2i64(<2 x i64> %v, i64 %x) {
+  %res = insertelement <2 x i64> %v, i64 %x, i32 0
+  ret <2 x i64> %res
+}
+
 ; CHECK-LABEL: shuffle_v2i64:
 ; NO-SIMD128-NOT: v8x16
 ; SIMD128-NEXT: .param v128, v128{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: v8x16.shuffle $push[[R:[0-9]+]]=, $0, $1,
 ; SIMD128-SAME: 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @shuffle_v2i64(<2 x i64> %x, <2 x i64> %y) {
   %res = shufflevector <2 x i64> %x, <2 x i64> %y, <2 x i32> <i32 0, i32 3>
   ret <2 x i64> %res
 }
 
+; CHECK-LABEL: shuffle_undef_v2i64:
+; NO-SIMD128-NOT: v8x16
+; SIMD128-NEXT: .param v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: v8x16.shuffle $push[[R:[0-9]+]]=, $0, $0,
+; SIMD128-SAME: 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @shuffle_undef_v2i64(<2 x i64> %x, <2 x i64> %y) {
+  %res = shufflevector <2 x i64> %x, <2 x i64> %y,
+    <2 x i32> <i32 1, i32 undef>
+  ret <2 x i64> %res
+}
+
 ; CHECK-LABEL: build_v2i64:
 ; NO-SIMD128-NOT: i64x2
 ; SIMD128-VM-NOT: i64x2
 ; SIMD128-NEXT: .param i64, i64{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: i64x2.splat $push[[L0:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: i64x2.replace_lane $push[[R:[0-9]+]]=, $pop[[L0]], 1, $1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}

 ; SIMD128-NEXT: .result f32{{$}}
 ; SIMD128-NEXT: f32x4.extract_lane $push[[R:[0-9]+]]=, $0, 3{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define float @extract_v4f32(<4 x float> %v) {
   %elem = extractelement <4 x float> %v, i32 3
   ret float %elem
 }
 
+; CHECK-LABEL: extract_undef_v4f32:
+; NO-SIMD128-NOT: f32x4
+; SIMD128-NEXT: .param v128{{$}}
+; SIMD128-NEXT: .result f32{{$}}
+; SIMD128-NEXT: f32x4.extract_lane $push[[R:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define float @extract_undef_v4f32(<4 x float> %v) {
+  %elem = extractelement <4 x float> %v, i32 undef
+  ret float %elem
+}
+
 ; CHECK-LABEL: replace_v4f32:
 ; NO-SIMD128-NOT: f32x4
 ; SIMD128-NEXT: .param v128, f32{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: f32x4.replace_lane $push[[R:[0-9]+]]=, $0, 2, $1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @replace_v4f32(<4 x float> %v, float %x) {
   %res = insertelement <4 x float> %v, float %x, i32 2
   ret <4 x float> %res
 }
 
+; CHECK-LABEL: replace_undef_v4f32:
+; NO-SIMD128-NOT: f32x4
+; SIMD128-NEXT: .param v128, f32{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: f32x4.replace_lane $push[[R:[0-9]+]]=, $0, 0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x float> @replace_undef_v4f32(<4 x float> %v, float %x) {
+  %res = insertelement <4 x float> %v, float %x, i32 0
+  ret <4 x float> %res
+}
+
 ; CHECK-LABEL: shuffle_v4f32:
 ; NO-SIMD128-NOT: v8x16
 ; SIMD128-NEXT: .param v128, v128{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: v8x16.shuffle $push[[R:[0-9]+]]=, $0, $1,
 ; SIMD128-SAME: 0, 1, 2, 3, 20, 21, 22, 23, 8, 9, 10, 11, 28, 29, 30, 31{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @shuffle_v4f32(<4 x float> %x, <4 x float> %y) {
   %res = shufflevector <4 x float> %x, <4 x float> %y,
     <4 x i32> <i32 0, i32 5, i32 2, i32 7>
   ret <4 x float> %res
 }
 
+; CHECK-LABEL: shuffle_undef_v4f32:
+; NO-SIMD128-NOT: v8x16
+; SIMD128-NEXT: .param v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: v8x16.shuffle $push[[R:[0-9]+]]=, $0, $0,
+; SIMD128-SAME: 4, 5, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x float> @shuffle_undef_v4f32(<4 x float> %x, <4 x float> %y) {
+  %res = shufflevector <4 x float> %x, <4 x float> %y,
+    <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  ret <4 x float> %res
+}
+
 ; CHECK-LABEL: build_v4f32:
 ; NO-SIMD128-NOT: f32x4
 ; SIMD128-NEXT: .param f32, f32, f32, f32{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: f32x4.splat $push[[L0:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: f32x4.replace_lane $push[[L1:[0-9]+]]=, $pop[[L0]], 1, $1{{$}}
 ; SIMD128-NEXT: f32x4.replace_lane $push[[L2:[0-9]+]]=, $pop[[L1]], 2, $2{{$}}
 ; SIMD128-NEXT: f32x4.replace_lane $push[[R:[0-9]+]]=, $pop[[L2]], 3, $3{{$}}

 ; SIMD128-NEXT: .result f64{{$}}
 ; SIMD128-NEXT: f64x2.extract_lane $push[[R:[0-9]+]]=, $0, 1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define double @extract_v2f64(<2 x double> %v) {
   %elem = extractelement <2 x double> %v, i32 1
   ret double %elem
 }
 
+; CHECK-LABEL: extract_undef_v2f64:
+; NO-SIMD128-NOT: f64x2
+; SIMD128-VM-NOT: f64x2
+; SIMD128-NEXT: .param v128{{$}}
+; SIMD128-NEXT: .result f64{{$}}
+; SIMD128-NEXT: f64x2.extract_lane $push[[R:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define double @extract_undef_v2f64(<2 x double> %v) {
+  %elem = extractelement <2 x double> %v, i32 undef
+  ret double %elem
+}
+
 ; CHECK-LABEL: replace_v2f64:
 ; NO-SIMD128-NOT: f64x2
 ; SIMD128-VM-NOT: f64x2
 ; SIMD128-NEXT: .param v128, f64{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: f64x2.replace_lane $push[[R:[0-9]+]]=, $0, 0, $1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @replace_v2f64(<2 x double> %v, double %x) {
   %res = insertelement <2 x double> %v, double %x, i32 0
   ret <2 x double> %res
 }
 
+; CHECK-LABEL: replace_undef_v2f64:
+; NO-SIMD128-NOT: f64x2
+; SIMD128-VM-NOT: f64x2
+; SIMD128-NEXT: .param v128, f64{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: f64x2.replace_lane $push[[R:[0-9]+]]=, $0, 0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x double> @replace_undef_v2f64(<2 x double> %v, double %x) {
+  %res = insertelement <2 x double> %v, double %x, i32 0
+  ret <2 x double> %res
+}
+
 ; CHECK-LABEL: shuffle_v2f64:
 ; NO-SIMD128-NOT: v8x16
 ; SIMD128-NEXT: .param v128, v128{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: v8x16.shuffle $push[[R:[0-9]+]]=, $0, $1,
 ; SIMD128-SAME: 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @shuffle_v2f64(<2 x double> %x, <2 x double> %y) {
   %res = shufflevector <2 x double> %x, <2 x double> %y,
     <2 x i32> <i32 0, i32 3>
   ret <2 x double> %res
 }
 
+; CHECK-LABEL: shuffle_undef_v2f64:
+; NO-SIMD128-NOT: v8x16
+; SIMD128-NEXT: .param v128, v128{{$}}
+; SIMD128-NEXT: .result v128{{$}}
+; SIMD128-NEXT: v8x16.shuffle $push[[R:[0-9]+]]=, $0, $0,
+; SIMD128-SAME: 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x double> @shuffle_undef_v2f64(<2 x double> %x, <2 x double> %y) {
+  %res = shufflevector <2 x double> %x, <2 x double> %y,
+    <2 x i32> <i32 1, i32 undef>
+  ret <2 x double> %res
+}
+
 ; CHECK-LABEL: build_v2f64:
 ; NO-SIMD128-NOT: f64x2
 ; SIMD128-VM-NOT: f64x2
 ; SIMD128-NEXT: .param f64, f64{{$}}
 ; SIMD128-NEXT: .result v128{{$}}
 ; SIMD128-NEXT: f64x2.splat $push[[L0:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: f64x2.replace_lane $push[[R:[0-9]+]]=, $pop[[L0]], 1, $1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @build_v2f64(double %x0, double %x1) {
   %t0 = insertelement <2 x double> undef, double %x0, i32 0
   %res = insertelement <2 x double> %t0, double %x1, i32 1
   ret <2 x double> %res
 }