[AArch64][SVE] Fix lowering of "fcmp ueq/one" when using SVE

[david-arm]

Summary

We were previously lowering to the incorrect instructions for the
setcc DAG node when using the SETUEQ and SETONE floating point
condition codes. I have fixed this by marking the SETONE code
as Expand and letting the SETUNE code be legal. I have also
fixed up the patterns for FCMNE_PPzZZ and FCMNE_PPzZ0 to use
the correct opcode.

[dmgreen]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

1304

I'm not near a computer to do decent testing, but should SETUNE be expanded? It seems like it should become a fcmne, if I'm reading the pseudocode correctly. And the issue here is that SETUNE and SETONE were being treated backwards.

There are ISel patterns for FCMNE_PPzZZ and FCMNE_PPzZ0 that still try (but will never?) use SETONE. I think they can be using SETUNE though.

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

1304

I was just typing the same thing after running some tests. Yes, we should be able to change the existing patterns and just flip U to O for the two setCondCodeAction calls.

[paulwalker-arm]

llvm/test/CodeGen/AArch64/sve-select.ll

549–551

Not the fault of this patch just observational but this looks weird now you've fixed the code generation. This comes from D119424 whereby the original oeq is inverted. That patch uses getSetCCInverse which says Return the operation corresponding to !(X op Y).

Perhaps I've got my logic wrong but I think !(X oeq Y) => X une Y whereas this output clearly shows we're getting X one Y. What do you think @david-arm is my logic silly? or is there a potential bug to investigate?

[paulwalker-arm]

llvm/test/CodeGen/AArch64/sve-select.ll

549–551

I've figured out what's going on and created D121968.

[david-arm]

• Removed expansion of SETUNE and fixed up the fcmne patterns to use SETUNE instead of SETONE as they were incorrect before.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

for (auto VT : {MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32,
       setCondCodeAction(ISD::SETLT, VT, Expand);
       setCondCodeAction(ISD::SETOLE, VT, Expand);
       setCondCodeAction(ISD::SETLE, VT, Expand);
       setCondCodeAction(ISD::SETULT, VT, Expand);
       setCondCodeAction(ISD::SETULE, VT, Expand);
       setCondCodeAction(ISD::SETUGE, VT, Expand);
       setCondCodeAction(ISD::SETUGT, VT, Expand);
       setCondCodeAction(ISD::SETUEQ, VT, Expand);
-      setCondCodeAction(ISD::SETUNE, VT, Expand);
+      setCondCodeAction(ISD::SETONE, VT, Expand);
     }
 
     for (auto VT : {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16}) {
       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
       setOperationAction(ISD::MGATHER, VT, Custom);
       setOperationAction(ISD::MSCATTER, VT, Custom);
       setOperationAction(ISD::MLOAD, VT, Custom);
       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
if (VT.isFloatingPoint()) {
     setCondCodeAction(ISD::SETLT, VT, Expand);
     setCondCodeAction(ISD::SETOLE, VT, Expand);
     setCondCodeAction(ISD::SETLE, VT, Expand);
     setCondCodeAction(ISD::SETULT, VT, Expand);
     setCondCodeAction(ISD::SETULE, VT, Expand);
     setCondCodeAction(ISD::SETUGE, VT, Expand);
     setCondCodeAction(ISD::SETUGT, VT, Expand);
     setCondCodeAction(ISD::SETUEQ, VT, Expand);
-    setCondCodeAction(ISD::SETUNE, VT, Expand);
+    setCondCodeAction(ISD::SETONE, VT, Expand);
   }
 
   // Mark integer truncating stores/extending loads as having custom lowering
   if (VT.isInteger()) {
     MVT InnerVT = VT.changeVectorElementType(MVT::i8);
     while (InnerVT != VT) {
       setTruncStoreAction(VT, InnerVT, Custom);
       setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Custom);

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

let Predicates = [HasSVEorStreamingSVE] in {
   defm CMPHS_PPzZI : sve_int_ucmp_vi<0b00, "cmphs", SETUGE, SETULE>;
   defm CMPHI_PPzZI : sve_int_ucmp_vi<0b01, "cmphi", SETUGT, SETULT>;
   defm CMPLO_PPzZI : sve_int_ucmp_vi<0b10, "cmplo", SETULT, SETUGT>;
   defm CMPLS_PPzZI : sve_int_ucmp_vi<0b11, "cmpls", SETULE, SETUGE>;
 
   defm FCMGE_PPzZZ : sve_fp_3op_p_pd_cc<0b000, "fcmge", SETOGE, SETGE, SETOLE, SETLE>;
   defm FCMGT_PPzZZ : sve_fp_3op_p_pd_cc<0b001, "fcmgt", SETOGT, SETGT, SETOLT, SETLT>;
   defm FCMEQ_PPzZZ : sve_fp_3op_p_pd_cc<0b010, "fcmeq", SETOEQ, SETEQ, SETOEQ, SETEQ>;
-  defm FCMNE_PPzZZ : sve_fp_3op_p_pd_cc<0b011, "fcmne", SETONE, SETNE, SETONE, SETNE>;
+  defm FCMNE_PPzZZ : sve_fp_3op_p_pd_cc<0b011, "fcmne", SETUNE, SETNE, SETUNE, SETNE>;
   defm FCMUO_PPzZZ : sve_fp_3op_p_pd_cc<0b100, "fcmuo", SETUO, SETUO, SETUO, SETUO>;
   defm FACGE_PPzZZ : sve_fp_3op_p_pd<0b101, "facge", int_aarch64_sve_facge>;
   defm FACGT_PPzZZ : sve_fp_3op_p_pd<0b111, "facgt", int_aarch64_sve_facgt>;
 
   defm FCMGE_PPzZ0 : sve_fp_2op_p_pd<0b000, "fcmge", SETOGE, SETGE, SETOLE, SETLE>;
   defm FCMGT_PPzZ0 : sve_fp_2op_p_pd<0b001, "fcmgt", SETOGT, SETGT, SETOLT, SETLT>;
   defm FCMLT_PPzZ0 : sve_fp_2op_p_pd<0b010, "fcmlt", SETOLT, SETLT, SETOGT, SETGT>;
   defm FCMLE_PPzZ0 : sve_fp_2op_p_pd<0b011, "fcmle", SETOLE, SETLE, SETOGE, SETGE>;
   defm FCMEQ_PPzZ0 : sve_fp_2op_p_pd<0b100, "fcmeq", SETOEQ, SETEQ, SETOEQ, SETEQ>;
-  defm FCMNE_PPzZ0 : sve_fp_2op_p_pd<0b110, "fcmne", SETONE, SETNE, SETONE, SETNE>;
+  defm FCMNE_PPzZ0 : sve_fp_2op_p_pd<0b110, "fcmne", SETUNE, SETNE, SETUNE, SETNE>;
 
   defm WHILELT_PWW : sve_int_while4_rr<0b010, "whilelt", int_aarch64_sve_whilelt>;
   defm WHILELE_PWW : sve_int_while4_rr<0b011, "whilele", int_aarch64_sve_whilele>;
   defm WHILELO_PWW : sve_int_while4_rr<0b110, "whilelo", int_aarch64_sve_whilelo>;
   defm WHILELS_PWW : sve_int_while4_rr<0b111, "whilels", int_aarch64_sve_whilels>;
 
   defm WHILELT_PXX : sve_int_while8_rr<0b010, "whilelt", int_aarch64_sve_whilelt>;
   defm WHILELE_PXX : sve_int_while8_rr<0b011, "whilele", int_aarch64_sve_whilele>;

llvm/test/CodeGen/AArch64/sve-fcmp.ll

 ; CHECK-NEXT:    ret
   %y = fcmp ole <vscale x 4 x float> %x, %x2
   ret <vscale x 4 x i1> %y
 }
 define <vscale x 4 x i1> @one(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
 ; CHECK-LABEL: one:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z1.s, z0.s
+; CHECK-NEXT:    fcmgt p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    sel p0.b, p0, p0.b, p1.b
 ; CHECK-NEXT:    ret
   %y = fcmp one <vscale x 4 x float> %x, %x2
   ret <vscale x 4 x i1> %y
 }
 define <vscale x 4 x i1> @ord(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
 ; CHECK-LABEL: ord:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fcmuo p1.s, p0/z, z0.s, z1.s
 ; CHECK-NEXT:    not p0.b, p0/z, p1.b
 ; CHECK-NEXT:    ret
   %y = fcmp ord <vscale x 4 x float> %x, %x2
   ret <vscale x 4 x i1> %y
 }
 define <vscale x 4 x i1> @ueq(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
 ; CHECK-LABEL: ueq:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    fcmne p1.s, p0/z, z0.s, z1.s
-; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    fcmeq p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    sel p0.b, p0, p0.b, p1.b
 ; CHECK-NEXT:    ret
   %y = fcmp ueq <vscale x 4 x float> %x, %x2
   ret <vscale x 4 x i1> %y
 }
 define <vscale x 4 x i1> @ugt(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
 ; CHECK-LABEL: ugt:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s

 ; CHECK-NEXT:    ret
   %y = fcmp ule <vscale x 4 x float> %x, %x2
   ret <vscale x 4 x i1> %y
 }
 define <vscale x 4 x i1> @une(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
 ; CHECK-LABEL: une:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    fcmeq p1.s, p0/z, z0.s, z1.s
-; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, z1.s
 ; CHECK-NEXT:    ret
   %y = fcmp une <vscale x 4 x float> %x, %x2
   ret <vscale x 4 x i1> %y
 }
 define <vscale x 4 x i1> @uno(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
 ; CHECK-LABEL: uno:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s

 ; CHECK-NEXT:    ret
   %y = fcmp oeq <vscale x 2 x float> %x, %x2
   ret <vscale x 2 x i1> %y
 }
 define <vscale x 2 x i1> @ueq_2f32(<vscale x 2 x float> %x, <vscale x 2 x float> %x2) {
 ; CHECK-LABEL: ueq_2f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    fcmne p1.s, p0/z, z0.s, z1.s
-; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    fcmeq p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    sel p0.b, p0, p0.b, p1.b
 ; CHECK-NEXT:    ret
   %y = fcmp ueq <vscale x 2 x float> %x, %x2
   ret <vscale x 2 x i1> %y
 }
 define <vscale x 2 x i1> @oeq_2f64(<vscale x 2 x double> %x, <vscale x 2 x double> %x2) {
 ; CHECK-LABEL: oeq_2f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fcmeq p0.d, p0/z, z0.d, z1.d
 ; CHECK-NEXT:    ret
   %y = fcmp oeq <vscale x 2 x double> %x, %x2
   ret <vscale x 2 x i1> %y
 }
 define <vscale x 2 x i1> @ueq_2f64(<vscale x 2 x double> %x, <vscale x 2 x double> %x2) {
 ; CHECK-LABEL: ueq_2f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    fcmne p1.d, p0/z, z0.d, z1.d
-; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p1.d, p0/z, z0.d, z1.d
+; CHECK-NEXT:    fcmeq p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT:    sel p0.b, p0, p0.b, p1.b
 ; CHECK-NEXT:    ret
   %y = fcmp ueq <vscale x 2 x double> %x, %x2
   ret <vscale x 2 x i1> %y
 }
 define <vscale x 2 x i1> @oeq_2f16(<vscale x 2 x half> %x, <vscale x 2 x half> %x2) {
 ; CHECK-LABEL: oeq_2f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fcmeq p0.h, p0/z, z0.h, z1.h
 ; CHECK-NEXT:    ret
   %y = fcmp oeq <vscale x 2 x half> %x, %x2
   ret <vscale x 2 x i1> %y
 }
 define <vscale x 2 x i1> @ueq_2f16(<vscale x 2 x half> %x, <vscale x 2 x half> %x2) {
 ; CHECK-LABEL: ueq_2f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    fcmne p1.h, p0/z, z0.h, z1.h
-; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    fcmeq p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    sel p0.b, p0, p0.b, p1.b
 ; CHECK-NEXT:    ret
   %y = fcmp ueq <vscale x 2 x half> %x, %x2
   ret <vscale x 2 x i1> %y
 }
 define <vscale x 4 x i1> @oeq_4f16(<vscale x 4 x half> %x, <vscale x 4 x half> %x2) {
 ; CHECK-LABEL: oeq_4f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fcmeq p0.h, p0/z, z0.h, z1.h
 ; CHECK-NEXT:    ret
   %y = fcmp oeq <vscale x 4 x half> %x, %x2
   ret <vscale x 4 x i1> %y
 }
 define <vscale x 4 x i1> @ueq_4f16(<vscale x 4 x half> %x, <vscale x 4 x half> %x2) {
 ; CHECK-LABEL: ueq_4f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    fcmne p1.h, p0/z, z0.h, z1.h
-; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    fcmeq p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    sel p0.b, p0, p0.b, p1.b
 ; CHECK-NEXT:    ret
   %y = fcmp ueq <vscale x 4 x half> %x, %x2
   ret <vscale x 4 x i1> %y
 }
 define <vscale x 8 x i1> @oeq_8f16(<vscale x 8 x half> %x, <vscale x 8 x half> %x2) {
 ; CHECK-LABEL: oeq_8f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    fcmeq p0.h, p0/z, z0.h, z1.h
 ; CHECK-NEXT:    ret
   %y = fcmp oeq <vscale x 8 x half> %x, %x2
   ret <vscale x 8 x i1> %y
 }
 define <vscale x 8 x i1> @ueq_8f16(<vscale x 8 x half> %x, <vscale x 8 x half> %x2) {
 ; CHECK-LABEL: ueq_8f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
-; CHECK-NEXT:    fcmne p1.h, p0/z, z0.h, z1.h
-; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    fcmeq p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    sel p0.b, p0, p0.b, p1.b
 ; CHECK-NEXT:    ret
   %y = fcmp ueq <vscale x 8 x half> %x, %x2
   ret <vscale x 8 x i1> %y
 }
 
 define <vscale x 4 x i32> @oeq_4f32_sext(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
 ; CHECK-LABEL: oeq_4f32_sext:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    ret
   %y = fcmp ole <vscale x 4 x float> %x, zeroinitializer
   ret <vscale x 4 x i1> %y
 }
 define <vscale x 4 x i1> @one_zero(<vscale x 4 x float> %x) {
 ; CHECK-LABEL: one_zero:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    fcmlt p1.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    fcmgt p0.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    sel p0.b, p0, p0.b, p1.b
 ; CHECK-NEXT:    ret
   %y = fcmp one <vscale x 4 x float> %x, zeroinitializer
   ret <vscale x 4 x i1> %y
 }
 define <vscale x 4 x i1> @ueq_zero(<vscale x 4 x float> %x) {
 ; CHECK-LABEL: ueq_zero:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z1.s, #0 // =0x0
 ; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    fcmne p1.s, p0/z, z0.s, #0.0
-; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    fcmeq p0.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    sel p0.b, p0, p0.b, p1.b
 ; CHECK-NEXT:    ret
   %y = fcmp ueq <vscale x 4 x float> %x, zeroinitializer
   ret <vscale x 4 x i1> %y
 }
 define <vscale x 4 x i1> @ugt_zero(<vscale x 4 x float> %x) {
 ; CHECK-LABEL: ugt_zero:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s

 ; CHECK-NEXT:    ret
   %y = fcmp ule <vscale x 4 x float> %x, zeroinitializer
   ret <vscale x 4 x i1> %y
 }
 define <vscale x 4 x i1> @une_zero(<vscale x 4 x float> %x) {
 ; CHECK-LABEL: une_zero:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    fcmeq p1.s, p0/z, z0.s, #0.0
-; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, #0.0
 ; CHECK-NEXT:    ret
   %y = fcmp une <vscale x 4 x float> %x, zeroinitializer
   ret <vscale x 4 x i1> %y
 }

llvm/test/CodeGen/AArch64/sve-fixed-length-float-compares.ll

 ;
 
 define void @fcmp_ueq_v16f16(<16 x half>* %a, <16 x half>* %b, <16 x i16>* %c) #0 {
 ; CHECK-LABEL: fcmp_ueq_v16f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h, vl16
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
-; CHECK-NEXT:    fcmne p1.h, p0/z, z0.h, z1.h
-; CHECK-NEXT:    mov z1.h, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    fcmeq p2.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    mov p1.b, p2/m, p2.b
 ; CHECK-NEXT:    mov z0.h, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    eor z0.d, z0.d, z1.d
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x2]
 ; CHECK-NEXT:    ret
   %op1 = load <16 x half>, <16 x half>* %a
   %op2 = load <16 x half>, <16 x half>* %b
   %cmp = fcmp ueq <16 x half> %op1, %op2
   %sext = sext <16 x i1> %cmp to <16 x i16>
   store <16 x i16> %sext, <16 x i16>* %c
   ret void
 }
 
 ;
 ; FCMP ONE
 ;
 
 define void @fcmp_one_v16f16(<16 x half>* %a, <16 x half>* %b, <16 x i16>* %c) #0 {
 ; CHECK-LABEL: fcmp_one_v16f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h, vl16
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
-; CHECK-NEXT:    fcmne p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    fcmgt p1.h, p0/z, z1.h, z0.h
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    mov p1.b, p2/m, p2.b
 ; CHECK-NEXT:    mov z0.h, p1/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x2]
 ; CHECK-NEXT:    ret
   %op1 = load <16 x half>, <16 x half>* %a
   %op2 = load <16 x half>, <16 x half>* %b
   %cmp = fcmp one <16 x half> %op1, %op2
   %sext = sext <16 x i1> %cmp to <16 x i16>
   store <16 x i16> %sext, <16 x i16>* %c
   ret void
 }
 
 ;
 ; FCMP UNE
 ;
 
 define void @fcmp_une_v16f16(<16 x half>* %a, <16 x half>* %b, <16 x i16>* %c) #0 {
 ; CHECK-LABEL: fcmp_une_v16f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h, vl16
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
-; CHECK-NEXT:    fcmeq p1.h, p0/z, z0.h, z1.h
-; CHECK-NEXT:    mov z1.h, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    fcmne p1.h, p0/z, z0.h, z1.h
 ; CHECK-NEXT:    mov z0.h, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    eor z0.d, z0.d, z1.d
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x2]
 ; CHECK-NEXT:    ret
   %op1 = load <16 x half>, <16 x half>* %a
   %op2 = load <16 x half>, <16 x half>* %b
   %cmp = fcmp une <16 x half> %op1, %op2
   %sext = sext <16 x i1> %cmp to <16 x i16>
   store <16 x i16> %sext, <16 x i16>* %c
   ret void

llvm/test/CodeGen/AArch64/sve-fixed-length-ptest.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64 -mattr=+sve < %s | FileCheck %s
 
 define i1 @ptest_v16i1_256bit_min_sve(float* %a, float * %b) vscale_range(2, 0) {
 ; CHECK-LABEL: ptest_v16i1_256bit_min_sve:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov x8, #8
 ; CHECK-NEXT:    ptrue p0.s, vl8
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x0]
-; CHECK-NEXT:    fcmeq p1.s, p0/z, z0.s, #0.0
-; CHECK-NEXT:    fcmeq p0.s, p0/z, z1.s, #0.0
+; CHECK-NEXT:    fcmne p1.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    fcmne p0.s, p0/z, z1.s, #0.0
 ; CHECK-NEXT:    mov z0.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z1.s, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z2.s, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    eor z0.d, z0.d, z1.d
-; CHECK-NEXT:    eor z1.d, z2.d, z1.d
+; CHECK-NEXT:    mov z1.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
 ; CHECK-NEXT:    uzp1 z1.b, z1.b, z1.b
-; CHECK-NEXT:    ptrue p0.b, vl16
 ; CHECK-NEXT:    mov v1.d[1], v0.d[0]
+; CHECK-NEXT:    ptrue p0.b, vl16
 ; CHECK-NEXT:    orv b0, p0, z1.b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
   %v0 = bitcast float* %a to <16 x float>*
   %v1 = load <16 x float>, <16 x float>* %v0, align 4
   %v2 = fcmp une <16 x float> %v1, zeroinitializer
   %v3 = call i1 @llvm.vector.reduce.or.i1.v16i1 (<16 x i1> %v2)
   ret i1 %v3
 }
 
 define i1 @ptest_v16i1_512bit_min_sve(float* %a, float * %b) vscale_range(4, 0) {
 ; CHECK-LABEL: ptest_v16i1_512bit_min_sve:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s, vl16
-; CHECK-NEXT:    mov z1.s, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
-; CHECK-NEXT:    fcmeq p0.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, #0.0
 ; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    ptrue p0.b, vl16
-; CHECK-NEXT:    eor z0.d, z0.d, z1.d
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
 ; CHECK-NEXT:    orv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
   %v0 = bitcast float* %a to <16 x float>*
   %v1 = load <16 x float>, <16 x float>* %v0, align 4
   %v2 = fcmp une <16 x float> %v1, zeroinitializer
   %v3 = call i1 @llvm.vector.reduce.or.i1.v16i1 (<16 x i1> %v2)
   ret i1 %v3
 }
 
 define i1 @ptest_v16i1_512bit_sve(float* %a, float * %b) vscale_range(4, 4) {
 ; CHECK-LABEL: ptest_v16i1_512bit_sve:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    mov z1.s, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
-; CHECK-NEXT:    fcmeq p0.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, #0.0
 ; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    ptrue p0.b, vl16
-; CHECK-NEXT:    eor z0.d, z0.d, z1.d
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
 ; CHECK-NEXT:    orv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
   %v0 = bitcast float* %a to <16 x float>*
   %v1 = load <16 x float>, <16 x float>* %v0, align 4
   %v2 = fcmp une <16 x float> %v1, zeroinitializer
   %v3 = call i1 @llvm.vector.reduce.or.i1.v16i1 (<16 x i1> %v2)
   ret i1 %v3
 }
 
 define i1 @ptest_or_v16i1_512bit_min_sve(float* %a, float * %b) vscale_range(4, 0) {
 ; CHECK-LABEL: ptest_or_v16i1_512bit_min_sve:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s, vl16
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
-; CHECK-NEXT:    fcmeq p1.s, p0/z, z0.s, #0.0
-; CHECK-NEXT:    fcmeq p0.s, p0/z, z1.s, #0.0
-; CHECK-NEXT:    mov z0.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z1.s, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z2.s, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    eor z0.d, z0.d, z1.d
-; CHECK-NEXT:    eor z1.d, z2.d, z1.d
+; CHECK-NEXT:    fcmne p1.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    fcmne p0.s, p0/z, z1.s, #0.0
+; CHECK-NEXT:    mov p0.b, p1/m, p1.b
+; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    ptrue p0.b, vl16
-; CHECK-NEXT:    orr z0.d, z0.d, z1.d
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
 ; CHECK-NEXT:    orv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
   %v0 = bitcast float* %a to <16 x float>*
   %v1 = load <16 x float>, <16 x float>* %v0, align 4

 ;
 
 define i1 @ptest_and_v16i1_512bit_sve(float* %a, float * %b) vscale_range(4, 4) {
 ; CHECK-LABEL: ptest_and_v16i1_512bit_sve:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
-; CHECK-NEXT:    fcmeq p1.s, p0/z, z0.s, #0.0
-; CHECK-NEXT:    fcmeq p0.s, p0/z, z1.s, #0.0
-; CHECK-NEXT:    mov z0.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z1.s, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    eor z0.d, z0.d, z1.d
-; CHECK-NEXT:    mov z1.s, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    bic z0.d, z0.d, z1.d
+; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    mov z0.s, #0 // =0x0
+; CHECK-NEXT:    fcmne p0.s, p0/z, z1.s, z0.s
+; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    ptrue p0.b, vl16
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
 ; CHECK-NEXT:    andv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
   %v0 = bitcast float* %a to <16 x float>*
   %v1 = load <16 x float>, <16 x float>* %v0, align 4
   %v2 = fcmp une <16 x float> %v1, zeroinitializer
   %v3 = bitcast float* %b to <16 x float>*
   %v4 = load <16 x float>, <16 x float>* %v3, align 4
   %v5 = fcmp une <16 x float> %v4, zeroinitializer
   %v6 = and <16 x i1> %v2, %v5
   %v7 = call i1 @llvm.vector.reduce.and.i1.v16i1 (<16 x i1> %v6)
   ret i1 %v7
 }
 
 define i1 @ptest_and_v16i1_512bit_min_sve(float* %a, float * %b) vscale_range(4, 0) {
 ; CHECK-LABEL: ptest_and_v16i1_512bit_min_sve:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s, vl16
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
-; CHECK-NEXT:    fcmeq p1.s, p0/z, z0.s, #0.0
-; CHECK-NEXT:    fcmeq p0.s, p0/z, z1.s, #0.0
-; CHECK-NEXT:    mov z0.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z1.s, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    eor z0.d, z0.d, z1.d
-; CHECK-NEXT:    mov z1.s, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    bic z0.d, z0.d, z1.d
+; CHECK-NEXT:    fcmne p1.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    fcmne p0.s, p0/z, z1.s, #0.0
+; CHECK-NEXT:    and p0.b, p1/z, p1.b, p0.b
+; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    ptrue p0.b, vl16
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
 ; CHECK-NEXT:    andv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
   %v0 = bitcast float* %a to <16 x float>*

llvm/test/CodeGen/AArch64/sve-select.ll

; CHECK-NEXT:    ret
     %mask = icmp eq i64 %x0, 0
     %sel = select i1 %mask, <vscale x 16 x i1> %a, <vscale x 16 x i1> %b
     ret <vscale x 16 x i1> %sel
 }
 
 define <vscale x 4 x float> @select_f32_invert_fmul(<vscale x 4 x float> %a, <vscale x 4 x float> %b) #0 {
 ; CHECK-LABEL: select_f32_invert_fmul:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    fcmeq p1.s, p0/z, z0.s, #0.0
-; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, #0.0
 ; CHECK-NEXT:    fmul z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
   %p = fcmp oeq <vscale x 4 x float> %a, zeroinitializer
   %fmul = fmul <vscale x 4 x float> %a, %b
   %sel = select <vscale x 4 x i1> %p, <vscale x 4 x float> %a, <vscale x 4 x float> %fmul
   ret <vscale x 4 x float> %sel
 }
 
 define <vscale x 4 x float> @select_f32_invert_fadd(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
 ; CHECK-LABEL: select_f32_invert_fadd:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    fcmeq p1.s, p0/z, z0.s, #0.0
-; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, #0.0
 ; CHECK-NEXT:    fadd z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
   %p = fcmp oeq <vscale x 4 x float> %a, zeroinitializer
   %fadd = fadd <vscale x 4 x float> %a, %b
   %sel = select <vscale x 4 x i1> %p, <vscale x 4 x float> %a, <vscale x 4 x float> %fadd
   ret <vscale x 4 x float> %sel
 }