[PowerPC] MMA - Add __builtin_vsx_build_pair and __builtin_mma_build_acc builtins

[saghir]

Summary

This patch adds the following built-ins:

builtin_vsx_build_pair
builtin_mma_build_acc

[lei]

This comment was removed by lei.

[lei]

Actually we should not be removing the deprecated bultins. Just need to add the new ones.

[nemanjai]

In D107647#2937706, @lei wrote:

Actually we should not be removing the deprecated bultins. Just need to add the new ones.

Yes, and also the semantics are different.

[saghir]

Change the implementation to add new builtins and keep the depracated builtins.

[saghir]

Added some more tests.

[saghir]

Added comments, and re-organized tests.

[lei]

clang/lib/CodeGen/CGBuiltin.cpp

15740

doesn't look like we need the interm var IsLE. Just use the call directly within the if stmt.

if (getTarget().isLittleEndian()) {

15742–15744

unsigned NumVecs = (BuiltinID == PPC::BI__builtin_mma_build_acc) ? 4 : 2;

clang/test/CodeGen/builtins-ppc-pair-mma.c

2

future -> pwr10
We need to add BE tests.

clang/test/Sema/ppc-pair-mma-types.c

2

this should be -target-cpu pwer10 now.

2

Please add BE testing.

265

This looks like a dup of testVPLocal(). Why not just add the new call line to that function right below the call to the deprecated function?

clang/test/SemaCXX/ppc-pair-mma-types.cpp

2

please update to pwr10

[nemanjai]

There are some questions to answer here before proceeding:

1. Why do we not get paired vector loads and stores in the back end test cases for either little or big endian and regardless of whether we use the old or new builtins/intrinsics?
2. Is the code generated the same as that for GCC when:
   • The inputs are all from registers
   • The inputs are all from memory
   • The inputs are a mix of memory and registers
3. What do execution tests show for both little endian and big endian targets? Namely, write execution test cases that do everything mentioned in 2. above for both sets of builtins and confirm:
   • The behaviour is the same both with GCC and Clang for both LE and BE
   • The behaviour for new builtins is the same on LE and BE with both GCC and Clang
   • The behaviour for old builtins is different on LE and BE with both GCC and Clang

I am requesting changes until these questions are adequately addressed.

llvm/include/llvm/IR/IntrinsicsPowerPC.td

1442

I find the need for this rather surprising. The new builtins should do the exact same thing as the old builtins but with elements in reversed order. So why do we need new intrinsics? Can we not just call the old intrinsics in the front end codegen but with arguments in reversed order?

llvm/test/CodeGen/PowerPC/paired-vector-intrinsics.ll

73

This indicates some kind of problem. Why are we moving a value from v2 to v3 and then not using v3?

[saghir]

Addressed review comments.

[nemanjai]

LGTM other than the code can be simplified as suggested.

clang/lib/CodeGen/CGBuiltin.cpp

15740–15747

This entire block seems to simply be std::reverse(Ops.begin() + 1, Ops.end())

Also, please add a note that the very first operand is the pointer to the pair/accumulator that is actually being built.

[lei]

LGTM once the code is simplified as Nemanja suggested.

[saghir]

Addressed review comments.

clang/include/clang/Basic/BuiltinsPPC.def

 // UNALIASED_CUSTOM_BUILTIN macro is used for built-ins that have
 // the same name as that of the intrinsic they generate, i.e. the
 // ID and INTR are the same.
 // This avoids repeating the ID and INTR in the macro expression.
 
 UNALIASED_CUSTOM_BUILTIN(vsx_lxvp, "W256SLLiW256C*", false)
 UNALIASED_CUSTOM_BUILTIN(vsx_stxvp, "vW256SLLiW256C*", false)
 UNALIASED_CUSTOM_BUILTIN(vsx_assemble_pair, "vW256*VV", false)
+UNALIASED_CUSTOM_BUILTIN(vsx_build_pair, "vW256*VV", false)
 UNALIASED_CUSTOM_BUILTIN(vsx_disassemble_pair, "vv*W256*", false)
 
 UNALIASED_CUSTOM_BUILTIN(mma_assemble_acc, "vW512*VVVV", false)
+UNALIASED_CUSTOM_BUILTIN(mma_build_acc, "vW512*VVVV", false)
 UNALIASED_CUSTOM_BUILTIN(mma_disassemble_acc, "vv*W512*", false)
 UNALIASED_CUSTOM_BUILTIN(mma_xxmtacc, "vW512*", true)
 UNALIASED_CUSTOM_BUILTIN(mma_xxmfacc, "vW512*", true)
 UNALIASED_CUSTOM_BUILTIN(mma_xxsetaccz, "vW512*", false)
 UNALIASED_CUSTOM_BUILTIN(mma_xvi4ger8, "vW512*VV", false)
 UNALIASED_CUSTOM_BUILTIN(mma_xvi8ger4, "vW512*VV", false)
 UNALIASED_CUSTOM_BUILTIN(mma_xvi16ger2, "vW512*VV", false)
 UNALIASED_CUSTOM_BUILTIN(mma_xvi16ger2s, "vW512*VV", false)

clang/lib/CodeGen/CGBuiltin.cpp

if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc ||
       for (unsigned i=0; i<NumVecs; i++) {
         Value *Vec = Builder.CreateExtractValue(Call, i);
         llvm::ConstantInt* Index = llvm::ConstantInt::get(IntTy, i);
         Value *GEP = Builder.CreateInBoundsGEP(VTy, Ptr, Index);
         Builder.CreateAlignedStore(Vec, GEP, MaybeAlign(16));
       }
       return Call;
     }
+    if (BuiltinID == PPC::BI__builtin_vsx_build_pair ||
+        BuiltinID == PPC::BI__builtin_mma_build_acc) {
+      bool IsLE = getTarget().isLittleEndian();
+      if (IsLE) {
+        SmallVector<Value *, 4> RevOps;
+        unsigned NumVecs = 2;
+        if (BuiltinID == PPC::BI__builtin_mma_build_acc)
+          NumVecs = 4;
+        for (unsigned i = NumVecs; i > 0; --i)
+          RevOps.push_back(Ops[i]);
+        for (unsigned i = 1; i <= NumVecs; ++i)
+          Ops[i] = RevOps[i - 1];
+      }
+    }
     bool Accumulate;
     switch (BuiltinID) {
   #define CUSTOM_BUILTIN(Name, Intr, Types, Acc) \
     case PPC::BI__builtin_##Name: \
       ID = Intrinsic::ppc_##Intr; \
       Accumulate = Acc; \
       break;
   #include "clang/Basic/BuiltinsPPC.def"

clang/test/CodeGen/builtins-ppc-pair-mma.c

 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // RUN: %clang_cc1 -O3 -triple powerpc64le-unknown-unknown -target-cpu future -emit-llvm %s -o - | FileCheck %s
 
 // CHECK-LABEL: @test1(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.assemble.acc(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]], <16 x i8> [[VC]], <16 x i8> [[VC]])
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[RESP:%.*]] to <512 x i1>*
 // CHECK-NEXT:    store <512 x i1> [[TMP0]], <512 x i1>* [[TMP1]], align 64, !tbaa [[TBAA2:![0-9]+]]
 // CHECK-NEXT:    ret void
 //
 void test1(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, unsigned char *resp) {
   __vector_quad vq = *((__vector_quad *)vqp);
   __vector_pair vp = *((__vector_pair *)vpp);
   __vector_quad res;
   __builtin_mma_assemble_acc(&res, vc, vc, vc, vc);
   *((__vector_quad *)resp) = res;
 }
 
+// CHECK-LABEL: @test1a(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.build.acc(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]], <16 x i8> [[VC]], <16 x i8> [[VC]])
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[RESP:%.*]] to <512 x i1>*
+// CHECK-NEXT:    store <512 x i1> [[TMP0]], <512 x i1>* [[TMP1]], align 64, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+void test1a(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, unsigned char *resp) {
+  __vector_quad vq = *((__vector_quad *)vqp);
+  __vector_pair vp = *((__vector_pair *)vpp);
+  __vector_quad res;
+  __builtin_mma_build_acc(&res, vc, vc, vc, vc);
+  *((__vector_quad *)resp) = res;
+}
+
 // CHECK-LABEL: @test2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8* [[VQP:%.*]] to <512 x i1>*
 // CHECK-NEXT:    [[TMP1:%.*]] = load <512 x i1>, <512 x i1>* [[TMP0]], align 64
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.acc(<512 x i1> [[TMP1]])
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[RESP:%.*]] to <16 x i8>*
 // CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[TMP2]], 0
 // CHECK-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* [[TMP3]], align 16

 void test3(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, unsigned char *resp) {
   __vector_quad vq = *((__vector_quad *)vqp);
   __vector_pair vp = *((__vector_pair *)vpp);
   __vector_pair res;
   __builtin_vsx_assemble_pair(&res, vc, vc);
   *((__vector_pair *)resp) = res;
 }
 
+// CHECK-LABEL: @test3a(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.build.pair(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[RESP:%.*]] to <256 x i1>*
+// CHECK-NEXT:    store <256 x i1> [[TMP0]], <256 x i1>* [[TMP1]], align 32, !tbaa [[TBAA6]]
+// CHECK-NEXT:    ret void
+//
+void test3a(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, unsigned char *resp) {
+  __vector_quad vq = *((__vector_quad *)vqp);
+  __vector_pair vp = *((__vector_pair *)vpp);
+  __vector_pair res;
+  __builtin_vsx_build_pair(&res, vc, vc);
+  *((__vector_pair *)resp) = res;
+}
+
 // CHECK-LABEL: @test4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8* [[VPP:%.*]] to <256 x i1>*
 // CHECK-NEXT:    [[TMP1:%.*]] = load <256 x i1>, <256 x i1>* [[TMP0]], align 32
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[TMP1]])
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[RESP:%.*]] to <16 x i8>*
 // CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[TMP2]], 0
 // CHECK-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* [[TMP3]], align 16

clang/test/Sema/ppc-pair-mma-types.c

 // RUN: %clang_cc1 -triple powerpc64le-unknown-unknown -fsyntax-only \
 // RUN:   -target-cpu future %s -verify
 
 // The use of PPC MMA types is strongly restricted. Non-pointer MMA variables
 // can only be declared in functions and a limited number of operations are
 // supported on these types. This test case checks that invalid uses of MMA
 // types are correctly prevented.
 
 // vector quad
 
void testVPLocal(int *ptr, vector unsigned char vc) {
   __vector_pair vp2;
   __builtin_vsx_assemble_pair(&vp2, vc, vc);
   __vector_pair vp3;
   __vector_quad vq;
   __builtin_mma_xvf64ger(&vq, vp3, vc);
   *vpp = vp3;
 }
 
+// local
+void testVPLocal1(int *ptr, vector unsigned char vc) {
+  __vector_pair *vpp = (__vector_pair *)ptr;
+  __vector_pair vp1 = *vpp;
+  __vector_pair vp2;
+  __builtin_vsx_build_pair(&vp2, vc, vc);
+  __vector_pair vp3;
+  __vector_quad vq;
+  __builtin_mma_xvf64ger(&vq, vp3, vc);
+  *vpp = vp3;
+}
+
 // struct field
 struct TestVPStruct {
   int a;
   float b;
   __vector_pair c; // expected-error {{invalid use of PPC MMA type}}
   __vector_pair *vp;
 };
 

clang/test/SemaCXX/ppc-pair-mma-types.cpp

 // RUN: %clang_cc1 -triple powerpc64le-unknown-unknown -fsyntax-only \
 // RUN:   -fcxx-exceptions -target-cpu future %s -verify
 
 // vector quad
 
 // alias
 using vq_t = __vector_quad;
 void testVQAlias(int *inp, int *outp) {
   vq_t *vqin = (vq_t *)inp;
   vq_t *vqout = (vq_t *)outp;
auto f1 = [](void *ptr) -> __vector_pair {
     __vector_pair *vpp = (__vector_pair *)ptr;
     return *vpp; // expected-error {{invalid use of PPC MMA type}}
   };
   auto f2 = [](void *ptr) {
     __vector_pair *vpp = (__vector_pair *)ptr;
     return *vpp; // expected-error {{invalid use of PPC MMA type}}
   };
   auto f3 = [](vector unsigned char vc) { __vector_pair vp; __builtin_vsx_assemble_pair(&vp, vc, vc); return vp; }; // expected-error {{invalid use of PPC MMA type}}
+  auto f4 = [](vector unsigned char vc) { __vector_pair vp; __builtin_vsx_build_pair(&vp, vc, vc); return vp; }; // expected-error {{invalid use of PPC MMA type}}
 }
 
 // cast
 void TestVPCast() {
   __vector_pair vp;
   int *ip = reinterpret_cast<int *>(&vp);
   __vector_pair *vp2 = reinterpret_cast<__vector_pair *>(ip);
 }
 
 // throw
 void TestVPThrow() {
   __vector_pair vp;
   throw vp; // expected-error {{invalid use of PPC MMA type}}
 }

llvm/include/llvm/IR/IntrinsicsPowerPC.td

def int_ppc_setrnd : GCCBuiltin<"__builtin_setrnd">,
       Intrinsic<[llvm_double_ty], [llvm_i32_ty], []>;
 }
 
 let TargetPrefix = "ppc" in {
   def int_ppc_vsx_assemble_pair :
         Intrinsic<[llvm_v256i1_ty],
                   [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
 
+  def int_ppc_vsx_build_pair :
+        Intrinsic<[llvm_v256i1_ty],
+                  [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
+
   def int_ppc_vsx_disassemble_pair :
         Intrinsic<[llvm_v16i8_ty, llvm_v16i8_ty],
                   [llvm_v256i1_ty], [IntrNoMem]>;
 
   def int_ppc_mma_assemble_acc :
         Intrinsic<[llvm_v512i1_ty],
                   [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty],
                   [IntrNoMem]>;
 
+  def int_ppc_mma_build_acc :
+        Intrinsic<[llvm_v512i1_ty],
+                  [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty],
+                  [IntrNoMem]>;
+
   def int_ppc_mma_disassemble_acc :
         Intrinsic<[llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty],
                   [llvm_v512i1_ty], [IntrNoMem]>;
 
   def int_ppc_mma_xxmtacc :
         Intrinsic<[llvm_v512i1_ty], [llvm_v512i1_ty], [IntrNoMem]>;
 
   def int_ppc_mma_xxmfacc :

llvm/lib/Target/PowerPC/PPCInstrPrefix.td

 }
 
 let Predicates = [MMA] in {
   def : Pat<(v512i1 (PPCAccBuild v4i32:$vs1, v4i32:$vs0, v4i32:$vs3, v4i32:$vs2)),
             (XXMTACC Concats.VecsToVecQuad)>;
   def : Pat<(v512i1 (int_ppc_mma_assemble_acc v16i8:$vs1, v16i8:$vs0,
                                               v16i8:$vs3, v16i8:$vs2)),
             (XXMTACC Concats.VecsToVecQuad)>;
+  def : Pat<(v512i1 (int_ppc_mma_build_acc v16i8:$vs1, v16i8:$vs0,
+                                           v16i8:$vs3, v16i8:$vs2)),
+            (XXMTACC Concats.VecsToVecQuad)>;
   def : Pat<(v512i1 (PPCxxmfacc v512i1:$AS)), (XXMFACC acc:$AS)>;
   def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, 0)),
             Extracts.Vec0>;
   def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, 1)),
             Extracts.Vec1>;
   def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, 2)),
             Extracts.Vec2>;
   def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, 3)),
             Extracts.Vec3>;
 }
 
 let Predicates = [PairedVectorMemops] in {
   def : Pat<(v256i1 (PPCPairBuild v4i32:$vs1, v4i32:$vs0)),
             Concats.VecsToVecPair0>;
   def : Pat<(v256i1 (int_ppc_vsx_assemble_pair v16i8:$vs1, v16i8:$vs0)),
             Concats.VecsToVecPair0>;
+  def : Pat<(v256i1 (int_ppc_vsx_build_pair v16i8:$vs1, v16i8:$vs0)),
+            Concats.VecsToVecPair0>;
   def : Pat<(v4i32 (PPCPairExtractVsx vsrpevenrc:$v, 0)),
             (v4i32 (EXTRACT_SUBREG $v, sub_vsx0))>;
   def : Pat<(v4i32 (PPCPairExtractVsx vsrpevenrc:$v, 1)),
             (v4i32 (EXTRACT_SUBREG $v, sub_vsx1))>;
 }
 
 let mayLoad = 1, mayStore = 0, Predicates = [PairedVectorMemops] in {
   def LXVP : DQForm_XTp5_RA17_MEM<6, 0, (outs vsrprc:$XTp),

llvm/test/CodeGen/PowerPC/mma-intrinsics.ll

 ; CHECK-BE-NEXT:    stxv vs2, 32(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = tail call <512 x i1> @llvm.ppc.mma.assemble.acc(<16 x i8> %vc, <16 x i8> %vc, <16 x i8> %vc, <16 x i8> %vc)
   store <512 x i1> %0, <512 x i1>* %ptr, align 64
   ret void
 }
 
+; build_acc
+declare <512 x i1> @llvm.ppc.mma.build.acc(<16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>)
+define void @build_acc1(<512 x i1>* %ptr, <16 x i8> %vc) {
+; CHECK-LABEL: build_acc1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmr v3, v2
+; CHECK-NEXT:    xxlor vs0, v2, v2
+; CHECK-NEXT:    xxlor vs1, v3, v3
+; CHECK-NEXT:    xxlor vs2, v2, v2
+; CHECK-NEXT:    xxlor vs3, v3, v3
+; CHECK-NEXT:    stxv vs0, 48(r3)
+; CHECK-NEXT:    stxv vs1, 32(r3)
+; CHECK-NEXT:    stxv vs2, 16(r3)
+; CHECK-NEXT:    stxv vs3, 0(r3)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: build_acc1:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    vmr v3, v2
+; CHECK-BE-NEXT:    xxlor vs0, v2, v2
+; CHECK-BE-NEXT:    xxlor vs1, v3, v3
+; CHECK-BE-NEXT:    xxlor vs2, v2, v2
+; CHECK-BE-NEXT:    xxlor vs3, v3, v3
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    stxv vs3, 48(r3)
+; CHECK-BE-NEXT:    stxv vs2, 32(r3)
+; CHECK-BE-NEXT:    blr
+entry:
+  %0 = tail call <512 x i1> @llvm.ppc.mma.build.acc(<16 x i8> %vc, <16 x i8> %vc, <16 x i8> %vc, <16 x i8> %vc)
+  store <512 x i1> %0, <512 x i1>* %ptr, align 64
+  ret void
+}
+
+; build_acc
+define void @build_acc2(<512 x i1>* %ptr, <16 x i8> %in1, <16 x i8> %in2, <16 x i8> %in3, <16 x i8> %in4) {
+; CHECK-LABEL: build_acc2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlor vs0, v5, v5
+; CHECK-NEXT:    xxlor vs1, v4, v4
+; CHECK-NEXT:    # kill: def $vsrp0 killed $vsrp0 def $uacc0
+; CHECK-NEXT:    vmr v4, v3
+; CHECK-NEXT:    vmr v5, v2
+; CHECK-NEXT:    xxlor vs2, v4, v4
+; CHECK-NEXT:    xxlor vs3, v5, v5
+; CHECK-NEXT:    stxv vs0, 48(r3)
+; CHECK-NEXT:    stxv vs1, 32(r3)
+; CHECK-NEXT:    stxv vs2, 16(r3)
+; CHECK-NEXT:    stxv vs3, 0(r3)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: build_acc2:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    xxlor vs0, v5, v5
+; CHECK-BE-NEXT:    xxlor vs1, v4, v4
+; CHECK-BE-NEXT:    # kill: def $vsrp0 killed $vsrp0 def $uacc0
+; CHECK-BE-NEXT:    vmr v4, v3
+; CHECK-BE-NEXT:    vmr v5, v2
+; CHECK-BE-NEXT:    xxlor vs2, v4, v4
+; CHECK-BE-NEXT:    xxlor vs3, v5, v5
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    stxv vs3, 48(r3)
+; CHECK-BE-NEXT:    stxv vs2, 32(r3)
+; CHECK-BE-NEXT:    blr
+entry:
+%0 = tail call <512 x i1> @llvm.ppc.mma.build.acc(<16 x i8> %in4, <16 x i8> %in3, <16 x i8> %in2, <16 x i8> %in1)
+store <512 x i1> %0, <512 x i1>* %ptr, align 64
+ret void
+}
+
+; build_pair
+declare <256 x i1> @llvm.ppc.vsx.build.pair(<16 x i8>, <16 x i8>)
+define void @build_pair(<256 x i1>* %ptr, <16 x i8> %in1, <16 x i8> %in2) {
+; CHECK-LABEL: build_pair:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmr v4, v3
+; CHECK-NEXT:    vmr v5, v2
+; CHECK-NEXT:    stxv v4, 16(r3)
+; CHECK-NEXT:    stxv v5, 0(r3)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: build_pair:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    vmr v4, v3
+; CHECK-BE-NEXT:    vmr v5, v2
+; CHECK-BE-NEXT:    stxv v2, 16(r3)
+; CHECK-BE-NEXT:    stxv v4, 0(r3)
+; CHECK-BE-NEXT:    blr
+entry:
+%0 = tail call <256 x i1> @llvm.ppc.vsx.build.pair(<16 x i8> %in2, <16 x i8> %in1)
+store <256 x i1> %0, <256 x i1>* %ptr, align 64
+ret void
+}
+
 ; xxmtacc
 declare <512 x i1> @llvm.ppc.mma.xxmtacc(<512 x i1>)
 define void @int_xxmtacc(<512 x i1>* %ptr, <16 x i8> %vc) {
 ; CHECK-LABEL: int_xxmtacc:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmr v3, v2
 ; CHECK-NEXT:    xxlor vs0, v2, v2
 ; CHECK-NEXT:    xxlor vs1, v3, v3
entry:
   ret void
 }
 
 declare <512 x i1> @llvm.ppc.mma.xvi4ger8pp(<512 x i1>, <16 x i8>, <16 x i8>)
 define void @testBranch(<512 x i1>* %ptr, <16 x i8> %vc, i32 %val) {
 ; CHECK-LABEL: testBranch:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    cmplwi r7, 0
-; CHECK-NEXT:    beq cr0, .LBB5_2
+; CHECK-NEXT:    beq cr0, .LBB8_2
 ; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    xxsetaccz acc0
-; CHECK-NEXT:    b .LBB5_3
-; CHECK-NEXT:  .LBB5_2: # %if.else
+; CHECK-NEXT:    b .LBB8_3
+; CHECK-NEXT:  .LBB8_2: # %if.else
 ; CHECK-NEXT:    lxv vs1, 32(r3)
 ; CHECK-NEXT:    lxv vs0, 48(r3)
 ; CHECK-NEXT:    lxv vs3, 0(r3)
 ; CHECK-NEXT:    lxv vs2, 16(r3)
 ; CHECK-NEXT:    xxmtacc acc0
 ; CHECK-NEXT:    xvi4ger8pp acc0, v2, v2
-; CHECK-NEXT:  .LBB5_3: # %if.end
+; CHECK-NEXT:  .LBB8_3: # %if.end
 ; CHECK-NEXT:    xxmfacc acc0
 ; CHECK-NEXT:    stxv vs0, 48(r3)
 ; CHECK-NEXT:    stxv vs1, 32(r3)
 ; CHECK-NEXT:    stxv vs2, 16(r3)
 ; CHECK-NEXT:    stxv vs3, 0(r3)
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: testBranch:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    cmplwi r7, 0
-; CHECK-BE-NEXT:    beq cr0, .LBB5_2
+; CHECK-BE-NEXT:    beq cr0, .LBB8_2
 ; CHECK-BE-NEXT:  # %bb.1: # %if.then
 ; CHECK-BE-NEXT:    xxsetaccz acc0
-; CHECK-BE-NEXT:    b .LBB5_3
-; CHECK-BE-NEXT:  .LBB5_2: # %if.else
+; CHECK-BE-NEXT:    b .LBB8_3
+; CHECK-BE-NEXT:  .LBB8_2: # %if.else
 ; CHECK-BE-NEXT:    lxv vs1, 16(r3)
 ; CHECK-BE-NEXT:    lxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    lxv vs3, 48(r3)
 ; CHECK-BE-NEXT:    lxv vs2, 32(r3)
 ; CHECK-BE-NEXT:    xxmtacc acc0
 ; CHECK-BE-NEXT:    xvi4ger8pp acc0, v2, v2
-; CHECK-BE-NEXT:  .LBB5_3: # %if.end
+; CHECK-BE-NEXT:  .LBB8_3: # %if.end
 ; CHECK-BE-NEXT:    xxmfacc acc0
 ; CHECK-BE-NEXT:    stxv vs1, 16(r3)
 ; CHECK-BE-NEXT:    stxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    stxv vs3, 48(r3)
 ; CHECK-BE-NEXT:    stxv vs2, 32(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %tobool = icmp eq i32 %val, 0

 ; CHECK-NEXT:    cmpwi r4, 1
 ; CHECK-NEXT:    bltlr cr0
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
 ; CHECK-NEXT:    clrldi r4, r4, 32
 ; CHECK-NEXT:    li r6, 0
 ; CHECK-NEXT:    mtctr r4
 ; CHECK-NEXT:    li r4, 0
 ; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  .LBB9_2: # %for.body
+; CHECK-NEXT:  .LBB12_2: # %for.body
 ; CHECK-NEXT:    #
 ; CHECK-NEXT:    rldic r7, r6, 4, 28
 ; CHECK-NEXT:    xxsetaccz acc2
 ; CHECK-NEXT:    xxsetaccz acc1
 ; CHECK-NEXT:    addi r6, r6, 6
 ; CHECK-NEXT:    lxvx vs0, r5, r7
 ; CHECK-NEXT:    add r7, r5, r7
 ; CHECK-NEXT:    lxv vs1, 16(r7)

 ; CHECK-NEXT:    stxv vs4, 112(r7)
 ; CHECK-NEXT:    stxv vs5, 96(r7)
 ; CHECK-NEXT:    stxv vs6, 80(r7)
 ; CHECK-NEXT:    stxv vs7, 64(r7)
 ; CHECK-NEXT:    stxv vs0, 176(r7)
 ; CHECK-NEXT:    stxv vs1, 160(r7)
 ; CHECK-NEXT:    stxv vs2, 144(r7)
 ; CHECK-NEXT:    stxv vs3, 128(r7)
-; CHECK-NEXT:    bdnz .LBB9_2
+; CHECK-NEXT:    bdnz .LBB12_2
 ; CHECK-NEXT:  # %bb.3: # %for.cond.cleanup
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: testcse4:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    cmpwi r4, 1
 ; CHECK-BE-NEXT:    bltlr cr0
 ; CHECK-BE-NEXT:  # %bb.1: # %for.body.preheader
 ; CHECK-BE-NEXT:    clrldi r4, r4, 32
 ; CHECK-BE-NEXT:    li r6, 0
 ; CHECK-BE-NEXT:    mtctr r4
 ; CHECK-BE-NEXT:    li r4, 0
 ; CHECK-BE-NEXT:    .p2align 4
-; CHECK-BE-NEXT:  .LBB9_2: # %for.body
+; CHECK-BE-NEXT:  .LBB12_2: # %for.body
 ; CHECK-BE-NEXT:    #
 ; CHECK-BE-NEXT:    rldic r7, r6, 4, 28
 ; CHECK-BE-NEXT:    xxsetaccz acc2
 ; CHECK-BE-NEXT:    xxsetaccz acc1
 ; CHECK-BE-NEXT:    addi r6, r6, 6
 ; CHECK-BE-NEXT:    lxvx vs0, r5, r7
 ; CHECK-BE-NEXT:    add r7, r5, r7
 ; CHECK-BE-NEXT:    lxv vs1, 16(r7)

 ; CHECK-BE-NEXT:    stxv vs5, 80(r7)
 ; CHECK-BE-NEXT:    stxv vs4, 64(r7)
 ; CHECK-BE-NEXT:    stxv vs7, 112(r7)
 ; CHECK-BE-NEXT:    stxv vs6, 96(r7)
 ; CHECK-BE-NEXT:    stxv vs1, 144(r7)
 ; CHECK-BE-NEXT:    stxv vs0, 128(r7)
 ; CHECK-BE-NEXT:    stxv vs3, 176(r7)
 ; CHECK-BE-NEXT:    stxv vs2, 160(r7)
-; CHECK-BE-NEXT:    bdnz .LBB9_2
+; CHECK-BE-NEXT:    bdnz .LBB12_2
 ; CHECK-BE-NEXT:  # %bb.3: # %for.cond.cleanup
 ; CHECK-BE-NEXT:    blr
 entry:
   %cmp55 = icmp sgt i32 %lim, 0
   br i1 %cmp55, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:                               ; preds = %entry
   %wide.trip.count = zext i32 %lim to i64

llvm/test/CodeGen/PowerPC/paired-vector-intrinsics.ll

 ; CHECK-BE-NOMMA-NEXT:    stxv v2, 0(r3)
 ; CHECK-BE-NOMMA-NEXT:    blr
 entry:
   %0 = tail call <256 x i1> @llvm.ppc.vsx.assemble.pair(<16 x i8> %vc, <16 x i8> %vc)
   store <256 x i1> %0, <256 x i1>* %ptr, align 32
   ret void
 }
 
+; build_pair
+declare <256 x i1> @llvm.ppc.vsx.build.pair(<16 x i8>, <16 x i8>)
+define void @build_pair(<256 x i1>* %ptr, <16 x i8> %vc) {
+; CHECK-LABEL: build_pair:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmr v3, v2
+; CHECK-NEXT:    stxv v2, 16(r3)
+; CHECK-NEXT:    stxv v3, 0(r3)
+; CHECK-NEXT:    blr
+;
+; CHECK-NOMMA-LABEL: build_pair:
+; CHECK-NOMMA:       # %bb.0: # %entry
+; CHECK-NOMMA-NEXT:    vmr v3, v2
+; CHECK-NOMMA-NEXT:    stxv v2, 16(r3)
+; CHECK-NOMMA-NEXT:    stxv v3, 0(r3)
+; CHECK-NOMMA-NEXT:    blr
+;
+; CHECK-BE-LABEL: build_pair:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    vmr v3, v2
+; CHECK-BE-NEXT:    stxv v2, 16(r3)
+; CHECK-BE-NEXT:    stxv v2, 0(r3)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-BE-NOMMA-LABEL: build_pair:
+; CHECK-BE-NOMMA:       # %bb.0: # %entry
+; CHECK-BE-NOMMA-NEXT:    vmr v3, v2
+; CHECK-BE-NOMMA-NEXT:    stxv v2, 16(r3)
+; CHECK-BE-NOMMA-NEXT:    stxv v2, 0(r3)
+; CHECK-BE-NOMMA-NEXT:    blr
+entry:
+  %0 = tail call <256 x i1> @llvm.ppc.vsx.build.pair(<16 x i8> %vc, <16 x i8> %vc)
+  store <256 x i1> %0, <256 x i1>* %ptr, align 32
+  ret void
+}
+
 ; disassemble_pair
 declare { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1>)
 define void @disass_pair(<256 x i1>* %ptr1, <16 x i8>* %ptr2, <16 x i8>* %ptr3) {
 ; CHECK-LABEL: disass_pair:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    lxv v3, 0(r3)
 ; CHECK-NEXT:    lxv v2, 16(r3)
 ; CHECK-NEXT:    stxv v3, 0(r4)