[PowerPC] Add fix to partword atomic operations

[stefanp]

Summary

Partword atomic binaries are not zero extended as they should be.
This patch fixes them to ensure that they are zero extended.

[nemanjai]

LGTM. Of course, please ensure that this fixes the issue when the dependent patch is applied prior to committing this.

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

11359–11366

// Since the shift amount is not a constant, we need to clear
// the upper bits with a separate RLWINM.

llvm/test/CodeGen/PowerPC/ppc-partword-atomic.ll

104

Presumably this goes away when https://reviews.llvm.org/D101342 lands? Can you please apply that on top of this patch to make sure? Similarly with line 49.

[stefanp]

Added code comment.

[stefanp]

llvm/test/CodeGen/PowerPC/ppc-partword-atomic.ll

104

Presumably this goes away when https://reviews.llvm.org/D101342 lands? Can you please apply that on top of this patch to make sure? Similarly with line 49.

I have checked this and if I add the patch D101342 then those two lines disappear.
Here is the diff in the test case with the patch added:

--- a/llvm/test/CodeGen/PowerPC/ppc-partword-atomic.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc-partword-atomic.ll
@@ -46,10 +46,9 @@ define dso_local zeroext i32 @testI8(i8 zeroext %val) local_unnamed_addr #0 {
 ; PWR9-NEXT:    stbcx. 3, 0, 5
 ; PWR9-NEXT:    bne 0, .LBB0_1
 ; PWR9-NEXT:  # %bb.2: # %entry
-; PWR9-NEXT:    clrlwi 3, 4, 24
-; PWR9-NEXT:    addis 4, 2, global_int@toc@ha
+; PWR9-NEXT:    addis 3, 2, global_int@toc@ha
 ; PWR9-NEXT:    lwsync
-; PWR9-NEXT:    stw 3, global_int@toc@l(4)
+; PWR9-NEXT:    stw 4, global_int@toc@l(3)
 ; PWR9-NEXT:    li 3, 55
 ; PWR9-NEXT:    blr
 entry:
@@ -100,10 +99,9 @@ define dso_local zeroext i32 @testI16(i16 zeroext %val) local_unnamed_addr #0 {
 ; PWR9-NEXT:    sthcx. 3, 0, 5
 ; PWR9-NEXT:    bne 0, .LBB1_1
 ; PWR9-NEXT:  # %bb.2: # %entry
-; PWR9-NEXT:    clrlwi 3, 4, 16
-; PWR9-NEXT:    addis 4, 2, global_int@toc@ha
+; PWR9-NEXT:    addis 3, 2, global_int@toc@ha
 ; PWR9-NEXT:    lwsync
-; PWR9-NEXT:    stw 3, global_int@toc@l(4)
+; PWR9-NEXT:    stw 4, global_int@toc@l(3)
 ; PWR9-NEXT:    li 3, 55
 ; PWR9-NEXT:    blr
 entry:

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary(
   Register Incr2Reg = RegInfo.createVirtualRegister(GPRC);
   Register MaskReg = RegInfo.createVirtualRegister(GPRC);
   Register Mask2Reg = RegInfo.createVirtualRegister(GPRC);
   Register Mask3Reg = RegInfo.createVirtualRegister(GPRC);
   Register Tmp2Reg = RegInfo.createVirtualRegister(GPRC);
   Register Tmp3Reg = RegInfo.createVirtualRegister(GPRC);
   Register Tmp4Reg = RegInfo.createVirtualRegister(GPRC);
   Register TmpDestReg = RegInfo.createVirtualRegister(GPRC);
+  Register SrwDestReg = RegInfo.createVirtualRegister(GPRC);
   Register Ptr1Reg;
   Register TmpReg =
       (!BinOpcode) ? Incr2Reg : RegInfo.createVirtualRegister(GPRC);
 
   //  thisMBB:
   //   ...
   //   fallthrough --> loopMBB
   BB->addSuccessor(loopMBB);
MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary(
   //   lwarx tmpDest, ptr
   //   add tmp, tmpDest, incr2
   //   andc tmp2, tmpDest, mask
   //   and tmp3, tmp, mask
   //   or tmp4, tmp3, tmp2
   //   stwcx. tmp4, ptr
   //   bne- loopMBB
   //   fallthrough --> exitMBB
-  //   srw dest, tmpDest, shift
+  //   srw SrwDest, tmpDest, shift
+  //   rlwinm SrwDest, SrwDest, 0, 24 [16], 31
   if (ptrA != ZeroReg) {
     Ptr1Reg = RegInfo.createVirtualRegister(RC);
     BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg)
         .addReg(ptrA)
         .addReg(ptrB);
   } else {
     Ptr1Reg = ptrB;
   }
BuildMI(BB, dl, TII->get(PPC::BCC))
       .addReg(PPC::CR0)
       .addMBB(loopMBB);
   BB->addSuccessor(loopMBB);
   BB->addSuccessor(exitMBB);
 
   //  exitMBB:
   //   ...
   BB = exitMBB;
-  BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest)
+  // Since the shift amount is not a constant, we need to clear
+  // the upper bits with a separate RLWINM.
+  BuildMI(*BB, BB->begin(), dl, TII->get(PPC::RLWINM), dest)
+      .addReg(SrwDestReg)
+      .addImm(0)
+      .addImm(is8bit ? 24 : 16)
+      .addImm(31);
+  BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), SrwDestReg)
       .addReg(TmpDestReg)
       .addReg(ShiftReg);
   return BB;
 }
 
 llvm::MachineBasicBlock *
 PPCTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
                                     MachineBasicBlock *MBB) const {

llvm/test/CodeGen/PowerPC/atomics.ll

 ; PPC32-NEXT:    add r8, r4, r7
 ; PPC32-NEXT:    andc r9, r7, r6
 ; PPC32-NEXT:    and r8, r8, r6
 ; PPC32-NEXT:    or r8, r8, r9
 ; PPC32-NEXT:    stwcx. r8, 0, r5
 ; PPC32-NEXT:    bne cr0, .LBB12_1
 ; PPC32-NEXT:  # %bb.2:
 ; PPC32-NEXT:    srw r3, r7, r3
+; PPC32-NEXT:    clrlwi r3, r3, 24
 ; PPC32-NEXT:    blr
 ;
 ; PPC64-LABEL: add_i8_monotonic:
 ; PPC64:       # %bb.0:
 ; PPC64-NEXT:    rlwinm r7, r3, 3, 27, 28
 ; PPC64-NEXT:    li r6, 255
 ; PPC64-NEXT:    rldicr r5, r3, 0, 61
 ; PPC64-NEXT:    xori r3, r7, 24
 ; PPC64-NEXT:    slw r4, r4, r3
 ; PPC64-NEXT:    slw r6, r6, r3
 ; PPC64-NEXT:  .LBB12_1:
 ; PPC64-NEXT:    lwarx r7, 0, r5
 ; PPC64-NEXT:    add r8, r4, r7
 ; PPC64-NEXT:    andc r9, r7, r6
 ; PPC64-NEXT:    and r8, r8, r6
 ; PPC64-NEXT:    or r8, r8, r9
 ; PPC64-NEXT:    stwcx. r8, 0, r5
 ; PPC64-NEXT:    bne cr0, .LBB12_1
 ; PPC64-NEXT:  # %bb.2:
 ; PPC64-NEXT:    srw r3, r7, r3
+; PPC64-NEXT:    clrlwi r3, r3, 24
 ; PPC64-NEXT:    blr
   %val = atomicrmw add i8* %mem, i8 %operand monotonic
   ret i8 %val
 }
 define i16 @xor_i16_seq_cst(i16* %mem, i16 %operand) {
 ; PPC32-LABEL: xor_i16_seq_cst:
 ; PPC32:       # %bb.0:
 ; PPC32-NEXT:    li r6, 0

 ; PPC32-NEXT:    xor r8, r4, r7
 ; PPC32-NEXT:    andc r9, r7, r6
 ; PPC32-NEXT:    and r8, r8, r6
 ; PPC32-NEXT:    or r8, r8, r9
 ; PPC32-NEXT:    stwcx. r8, 0, r5
 ; PPC32-NEXT:    bne cr0, .LBB13_1
 ; PPC32-NEXT:  # %bb.2:
 ; PPC32-NEXT:    srw r3, r7, r3
+; PPC32-NEXT:    clrlwi r3, r3, 16
 ; PPC32-NEXT:    lwsync
 ; PPC32-NEXT:    blr
 ;
 ; PPC64-LABEL: xor_i16_seq_cst:
 ; PPC64:       # %bb.0:
 ; PPC64-NEXT:    li r6, 0
 ; PPC64-NEXT:    rlwinm r7, r3, 3, 27, 27
 ; PPC64-NEXT:    rldicr r5, r3, 0, 61
 ; PPC64-NEXT:    ori r6, r6, 65535
 ; PPC64-NEXT:    xori r3, r7, 16
 ; PPC64-NEXT:    slw r4, r4, r3
 ; PPC64-NEXT:    slw r6, r6, r3
 ; PPC64-NEXT:    sync
 ; PPC64-NEXT:  .LBB13_1:
 ; PPC64-NEXT:    lwarx r7, 0, r5
 ; PPC64-NEXT:    xor r8, r4, r7
 ; PPC64-NEXT:    andc r9, r7, r6
 ; PPC64-NEXT:    and r8, r8, r6
 ; PPC64-NEXT:    or r8, r8, r9
 ; PPC64-NEXT:    stwcx. r8, 0, r5
 ; PPC64-NEXT:    bne cr0, .LBB13_1
 ; PPC64-NEXT:  # %bb.2:
 ; PPC64-NEXT:    srw r3, r7, r3
+; PPC64-NEXT:    clrlwi r3, r3, 16
 ; PPC64-NEXT:    lwsync
 ; PPC64-NEXT:    blr
   %val = atomicrmw xor i16* %mem, i16 %operand seq_cst
   ret i16 %val
 }
 define i32 @xchg_i32_acq_rel(i32* %mem, i32 %operand) {
 ; CHECK-LABEL: xchg_i32_acq_rel:
 ; CHECK:       # %bb.0:

llvm/test/CodeGen/PowerPC/ppc-partword-atomic.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-- -mcpu=pwr7 %s -o - | FileCheck %s --check-prefix=PWR7
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-- -mcpu=pwr9 %s -o - | FileCheck %s --check-prefix=PWR9
+
+@value8 = dso_local global { i8 } zeroinitializer, align 1
+@value16 = dso_local global { i16 } zeroinitializer, align 2
+@global_int = dso_local local_unnamed_addr global i32 0, align 4
+
+define dso_local zeroext i32 @testI8(i8 zeroext %val) local_unnamed_addr #0 {
+; PWR7-LABEL: testI8:
+; PWR7:       # %bb.0: # %entry
+; PWR7-NEXT:    addis 4, 2, value8@toc@ha
+; PWR7-NEXT:    li 6, 255
+; PWR7-NEXT:    sync
+; PWR7-NEXT:    addi 5, 4, value8@toc@l
+; PWR7-NEXT:    rlwinm 4, 5, 3, 27, 28
+; PWR7-NEXT:    rldicr 5, 5, 0, 61
+; PWR7-NEXT:    xori 4, 4, 24
+; PWR7-NEXT:    slw 7, 3, 4
+; PWR7-NEXT:    slw 3, 6, 4
+; PWR7-NEXT:    and 6, 7, 3
+; PWR7-NEXT:  .LBB0_1: # %entry
+; PWR7-NEXT:    #
+; PWR7-NEXT:    lwarx 7, 0, 5
+; PWR7-NEXT:    andc 8, 7, 3
+; PWR7-NEXT:    or 8, 6, 8
+; PWR7-NEXT:    stwcx. 8, 0, 5
+; PWR7-NEXT:    bne 0, .LBB0_1
+; PWR7-NEXT:  # %bb.2: # %entry
+; PWR7-NEXT:    srw 3, 7, 4
+; PWR7-NEXT:    addis 5, 2, global_int@toc@ha
+; PWR7-NEXT:    lwsync
+; PWR7-NEXT:    clrlwi 4, 3, 24
+; PWR7-NEXT:    li 3, 55
+; PWR7-NEXT:    stw 4, global_int@toc@l(5)
+; PWR7-NEXT:    blr
+;
+; PWR9-LABEL: testI8:
+; PWR9:       # %bb.0: # %entry
+; PWR9-NEXT:    addis 4, 2, value8@toc@ha
+; PWR9-NEXT:    sync
+; PWR9-NEXT:    addi 5, 4, value8@toc@l
+; PWR9-NEXT:  .LBB0_1: # %entry
+; PWR9-NEXT:    #
+; PWR9-NEXT:    lbarx 4, 0, 5
+; PWR9-NEXT:    stbcx. 3, 0, 5
+; PWR9-NEXT:    bne 0, .LBB0_1
+; PWR9-NEXT:  # %bb.2: # %entry
+; PWR9-NEXT:    clrlwi 3, 4, 24
+; PWR9-NEXT:    addis 4, 2, global_int@toc@ha
+; PWR9-NEXT:    lwsync
+; PWR9-NEXT:    stw 3, global_int@toc@l(4)
+; PWR9-NEXT:    li 3, 55
+; PWR9-NEXT:    blr
+entry:
+  %0 = atomicrmw xchg i8* getelementptr inbounds ({ i8 }, { i8 }* @value8, i64 0, i32 0), i8 %val seq_cst, align 1
+  %conv = zext i8 %0 to i32
+  store i32 %conv, i32* @global_int, align 4
+  ret i32 55
+}
+
+define dso_local zeroext i32 @testI16(i16 zeroext %val) local_unnamed_addr #0 {
+; PWR7-LABEL: testI16:
+; PWR7:       # %bb.0: # %entry
+; PWR7-NEXT:    addis 4, 2, value16@toc@ha
+; PWR7-NEXT:    li 6, 0
+; PWR7-NEXT:    sync
+; PWR7-NEXT:    addi 5, 4, value16@toc@l
+; PWR7-NEXT:    ori 6, 6, 65535
+; PWR7-NEXT:    rlwinm 4, 5, 3, 27, 27
+; PWR7-NEXT:    rldicr 5, 5, 0, 61
+; PWR7-NEXT:    xori 4, 4, 16
+; PWR7-NEXT:    slw 7, 3, 4
+; PWR7-NEXT:    slw 3, 6, 4
+; PWR7-NEXT:    and 6, 7, 3
+; PWR7-NEXT:  .LBB1_1: # %entry
+; PWR7-NEXT:    #
+; PWR7-NEXT:    lwarx 7, 0, 5
+; PWR7-NEXT:    andc 8, 7, 3
+; PWR7-NEXT:    or 8, 6, 8
+; PWR7-NEXT:    stwcx. 8, 0, 5
+; PWR7-NEXT:    bne 0, .LBB1_1
+; PWR7-NEXT:  # %bb.2: # %entry
+; PWR7-NEXT:    srw 3, 7, 4
+; PWR7-NEXT:    addis 5, 2, global_int@toc@ha
+; PWR7-NEXT:    lwsync
+; PWR7-NEXT:    clrlwi 4, 3, 16
+; PWR7-NEXT:    li 3, 55
+; PWR7-NEXT:    stw 4, global_int@toc@l(5)
+; PWR7-NEXT:    blr
+;
+; PWR9-LABEL: testI16:
+; PWR9:       # %bb.0: # %entry
+; PWR9-NEXT:    addis 4, 2, value16@toc@ha
+; PWR9-NEXT:    sync
+; PWR9-NEXT:    addi 5, 4, value16@toc@l
+; PWR9-NEXT:  .LBB1_1: # %entry
+; PWR9-NEXT:    #
+; PWR9-NEXT:    lharx 4, 0, 5
+; PWR9-NEXT:    sthcx. 3, 0, 5
+; PWR9-NEXT:    bne 0, .LBB1_1
+; PWR9-NEXT:  # %bb.2: # %entry
+; PWR9-NEXT:    clrlwi 3, 4, 16
+; PWR9-NEXT:    addis 4, 2, global_int@toc@ha
+; PWR9-NEXT:    lwsync
+; PWR9-NEXT:    stw 3, global_int@toc@l(4)
+; PWR9-NEXT:    li 3, 55
+; PWR9-NEXT:    blr
+entry:
+  %0 = atomicrmw xchg i16* getelementptr inbounds ({ i16 }, { i16 }* @value16, i64 0, i32 0), i16 %val seq_cst, align 2
+  %conv = zext i16 %0 to i32
+  store i32 %conv, i32* @global_int, align 4
+  ret i32 55
+}
+
+attributes #0 = { nounwind }