[mips][atomics] Fix partword atomic binary operation implementation

[sdardis]

Summary

Currently Mips::emitAtomicBinaryPartword() does not properly respect the
width of pointers. For MIPS64 this causes the memory address that the ll/sc
sequence uses to be truncated. At runtime this causes a segmentation fault.

This can be fixed by applying similar changes as r266204, so that a full 64bit
pointer is loaded.

[sdardis]

Fixed formatting.

[dsanders]

LGTM with a couple nits.

Also, does this fix any of the machine verifier failures that Quentin reported in https://llvm.org/bugs/show_bug.cgi?id=27458? I think it might fix atomic.ll or atomicops.ll.

From the commit message:
This can be fixed by applying similar changes as D18995, so that a full 64bit pointer is loaded.

Could you refer to the svn revision rather than the phabricator review?

test/CodeGen/Mips/atomicrmw.ll

8–19 ↗

(On Diff #55396)

This test looks very similar to AtomicLoadAdd16 in atomic.ll can we add checks to that instead? The new 'daddiu ..., $zero, -4' will pass the check in atomic.ll by accident at the moment since CHECK: addiu will match inside daddiu

16 ↗

(On Diff #55396)

This statement is dead

[sdardis]

Nits addressed.

[sdardis]

Committed as r267900.

[sdardis]

Unfortunately this patch doesn't fix the issues Quentin pointed out. The instructions here are taking operands of an incorrect register class (GPR32Opnd) rather than GPR64Opnd. I'll submit a patch for that.

lib/Target/Mips/MCTargetDesc/MipsABIInfo.h

public:
 
   unsigned GetStackPtr() const;
   unsigned GetFramePtr() const;
   unsigned GetBasePtr() const;
   unsigned GetNullPtr() const;
   unsigned GetZeroReg() const;
   unsigned GetPtrAdduOp() const;
   unsigned GetPtrAddiuOp() const;
+  unsigned GetPtrAndOp() const;
   unsigned GetGPRMoveOp() const;
   inline bool ArePtrs64bit() const { return IsN64(); }
   inline bool AreGprs64bit() const { return IsN32() || IsN64(); }
 
   unsigned GetEhDataReg(unsigned I) const;
 };
 }
 
 #endif

lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp

 unsigned MipsABIInfo::GetPtrAdduOp() const {
   return ArePtrs64bit() ? Mips::DADDu : Mips::ADDu;
 }
 
 unsigned MipsABIInfo::GetPtrAddiuOp() const {
   return ArePtrs64bit() ? Mips::DADDiu : Mips::ADDiu;
 }
 
+unsigned MipsABIInfo::GetPtrAndOp() const {
+  return ArePtrs64bit() ? Mips::AND64 : Mips::AND;
+}
+
 unsigned MipsABIInfo::GetGPRMoveOp() const {
   return ArePtrs64bit() ? Mips::OR64 : Mips::OR;
 }
 
 unsigned MipsABIInfo::GetEhDataReg(unsigned I) const {
   static const unsigned EhDataReg[] = {
     Mips::A0, Mips::A1, Mips::A2, Mips::A3
   };
   static const unsigned EhDataReg64[] = {
     Mips::A0_64, Mips::A1_64, Mips::A2_64, Mips::A3_64
   };
 
   return IsN64() ? EhDataReg64[I] : EhDataReg[I];
 }
 

lib/Target/Mips/MipsISelLowering.cpp

MachineBasicBlock *MipsTargetLowering::emitAtomicBinaryPartword(
     MachineInstr *MI, MachineBasicBlock *BB, unsigned Size, unsigned BinOpcode,
     bool Nand) const {
   assert((Size == 1 || Size == 2) &&
          "Unsupported size for EmitAtomicBinaryPartial.");
 
   MachineFunction *MF = BB->getParent();
   MachineRegisterInfo &RegInfo = MF->getRegInfo();
   const TargetRegisterClass *RC = getRegClassFor(MVT::i32);
+  bool ArePtrs64bit = ABI.ArePtrs64bit();
+  const TargetRegisterClass *RCp =
+    getRegClassFor(ArePtrs64bit ? MVT::i64 : MVT::i32);
   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
   DebugLoc DL = MI->getDebugLoc();
 
   unsigned Dest = MI->getOperand(0).getReg();
   unsigned Ptr = MI->getOperand(1).getReg();
   unsigned Incr = MI->getOperand(2).getReg();
 
-  unsigned AlignedAddr = RegInfo.createVirtualRegister(RC);
+  unsigned AlignedAddr = RegInfo.createVirtualRegister(RCp);
   unsigned ShiftAmt = RegInfo.createVirtualRegister(RC);
   unsigned Mask = RegInfo.createVirtualRegister(RC);
   unsigned Mask2 = RegInfo.createVirtualRegister(RC);
   unsigned NewVal = RegInfo.createVirtualRegister(RC);
   unsigned OldVal = RegInfo.createVirtualRegister(RC);
   unsigned Incr2 = RegInfo.createVirtualRegister(RC);
-  unsigned MaskLSB2 = RegInfo.createVirtualRegister(RC);
+  unsigned MaskLSB2 = RegInfo.createVirtualRegister(RCp);
   unsigned PtrLSB2 = RegInfo.createVirtualRegister(RC);
   unsigned MaskUpper = RegInfo.createVirtualRegister(RC);
   unsigned AndRes = RegInfo.createVirtualRegister(RC);
   unsigned BinOpRes = RegInfo.createVirtualRegister(RC);
   unsigned MaskedOldVal0 = RegInfo.createVirtualRegister(RC);
   unsigned StoreVal = RegInfo.createVirtualRegister(RC);
   unsigned MaskedOldVal1 = RegInfo.createVirtualRegister(RC);
   unsigned SrlRes = RegInfo.createVirtualRegister(RC);
MachineBasicBlock *MipsTargetLowering::emitAtomicBinaryPartword(
   //    andi    ptrlsb2,ptr,3
   //    sll     shiftamt,ptrlsb2,3
   //    ori     maskupper,$0,255               # 0xff
   //    sll     mask,maskupper,shiftamt
   //    nor     mask2,$0,mask
   //    sll     incr2,incr,shiftamt
 
   int64_t MaskImm = (Size == 1) ? 255 : 65535;
-  BuildMI(BB, DL, TII->get(Mips::ADDiu), MaskLSB2)
-    .addReg(Mips::ZERO).addImm(-4);
-  BuildMI(BB, DL, TII->get(Mips::AND), AlignedAddr)
+  BuildMI(BB, DL, TII->get(ABI.GetPtrAddiuOp()), MaskLSB2)
+    .addReg(ABI.GetNullPtr()).addImm(-4);
+  BuildMI(BB, DL, TII->get(ABI.GetPtrAndOp()), AlignedAddr)
     .addReg(Ptr).addReg(MaskLSB2);
-  BuildMI(BB, DL, TII->get(Mips::ANDi), PtrLSB2).addReg(Ptr).addImm(3);
+  BuildMI(BB, DL, TII->get(Mips::ANDi), PtrLSB2)
+      .addReg(Ptr, 0, ArePtrs64bit ? Mips::sub_32 : 0).addImm(3);
   if (Subtarget.isLittle()) {
     BuildMI(BB, DL, TII->get(Mips::SLL), ShiftAmt).addReg(PtrLSB2).addImm(3);
   } else {
     unsigned Off = RegInfo.createVirtualRegister(RC);
     BuildMI(BB, DL, TII->get(Mips::XORi), Off)
       .addReg(PtrLSB2).addImm((Size == 1) ? 3 : 2);
     BuildMI(BB, DL, TII->get(Mips::SLL), ShiftAmt).addReg(Off).addImm(3);
   }

test/CodeGen/Mips/atomic.ll

 ; RUN: llc -march=mipsel --disable-machine-licm -mcpu=mips32   -relocation-model=pic < %s | FileCheck %s -check-prefix=ALL -check-prefix=MIPS32-ANY -check-prefix=NO-SEB-SEH  -check-prefix=CHECK-EL -check-prefix=NOT-MICROMIPS
 ; RUN: llc -march=mipsel --disable-machine-licm -mcpu=mips32r2 -relocation-model=pic < %s | FileCheck %s -check-prefix=ALL -check-prefix=MIPS32-ANY -check-prefix=HAS-SEB-SEH -check-prefix=CHECK-EL -check-prefix=NOT-MICROMIPS
 ; RUN: llc -march=mipsel --disable-machine-licm -mcpu=mips32r6 -relocation-model=pic < %s | FileCheck %s -check-prefix=ALL -check-prefix=MIPS32-ANY -check-prefix=HAS-SEB-SEH -check-prefix=CHECK-EL -check-prefix=MIPSR6
 ; RUN: llc -march=mips64el --disable-machine-licm -mcpu=mips4    -relocation-model=pic < %s | FileCheck %s -check-prefix=ALL -check-prefix=MIPS64-ANY -check-prefix=NO-SEB-SEH  -check-prefix=CHECK-EL -check-prefix=NOT-MICROMIPS
 ; RUN: llc -march=mips64el --disable-machine-licm -mcpu=mips64   -relocation-model=pic < %s | FileCheck %s -check-prefix=ALL -check-prefix=MIPS64-ANY -check-prefix=NO-SEB-SEH  -check-prefix=CHECK-EL -check-prefix=NOT-MICROMIPS
 ; RUN: llc -march=mips64el --disable-machine-licm -mcpu=mips64r2 -relocation-model=pic < %s | FileCheck %s -check-prefix=ALL -check-prefix=MIPS64-ANY -check-prefix=HAS-SEB-SEH -check-prefix=CHECK-EL -check-prefix=NOT-MICROMIPS
 ; RUN: llc -march=mips64el --disable-machine-licm -mcpu=mips64r6 -relocation-model=pic < %s | FileCheck %s -check-prefix=ALL -check-prefix=MIPS64-ANY -check-prefix=HAS-SEB-SEH -check-prefix=CHECK-EL -check-prefix=MIPSR6
-; RUN: llc -march=mipsel --disable-machine-licm -mcpu=mips32r2 -mattr=micromips -relocation-model=pic < %s | FileCheck %s -check-prefix=ALL -check-prefix=MIPS32-ANY -check-prefix=HAS-SEB-SEH -check-prefix=CHECK-EL -check-prefix=MICROMIPS
+; RUN: llc -march=mips64 -O0 -mcpu=mips64r6 -relocation-model=pic < %s | FileCheck %s -check-prefix=ALL-LABEL -check-prefix=MIPS64-ANY -check-prefix=O0
+;; RUN: llc -march=mipsel --disable-machine-licm -mcpu=mips32r2 -mattr=micromips -relocation-model=pic < %s | FileCheck %s -check-prefix=ALL -check-prefix=MIPS32-ANY -check-prefix=HAS-SEB-SEH -check-prefix=CHECK-EL -check-prefix=MICROMIPS
 
 ; Keep one big-endian check so that we don't reduce testing, but don't add more
 ; since endianness doesn't affect the body of the atomic operations.
 ; RUN: llc -march=mips   --disable-machine-licm -mcpu=mips32 -relocation-model=pic < %s | FileCheck %s -check-prefix=ALL -check-prefix=MIPS32-ANY -check-prefix=NO-SEB-SEH -check-prefix=CHECK-EB -check-prefix=NOT-MICROMIPS
 
 @x = common global i32 0, align 4
 
 define i32 @AtomicLoadAdd32(i32 signext %incr) nounwind {
 entry:
   %0 = atomicrmw add i32* @x, i32 %incr monotonic
   ret i32 %0
 
 ; ALL-LABEL: AtomicLoadAdd32:
 
 ; MIPS32-ANY:    lw      $[[R0:[0-9]+]], %got(x)
 ; MIPS64-ANY:    ld      $[[R0:[0-9]+]], %got_disp(x)(
 
+; O0:        $[[BB0:[A-Z_0-9]+]]:
+; O0:            ld      $[[R1:[0-9]+]]
+; O0-NEXT:       ll      $[[R2:[0-9]+]], 0($[[R1]])
+
 ; ALL:       $[[BB0:[A-Z_0-9]+]]:
-; ALL:           ll      $[[R1:[0-9]+]], 0($[[R0]])
-; ALL:           addu    $[[R2:[0-9]+]], $[[R1]], $4
-; ALL:           sc      $[[R2]], 0($[[R0]])
-; NOT-MICROMIPS: beqz    $[[R2]], $[[BB0]]
-; MICROMIPS:     beqzc   $[[R2]], $[[BB0]]
-; MIPSR6:        beqzc   $[[R2]], $[[BB0]]
+; ALL:           ll      $[[R3:[0-9]+]], 0($[[R0]])
+; ALL:           addu    $[[R4:[0-9]+]], $[[R3]], $4
+; ALL:           sc      $[[R4]], 0($[[R0]])
+; NOT-MICROMIPS: beqz    $[[R4]], $[[BB0]]
+; MICROMIPS:     beqzc   $[[R4]], $[[BB0]]
+; MIPSR6:        beqzc   $[[R4]], $[[BB0]]
 }
 
 define i32 @AtomicLoadNand32(i32 signext %incr) nounwind {
 entry:
   %0 = atomicrmw nand i32* @x, i32 %incr monotonic
   ret i32 %0
 
 ; ALL-LABEL: AtomicLoadNand32:
 
 ; MIPS32-ANY:    lw      $[[R0:[0-9]+]], %got(x)
 ; MIPS64-ANY:    ld      $[[R0:[0-9]+]], %got_disp(x)(
 
+
+
 ; ALL:       $[[BB0:[A-Z_0-9]+]]:
 ; ALL:           ll      $[[R1:[0-9]+]], 0($[[R0]])
 ; ALL:           and     $[[R3:[0-9]+]], $[[R1]], $4
 ; ALL:           nor     $[[R2:[0-9]+]], $zero, $[[R3]]
 ; ALL:           sc      $[[R2]], 0($[[R0]])
 ; NOT-MICROMIPS: beqz    $[[R2]], $[[BB0]]
 ; MICROMIPS:     beqzc   $[[R2]], $[[BB0]]
 ; MIPSR6:        beqzc   $[[R2]], $[[BB0]]

 ; CHECK-EB:      xori    $[[R4:[0-9]+]], $[[R3]], 3
 ; CHECK-EB:      sll     $[[R5:[0-9]+]], $[[R4]], 3
 ; CHECK-EL:      sll     $[[R5:[0-9]+]], $[[R3]], 3
 ; ALL:           ori     $[[R6:[0-9]+]], $zero, 255
 ; ALL:           sllv    $[[R7:[0-9]+]], $[[R6]], $[[R5]]
 ; ALL:           nor     $[[R8:[0-9]+]], $zero, $[[R7]]
 ; ALL:           sllv    $[[R9:[0-9]+]], $4, $[[R5]]
 
+; O0:        $[[BB0:[A-Z_0-9]+]]:
+; O0:            ld      $[[R10:[0-9]+]]
+; O0-NEXT:       ll      $[[R11:[0-9]+]], 0($[[R10]])
+
 ; ALL:       $[[BB0:[A-Z_0-9]+]]:
-; ALL:           ll      $[[R10:[0-9]+]], 0($[[R2]])
-; ALL:           addu    $[[R11:[0-9]+]], $[[R10]], $[[R9]]
-; ALL:           and     $[[R12:[0-9]+]], $[[R11]], $[[R7]]
-; ALL:           and     $[[R13:[0-9]+]], $[[R10]], $[[R8]]
-; ALL:           or      $[[R14:[0-9]+]], $[[R13]], $[[R12]]
-; ALL:           sc      $[[R14]], 0($[[R2]])
-; NOT-MICROMIPS: beqz    $[[R14]], $[[BB0]]
-; MICROMIPS:     beqzc   $[[R14]], $[[BB0]]
-; MIPSR6:        beqzc   $[[R14]], $[[BB0]]
+; ALL:           ll      $[[R12:[0-9]+]], 0($[[R2]])
+; ALL:           addu    $[[R13:[0-9]+]], $[[R12]], $[[R9]]
+; ALL:           and     $[[R14:[0-9]+]], $[[R13]], $[[R7]]
+; ALL:           and     $[[R15:[0-9]+]], $[[R12]], $[[R8]]
+; ALL:           or      $[[R16:[0-9]+]], $[[R15]], $[[R14]]
+; ALL:           sc      $[[R16]], 0($[[R2]])
+; NOT-MICROMIPS: beqz    $[[R16]], $[[BB0]]
+; MICROMIPS:     beqzc   $[[R16]], $[[BB0]]
+; MIPSR6:        beqzc   $[[R16]], $[[BB0]]
 
-; ALL:           and     $[[R15:[0-9]+]], $[[R10]], $[[R7]]
-; ALL:           srlv    $[[R16:[0-9]+]], $[[R15]], $[[R5]]
+; ALL:           and     $[[R17:[0-9]+]], $[[R12]], $[[R7]]
+; ALL:           srlv    $[[R18:[0-9]+]], $[[R17]], $[[R5]]
 
-; NO-SEB-SEH:    sll     $[[R17:[0-9]+]], $[[R16]], 24
-; NO-SEB-SEH:    sra     $2, $[[R17]], 24
+; NO-SEB-SEH:    sll     $[[R19:[0-9]+]], $[[R18]], 24
+; NO-SEB-SEH:    sra     $2, $[[R19]], 24
 
-; HAS-SEB-SEH:   seb     $2, $[[R16]]
+; HAS-SEB-SEH:   seb     $2, $[[R18]]
 }
 
 define signext i8 @AtomicLoadSub8(i8 signext %incr) nounwind {
 entry:
   %0 = atomicrmw sub i8* @y, i8 %incr monotonic
   ret i8 %0
 
 ; ALL-LABEL: AtomicLoadSub8:
 
 ; MIPS32-ANY: lw      $[[R0:[0-9]+]], %got(y)
 ; MIPS64-ANY: ld      $[[R0:[0-9]+]], %got_disp(y)(
 
 ; ALL:        addiu   $[[R1:[0-9]+]], $zero, -4
 ; ALL:        and     $[[R2:[0-9]+]], $[[R0]], $[[R1]]
 ; ALL:        andi    $[[R3:[0-9]+]], $[[R0]], 3
 ; CHECK-EL:   sll     $[[R5:[0-9]+]], $[[R3]], 3
 ; CHECK-EB:   xori    $[[R4:[0-9]+]], $[[R3]], 3
 ; CHECK-EB:   sll     $[[R5:[0-9]+]], $[[R4]], 3
 ; ALL:        ori     $[[R6:[0-9]+]], $zero, 255
 ; ALL:        sllv    $[[R7:[0-9]+]], $[[R6]], $[[R5]]
 ; ALL:        nor     $[[R8:[0-9]+]], $zero, $[[R7]]
 ; ALL:        sllv    $[[R9:[0-9]+]], $4, $[[R5]]
 
+; O0:        $[[BB0:[A-Z_0-9]+]]:
+; O0:            ld      $[[R10:[0-9]+]]
+; O0-NEXT:       ll      $[[R11:[0-9]+]], 0($[[R10]])
+
 ; ALL:    $[[BB0:[A-Z_0-9]+]]:
-; ALL:        ll      $[[R10:[0-9]+]], 0($[[R2]])
-; ALL:        subu    $[[R11:[0-9]+]], $[[R10]], $[[R9]]
-; ALL:        and     $[[R12:[0-9]+]], $[[R11]], $[[R7]]
-; ALL:        and     $[[R13:[0-9]+]], $[[R10]], $[[R8]]
-; ALL:        or      $[[R14:[0-9]+]], $[[R13]], $[[R12]]
-; ALL:        sc      $[[R14]], 0($[[R2]])
-; NOT-MICROMIPS: beqz    $[[R14]], $[[BB0]]
-; MICROMIPS:  beqzc   $[[R14]], $[[BB0]]
-; MIPSR6:     beqzc   $[[R14]], $[[BB0]]
+; ALL:        ll      $[[R12:[0-9]+]], 0($[[R2]])
+; ALL:        subu    $[[R13:[0-9]+]], $[[R12]], $[[R9]]
+; ALL:        and     $[[R14:[0-9]+]], $[[R13]], $[[R7]]
+; ALL:        and     $[[R15:[0-9]+]], $[[R12]], $[[R8]]
+; ALL:        or      $[[R16:[0-9]+]], $[[R15]], $[[R14]]
+; ALL:        sc      $[[R16]], 0($[[R2]])
+; NOT-MICROMIPS: beqz    $[[R16]], $[[BB0]]
+; MICROMIPS:  beqzc   $[[R16]], $[[BB0]]
+; MIPSR6:     beqzc   $[[R16]], $[[BB0]]
 
-; ALL:        and     $[[R15:[0-9]+]], $[[R10]], $[[R7]]
-; ALL:        srlv    $[[R16:[0-9]+]], $[[R15]], $[[R5]]
+; ALL:        and     $[[R17:[0-9]+]], $[[R12]], $[[R7]]
+; ALL:        srlv    $[[R18:[0-9]+]], $[[R17]], $[[R5]]
 
-; NO-SEB-SEH: sll     $[[R17:[0-9]+]], $[[R16]], 24
-; NO-SEB-SEH: sra     $2, $[[R17]], 24
+; NO-SEB-SEH: sll     $[[R19:[0-9]+]], $[[R18]], 24
+; NO-SEB-SEH: sra     $2, $[[R19]], 24
 
-; HAS-SEB-SEH:seb     $2, $[[R16]]
+; HAS-SEB-SEH:seb     $2, $[[R18]]
 }
 
 define signext i8 @AtomicLoadNand8(i8 signext %incr) nounwind {
 entry:
   %0 = atomicrmw nand i8* @y, i8 %incr monotonic
   ret i8 %0
 
 ; ALL-LABEL: AtomicLoadNand8:
 
 ; MIPS32-ANY:    lw      $[[R0:[0-9]+]], %got(y)
 ; MIPS64-ANY:    ld      $[[R0:[0-9]+]], %got_disp(y)(
 
 ; ALL:           addiu   $[[R1:[0-9]+]], $zero, -4
 ; ALL:           and     $[[R2:[0-9]+]], $[[R0]], $[[R1]]
 ; ALL:           andi    $[[R3:[0-9]+]], $[[R0]], 3
 ; CHECK-EL:      sll     $[[R5:[0-9]+]], $[[R3]], 3
 ; CHECK-EB:      xori    $[[R4:[0-9]+]], $[[R3]], 3
 ; CHECK-EB:      sll     $[[R5:[0-9]+]], $[[R4]], 3
 ; ALL:           ori     $[[R6:[0-9]+]], $zero, 255
 ; ALL:           sllv    $[[R7:[0-9]+]], $[[R6]], $[[R5]]
 ; ALL:           nor     $[[R8:[0-9]+]], $zero, $[[R7]]
 ; ALL:           sllv    $[[R9:[0-9]+]], $4, $[[R5]]
 
-; ALL:       $[[BB0:[A-Z_0-9]+]]:
-; ALL:           ll      $[[R10:[0-9]+]], 0($[[R2]])
-; ALL:           and     $[[R18:[0-9]+]], $[[R10]], $[[R9]]
-; ALL:           nor     $[[R11:[0-9]+]], $zero, $[[R18]]
-; ALL:           and     $[[R12:[0-9]+]], $[[R11]], $[[R7]]
-; ALL:           and     $[[R13:[0-9]+]], $[[R10]], $[[R8]]
-; ALL:           or      $[[R14:[0-9]+]], $[[R13]], $[[R12]]
-; ALL:           sc      $[[R14]], 0($[[R2]])
-; NOT-MICROMIPS: beqz    $[[R14]], $[[BB0]]
-; MICROMIPS:     beqzc   $[[R14]], $[[BB0]]
-; MIPSR6:        beqzc   $[[R14]], $[[BB0]]
+; O0:        $[[BB0:[A-Z_0-9]+]]:
+; O0:            ld      $[[R10:[0-9]+]]
+; O0-NEXT:       ll      $[[R11:[0-9]+]], 0($[[R10]])
+
+; ALL:       $[[BB0:[A-Z_0-9]+]]:
+; ALL:           ll      $[[R12:[0-9]+]], 0($[[R2]])
+; ALL:           and     $[[R13:[0-9]+]], $[[R12]], $[[R9]]
+; ALL:           nor     $[[R14:[0-9]+]], $zero, $[[R13]]
+; ALL:           and     $[[R15:[0-9]+]], $[[R14]], $[[R7]]
+; ALL:           and     $[[R16:[0-9]+]], $[[R12]], $[[R8]]
+; ALL:           or      $[[R17:[0-9]+]], $[[R16]], $[[R15]]
+; ALL:           sc      $[[R17]], 0($[[R2]])
+; NOT-MICROMIPS: beqz    $[[R17]], $[[BB0]]
+; MICROMIPS:     beqzc   $[[R17]], $[[BB0]]
+; MIPSR6:        beqzc   $[[R17]], $[[BB0]]
 
-; ALL:           and     $[[R15:[0-9]+]], $[[R10]], $[[R7]]
-; ALL:           srlv    $[[R16:[0-9]+]], $[[R15]], $[[R5]]
+; ALL:           and     $[[R18:[0-9]+]], $[[R12]], $[[R7]]
+; ALL:           srlv    $[[R19:[0-9]+]], $[[R18]], $[[R5]]
 
-; NO-SEB-SEH:    sll     $[[R17:[0-9]+]], $[[R16]], 24
-; NO-SEB-SEH:    sra     $2, $[[R17]], 24
+; NO-SEB-SEH:    sll     $[[R20:[0-9]+]], $[[R19]], 24
+; NO-SEB-SEH:    sra     $2, $[[R20]], 24
 
-; HAS-SEB-SEH:   seb     $2, $[[R16]]
+; HAS-SEB-SEH:   seb     $2, $[[R19]]
 }
 
 define signext i8 @AtomicSwap8(i8 signext %newval) nounwind {
 entry:
   %0 = atomicrmw xchg i8* @y, i8 %newval monotonic
   ret i8 %0
 
 ; ALL-LABEL: AtomicSwap8:

 ; CHECK-EB:      xori    $[[R4:[0-9]+]], $[[R3]], 2
 ; CHECK-EB:      sll     $[[R5:[0-9]+]], $[[R4]], 3
 ; CHECK-EL:      sll     $[[R5:[0-9]+]], $[[R3]], 3
 ; ALL:           ori     $[[R6:[0-9]+]], $zero, 65535
 ; ALL:           sllv    $[[R7:[0-9]+]], $[[R6]], $[[R5]]
 ; ALL:           nor     $[[R8:[0-9]+]], $zero, $[[R7]]
 ; ALL:           sllv    $[[R9:[0-9]+]], $4, $[[R5]]
 
+; O0:        $[[BB0:[A-Z_0-9]+]]:
+; O0:            ld      $[[R10:[0-9]+]]
+; O0-NEXT:       ll      $[[R11:[0-9]+]], 0($[[R10]])
+
 ; ALL:       $[[BB0:[A-Z_0-9]+]]:
-; ALL:           ll      $[[R10:[0-9]+]], 0($[[R2]])
-; ALL:           addu    $[[R11:[0-9]+]], $[[R10]], $[[R9]]
-; ALL:           and     $[[R12:[0-9]+]], $[[R11]], $[[R7]]
-; ALL:           and     $[[R13:[0-9]+]], $[[R10]], $[[R8]]
-; ALL:           or      $[[R14:[0-9]+]], $[[R13]], $[[R12]]
-; ALL:           sc      $[[R14]], 0($[[R2]])
-; NOT-MICROMIPS: beqz    $[[R14]], $[[BB0]]
-; MICROMIPS:     beqzc   $[[R14]], $[[BB0]]
-; MIPSR6:        beqzc   $[[R14]], $[[BB0]]
+; ALL:           ll      $[[R12:[0-9]+]], 0($[[R2]])
+; ALL:           addu    $[[R13:[0-9]+]], $[[R12]], $[[R9]]
+; ALL:           and     $[[R14:[0-9]+]], $[[R13]], $[[R7]]
+; ALL:           and     $[[R15:[0-9]+]], $[[R12]], $[[R8]]
+; ALL:           or      $[[R16:[0-9]+]], $[[R15]], $[[R14]]
+; ALL:           sc      $[[R16]], 0($[[R2]])
+; NOT-MICROMIPS: beqz    $[[R16]], $[[BB0]]
+; MICROMIPS:     beqzc   $[[R16]], $[[BB0]]
+; MIPSR6:        beqzc   $[[R16]], $[[BB0]]
 
-; ALL:           and     $[[R15:[0-9]+]], $[[R10]], $[[R7]]
-; ALL:           srlv    $[[R16:[0-9]+]], $[[R15]], $[[R5]]
+; ALL:           and     $[[R17:[0-9]+]], $[[R12]], $[[R7]]
+; ALL:           srlv    $[[R18:[0-9]+]], $[[R17]], $[[R5]]
 
-; NO-SEB-SEH:    sll     $[[R17:[0-9]+]], $[[R16]], 16
-; NO-SEB-SEH:    sra     $2, $[[R17]], 16
+; NO-SEB-SEH:    sll     $[[R19:[0-9]+]], $[[R18]], 16
+; NO-SEB-SEH:    sra     $2, $[[R19]], 16
 
-; MIPS32R2:      seh     $2, $[[R16]]
+; MIPS32R2:      seh     $2, $[[R18]]
 }
 
 ; Test that the i16 return value from cmpxchg is recognised as signed,
 ; so that setCC doesn't end up comparing an unsigned value to a signed
 ; value.
 ; The rest of the functions here are testing the atomic expansion, so
 ; we just match the end of the function.
 define {i16, i1} @foo(i16* %addr, i16 %l, i16 %r, i16 %new) {