Support expanding partial-word cmpxchg to full-word cmpxchg in AtomicExpandPass.

[jyknight]

Summary

Many CPUs only have 4-byte cmpxchg, and not 1/2-byte instructions. For
those, you need to mask and shift appropriately and use a 4-byte
cmpxchg.

This change adds support for cmpxchg-based instruction sets (only SPARC,
in LLVM). The support can be extended for LL/SC-based instruction
sets (e.g. PPC and MIPS) in the future, supplanting the ISel expansions
those architectures currently use.

Tests added for SPARCv9.

llvm/trunk/include/llvm/Target/TargetLowering.h

public:
   /// Returns the maximum atomic operation size (in bits) supported by
   /// the backend. Atomic operations greater than this size (as well
   /// as ones that are not naturally aligned), will be expanded by
   /// AtomicExpandPass into an __atomic_* library call.
   unsigned getMaxAtomicSizeInBitsSupported() const {
     return MaxAtomicSizeInBitsSupported;
   }
 
+  /// Returns the size of the smallest cmpxchg or ll/sc instruction
+  /// the backend supports.  Any smaller operations are widened in
+  /// AtomicExpandPass.
+  ///
+  /// Note that *unlike* operations above the maximum size, atomic ops
+  /// are still natively supported below the minimum; they just
+  /// require a more complex expansion.
+  unsigned getMinCmpXchgSizeInBits() const { return MinCmpXchgSizeInBits; }
+
   /// Whether AtomicExpandPass should automatically insert fences and reduce
   /// ordering for this atomic. This should be true for most architectures with
   /// weak memory ordering. Defaults to false.
   virtual bool shouldInsertFencesForAtomic(const Instruction *I) const {
     return false;
   }
 
   /// Perform a load-linked operation on Addr, returning a "Value *" with the
protected:
   /// Set the maximum atomic operation size supported by the
   /// backend. Atomic operations greater than this size (as well as
   /// ones that are not naturally aligned), will be expanded by
   /// AtomicExpandPass into an __atomic_* library call.
   void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits) {
     MaxAtomicSizeInBitsSupported = SizeInBits;
   }
 
+  // Sets the minimum cmpxchg or ll/sc size supported by the backend.
+  void setMinCmpXchgSizeInBits(unsigned SizeInBits) {
+    MinCmpXchgSizeInBits = SizeInBits;
+  }
+
 public:
   //===--------------------------------------------------------------------===//
   // Addressing mode description hooks (used by LSR etc).
   //
 
   /// CodeGenPrepare sinks address calculations into the same BB as Load/Store
   /// instructions reading the address. This allows as much computation as
   /// possible to be done in the address mode for that operand. This hook lets
private:
 
   /// The preferred loop alignment.
   unsigned PrefLoopAlignment;
 
   /// Size in bits of the maximum atomics size the backend supports.
   /// Accesses larger than this will be expanded by AtomicExpandPass.
   unsigned MaxAtomicSizeInBitsSupported;
 
+  /// Size in bits of the minimum cmpxchg or ll/sc operation the
+  /// backend supports.
+  unsigned MinCmpXchgSizeInBits;
+
   /// If set to a physical register, this specifies the register that
   /// llvm.savestack/llvm.restorestack should save and restore.
   unsigned StackPointerRegisterToSaveRestore;
 
   /// This indicates the default register class to use for each ValueType the
   /// target supports natively.
   const TargetRegisterClass *RegClassForVT[MVT::LAST_VALUETYPE];
   unsigned char NumRegistersForVT[MVT::LAST_VALUETYPE];

llvm/trunk/lib/CodeGen/AtomicExpandPass.cpp

private:
     IntegerType *getCorrespondingIntegerType(Type *T, const DataLayout &DL);
     LoadInst *convertAtomicLoadToIntegerType(LoadInst *LI);
     bool tryExpandAtomicLoad(LoadInst *LI);
     bool expandAtomicLoadToLL(LoadInst *LI);
     bool expandAtomicLoadToCmpXchg(LoadInst *LI);
     StoreInst *convertAtomicStoreToIntegerType(StoreInst *SI);
     bool expandAtomicStore(StoreInst *SI);
     bool tryExpandAtomicRMW(AtomicRMWInst *AI);
-    bool expandAtomicOpToLLSC(
-        Instruction *I, Value *Addr, AtomicOrdering MemOpOrder,
+    Value *
+    insertRMWLLSCLoop(IRBuilder<> &Builder, Type *ResultTy, Value *Addr,
+                      AtomicOrdering MemOpOrder,
+                      function_ref<Value *(IRBuilder<> &, Value *)> PerformOp);
+    void expandAtomicOpToLLSC(
+        Instruction *I, Type *ResultTy, Value *Addr, AtomicOrdering MemOpOrder,
         function_ref<Value *(IRBuilder<> &, Value *)> PerformOp);
+    void expandPartwordAtomicRMW(
+        AtomicRMWInst *I,
+        TargetLoweringBase::AtomicExpansionKind ExpansionKind);
+    void expandPartwordCmpXchg(AtomicCmpXchgInst *I);
+
     AtomicCmpXchgInst *convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI);
+    static Value *insertRMWCmpXchgLoop(
+        IRBuilder<> &Builder, Type *ResultType, Value *Addr,
+        AtomicOrdering MemOpOrder,
+        function_ref<Value *(IRBuilder<> &, Value *)> PerformOp,
+        CreateCmpXchgInstFun CreateCmpXchg);
+
     bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI);
     bool isIdempotentRMW(AtomicRMWInst *AI);
     bool simplifyIdempotentRMW(AtomicRMWInst *AI);
 
     bool expandAtomicOpToLibcall(Instruction *I, unsigned Size, unsigned Align,
                                  Value *PointerOperand, Value *ValueOperand,
                                  Value *CASExpected, AtomicOrdering Ordering,
                                  AtomicOrdering Ordering2,
                                  ArrayRef<RTLIB::Libcall> Libcalls);
     void expandAtomicLoadToLibcall(LoadInst *LI);
     void expandAtomicStoreToLibcall(StoreInst *LI);
     void expandAtomicRMWToLibcall(AtomicRMWInst *I);
     void expandAtomicCASToLibcall(AtomicCmpXchgInst *I);
+
+    friend bool
+    llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
+                                   CreateCmpXchgInstFun CreateCmpXchg);
   };
 }
 
 char AtomicExpand::ID = 0;
 char &llvm::AtomicExpandID = AtomicExpand::ID;
 INITIALIZE_TM_PASS(AtomicExpand, "atomic-expand", "Expand Atomic instructions",
                    false, false)
 
if (LI) {
       if (CASI->getCompareOperand()->getType()->isPointerTy() ) {
         // TODO: add a TLI hook to control this so that each target can
         // convert to lowering the original type one at a time.
         CASI = convertCmpXchgToIntegerType(CASI);
         assert(CASI->getCompareOperand()->getType()->isIntegerTy() &&
                "invariant broken");
         MadeChange = true;
       }
-      
+      unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
+      unsigned ValueSize = getAtomicOpSize(CASI);
+      if (ValueSize < MinCASSize) {
+        assert(!TLI->shouldExpandAtomicCmpXchgInIR(CASI) &&
+               "MinCmpXchgSizeInBits not yet supported for LL/SC expansions.");
+        expandPartwordCmpXchg(CASI);
+      } else {
         if (TLI->shouldExpandAtomicCmpXchgInIR(CASI))
           MadeChange |= expandAtomicCmpXchg(CASI);
       }
     }
+  }
   return MadeChange;
 }
 
 bool AtomicExpand::bracketInstWithFences(Instruction *I, AtomicOrdering Order,
                                          bool IsStore, bool IsLoad) {
   IRBuilder<> Builder(I);
 
   auto LeadingFence = TLI->emitLeadingFence(Builder, Order, IsStore, IsLoad);
LoadInst *AtomicExpand::convertAtomicLoadToIntegerType(LoadInst *LI) {
   return NewLI;
 }
 
 bool AtomicExpand::tryExpandAtomicLoad(LoadInst *LI) {
   switch (TLI->shouldExpandAtomicLoadInIR(LI)) {
   case TargetLoweringBase::AtomicExpansionKind::None:
     return false;
   case TargetLoweringBase::AtomicExpansionKind::LLSC:
-    return expandAtomicOpToLLSC(
-        LI, LI->getPointerOperand(), LI->getOrdering(),
+    expandAtomicOpToLLSC(
+        LI, LI->getType(), LI->getPointerOperand(), LI->getOrdering(),
         [](IRBuilder<> &Builder, Value *Loaded) { return Loaded; });
+    return true;
   case TargetLoweringBase::AtomicExpansionKind::LLOnly:
     return expandAtomicLoadToLL(LI);
   case TargetLoweringBase::AtomicExpansionKind::CmpXChg:
     return expandAtomicLoadToCmpXchg(LI);
   }
   llvm_unreachable("Unhandled case in tryExpandAtomicLoad");
 }
 
default:
     llvm_unreachable("Unknown atomic op");
   }
 }
 
 bool AtomicExpand::tryExpandAtomicRMW(AtomicRMWInst *AI) {
   switch (TLI->shouldExpandAtomicRMWInIR(AI)) {
   case TargetLoweringBase::AtomicExpansionKind::None:
     return false;
-  case TargetLoweringBase::AtomicExpansionKind::LLSC:
-    return expandAtomicOpToLLSC(AI, AI->getPointerOperand(), AI->getOrdering(),
-                                [&](IRBuilder<> &Builder, Value *Loaded) {
-                                  return performAtomicOp(AI->getOperation(),
-                                                         Builder, Loaded,
-                                                         AI->getValOperand());
-                                });
-  case TargetLoweringBase::AtomicExpansionKind::CmpXChg:
-    return expandAtomicRMWToCmpXchg(AI, createCmpXchgInstFun);
+  case TargetLoweringBase::AtomicExpansionKind::LLSC: {
+    unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
+    unsigned ValueSize = getAtomicOpSize(AI);
+    if (ValueSize < MinCASSize) {
+      llvm_unreachable(
+          "MinCmpXchgSizeInBits not yet supported for LL/SC architectures.");
+    } else {
+      auto PerformOp = [&](IRBuilder<> &Builder, Value *Loaded) {
+        return performAtomicOp(AI->getOperation(), Builder, Loaded,
+      };
+      expandAtomicOpToLLSC(AI, AI->getType(), AI->getPointerOperand(),
+                           AI->getOrdering(), PerformOp);
+    }
+    return true;
+  }
+  case TargetLoweringBase::AtomicExpansionKind::CmpXChg: {
+    unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
+    unsigned ValueSize = getAtomicOpSize(AI);
+    if (ValueSize < MinCASSize) {
+      expandPartwordAtomicRMW(AI,
+                              TargetLoweringBase::AtomicExpansionKind::CmpXChg);
+    } else {
+      expandAtomicRMWToCmpXchg(AI, createCmpXchgInstFun);
+    }
+    return true;
+  }
   default:
     llvm_unreachable("Unhandled case in tryExpandAtomicRMW");
   }
 }
 
-bool AtomicExpand::expandAtomicOpToLLSC(
-    Instruction *I, Value *Addr, AtomicOrdering MemOpOrder,
-    function_ref<Value *(IRBuilder<> &, Value *)> PerformOp) {
+namespace {
+
+/// Result values from createMaskInstrs helper.
+struct PartwordMaskValues {
+  Type *WordType;
+  Type *ValueType;
+  Value *AlignedAddr;
+  Value *ShiftAmt;
+  Value *Mask;
+  Value *Inv_Mask;
+};
+} // end anonymous namespace
+
+/// This is a helper function which builds instructions to provide
+/// values necessary for partword atomic operations. It takes an
+/// incoming address, Addr, and ValueType, and constructs the address,
+/// shift-amounts and masks needed to work with a larger value of size
+/// WordSize.
+///
+/// AlignedAddr: Addr rounded down to a multiple of WordSize
+///
+/// ShiftAmt: Number of bits to right-shift a WordSize value loaded
+///           from AlignAddr for it to have the same value as if
+///           ValueType was loaded from Addr.
+///
+/// Mask: Value to mask with the value loaded from AlignAddr to
+///       include only the part that would've been loaded from Addr.
+///
+/// Inv_Mask: The inverse of Mask.
+
+static PartwordMaskValues createMaskInstrs(IRBuilder<> &Builder, Instruction *I,
+                                           Type *ValueType, Value *Addr,
+                                           unsigned WordSize) {
+  PartwordMaskValues Ret;
+
   BasicBlock *BB = I->getParent();
   Function *F = BB->getParent();
+  Module *M = I->getModule();
+
   LLVMContext &Ctx = F->getContext();
+  const DataLayout &DL = M->getDataLayout();
+
+  unsigned ValueSize = DL.getTypeStoreSize(ValueType);
+
+  assert(ValueSize < WordSize);
+
+  Ret.ValueType = ValueType;
+  Ret.WordType = Type::getIntNTy(Ctx, WordSize * 8);
+
+  Type *WordPtrType =
+      Ret.WordType->getPointerTo(Addr->getType()->getPointerAddressSpace());
+
+  Value *AddrInt = Builder.CreatePtrToInt(Addr, DL.getIntPtrType(Ctx));
+  Ret.AlignedAddr = Builder.CreateIntToPtr(
+      Builder.CreateAnd(AddrInt, ~(uint64_t)(WordSize - 1)), WordPtrType,
+      "AlignedAddr");
+
+  Value *PtrLSB = Builder.CreateAnd(AddrInt, WordSize - 1, "PtrLSB");
+  if (DL.isLittleEndian()) {
+    // turn bytes into bits
+    Ret.ShiftAmt = Builder.CreateShl(PtrLSB, 3);
+  } else {
+    // turn bytes into bits, and count from the other side.
+    Ret.ShiftAmt =
+        Builder.CreateShl(Builder.CreateXor(PtrLSB, WordSize - ValueSize), 3);
+  }
+
+  Ret.ShiftAmt = Builder.CreateTrunc(Ret.ShiftAmt, Ret.WordType, "ShiftAmt");
+  Ret.Mask = Builder.CreateShl(
+      ConstantInt::get(Ret.WordType, (1 << ValueSize * 8) - 1), Ret.ShiftAmt,
+      "Mask");
+  Ret.Inv_Mask = Builder.CreateNot(Ret.Mask, "Inv_Mask");
+
+  return Ret;
+}
+
+/// Emit IR to implement a masked version of a given atomicrmw
+/// operation. (That is, only the bits under the Mask should be
+/// affected by the operation)
+static Value *performMaskedAtomicOp(AtomicRMWInst::BinOp Op,
+                                    IRBuilder<> &Builder, Value *Loaded,
+                                    Value *Shifted_Inc, Value *Inc,
+                                    const PartwordMaskValues &PMV) {
+  switch (Op) {
+  case AtomicRMWInst::Xchg: {
+    Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);
+    Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, Shifted_Inc);
+    return FinalVal;
+  }
+  case AtomicRMWInst::Or:
+  case AtomicRMWInst::Xor:
+    // Or/Xor won't affect any other bits, so can just be done
+    // directly.
+    return performAtomicOp(Op, Builder, Loaded, Shifted_Inc);
+  case AtomicRMWInst::Add:
+  case AtomicRMWInst::Sub:
+  case AtomicRMWInst::And:
+  case AtomicRMWInst::Nand: {
+    // The other arithmetic ops need to be masked into place.
+    Value *NewVal = performAtomicOp(Op, Builder, Loaded, Shifted_Inc);
+    Value *NewVal_Masked = Builder.CreateAnd(NewVal, PMV.Mask);
+    Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);
+    Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Masked);
+    return FinalVal;
+  }
+  case AtomicRMWInst::Max:
+  case AtomicRMWInst::Min:
+  case AtomicRMWInst::UMax:
+  case AtomicRMWInst::UMin: {
+    // Finally, comparison ops will operate on the full value, so
+    // truncate down to the original size, and expand out again after
+    // doing the operation.
+    Value *Loaded_Shiftdown = Builder.CreateTrunc(
+        Builder.CreateLShr(Loaded, PMV.ShiftAmt), PMV.ValueType);
+    Value *NewVal = performAtomicOp(Op, Builder, Loaded_Shiftdown, Inc);
+    Value *NewVal_Shiftup = Builder.CreateShl(
+        Builder.CreateZExt(NewVal, PMV.WordType), PMV.ShiftAmt);
+    Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);
+    Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Shiftup);
+    return FinalVal;
+  }
+  default:
+    llvm_unreachable("Unknown atomic op");
+  }
+}
+
+/// Expand a sub-word atomicrmw operation into an appropriate
+/// word-sized operation.
+///
+/// It will create an LL/SC or cmpxchg loop, as appropriate, the same
+/// way as a typical atomicrmw expansion. The only difference here is
+/// that the operation inside of the loop must operate only upon a
+/// part of the value.
+void AtomicExpand::expandPartwordAtomicRMW(
+    AtomicRMWInst *AI, TargetLoweringBase::AtomicExpansionKind ExpansionKind) {
+
+  assert(ExpansionKind == TargetLoweringBase::AtomicExpansionKind::CmpXChg);
+
+  AtomicOrdering MemOpOrder = AI->getOrdering();
+
+  IRBuilder<> Builder(AI);
+
+  PartwordMaskValues PMV =
+      createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
+                       TLI->getMinCmpXchgSizeInBits() / 8);
+
+  Value *ValOperand_Shifted =
+      Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),
+                        PMV.ShiftAmt, "ValOperand_Shifted");
+
+  auto PerformPartwordOp = [&](IRBuilder<> &Builder, Value *Loaded) {
+    return performMaskedAtomicOp(AI->getOperation(), Builder, Loaded,
+                                 ValOperand_Shifted, AI->getValOperand(), PMV);
+  };
+
+  // TODO: When we're ready to support LLSC conversions too, use
+  // insertRMWLLSCLoop here for ExpansionKind==LLSC.
+  Value *OldResult =
+      insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder,
+                           PerformPartwordOp, createCmpXchgInstFun);
+  Value *FinalOldResult = Builder.CreateTrunc(
+      Builder.CreateLShr(OldResult, PMV.ShiftAmt), PMV.ValueType);
+  AI->replaceAllUsesWith(FinalOldResult);
+  AI->eraseFromParent();
+}
+
+void AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
+  // The basic idea here is that we're expanding a cmpxchg of a
+  // smaller memory size up to a word-sized cmpxchg. To do this, we
+  // need to add a retry-loop for strong cmpxchg, so that
+  // modifications to other parts of the word don't cause a spurious
+  // failure.
+
+  // This generates code like the following:
+  //     [[Setup mask values PMV.*]]
+  //     %NewVal_Shifted = shl i32 %NewVal, %PMV.ShiftAmt
+  //     %Cmp_Shifted = shl i32 %Cmp, %PMV.ShiftAmt
+  //     %InitLoaded = load i32* %addr
+  //     %InitLoaded_MaskOut = and i32 %InitLoaded, %PMV.Inv_Mask
+  //     br partword.cmpxchg.loop
+  // partword.cmpxchg.loop:
+  //     %Loaded_MaskOut = phi i32 [ %InitLoaded_MaskOut, %entry ],
+  //        [ %OldVal_MaskOut, %partword.cmpxchg.failure ]
+  //     %FullWord_NewVal = or i32 %Loaded_MaskOut, %NewVal_Shifted
+  //     %FullWord_Cmp = or i32 %Loaded_MaskOut, %Cmp_Shifted
+  //     %NewCI = cmpxchg i32* %PMV.AlignedAddr, i32 %FullWord_Cmp,
+  //        i32 %FullWord_NewVal success_ordering failure_ordering
+  //     %OldVal = extractvalue { i32, i1 } %NewCI, 0
+  //     %Success = extractvalue { i32, i1 } %NewCI, 1
+  //     br i1 %Success, label %partword.cmpxchg.end,
+  //        label %partword.cmpxchg.failure
+  // partword.cmpxchg.failure:
+  //     %OldVal_MaskOut = and i32 %OldVal, %PMV.Inv_Mask
+  //     %ShouldContinue = icmp ne i32 %Loaded_MaskOut, %OldVal_MaskOut
+  //     br i1 %ShouldContinue, label %partword.cmpxchg.loop,
+  //         label %partword.cmpxchg.end
+  // partword.cmpxchg.end:
+  //    %tmp1 = lshr i32 %OldVal, %PMV.ShiftAmt
+  //    %FinalOldVal = trunc i32 %tmp1 to i8
+  //    %tmp2 = insertvalue { i8, i1 } undef, i8 %FinalOldVal, 0
+  //    %Res = insertvalue { i8, i1 } %25, i1 %Success, 1
+
+  Value *Addr = CI->getPointerOperand();
+  Value *Cmp = CI->getCompareOperand();
+  Value *NewVal = CI->getNewValOperand();
+
+  BasicBlock *BB = CI->getParent();
+  Function *F = BB->getParent();
+  IRBuilder<> Builder(CI);
+  LLVMContext &Ctx = Builder.getContext();
+
+  const int WordSize = TLI->getMinCmpXchgSizeInBits() / 8;
+
+  BasicBlock *EndBB =
+      BB->splitBasicBlock(CI->getIterator(), "partword.cmpxchg.end");
+  auto FailureBB =
+      BasicBlock::Create(Ctx, "partword.cmpxchg.failure", F, EndBB);
+  auto LoopBB = BasicBlock::Create(Ctx, "partword.cmpxchg.loop", F, FailureBB);
+
+  // The split call above "helpfully" added a branch at the end of BB
+  // (to the wrong place).
+  std::prev(BB->end())->eraseFromParent();
+  Builder.SetInsertPoint(BB);
+
+  PartwordMaskValues PMV = createMaskInstrs(
+      Builder, CI, CI->getCompareOperand()->getType(), Addr, WordSize);
+
+  // Shift the incoming values over, into the right location in the word.
+  Value *NewVal_Shifted =
+      Builder.CreateShl(Builder.CreateZExt(NewVal, PMV.WordType), PMV.ShiftAmt);
+  Value *Cmp_Shifted =
+      Builder.CreateShl(Builder.CreateZExt(Cmp, PMV.WordType), PMV.ShiftAmt);
+
+  // Load the entire current word, and mask into place the expected and new
+  // values
+  LoadInst *InitLoaded = Builder.CreateLoad(PMV.WordType, PMV.AlignedAddr);
+  InitLoaded->setVolatile(CI->isVolatile());
+  Value *InitLoaded_MaskOut = Builder.CreateAnd(InitLoaded, PMV.Inv_Mask);
+  Builder.CreateBr(LoopBB);
+
+  // partword.cmpxchg.loop:
+  Builder.SetInsertPoint(LoopBB);
+  PHINode *Loaded_MaskOut = Builder.CreatePHI(PMV.WordType, 2);
+  Loaded_MaskOut->addIncoming(InitLoaded_MaskOut, BB);
+
+  // Mask/Or the expected and new values into place in the loaded word.
+  Value *FullWord_NewVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Shifted);
+  Value *FullWord_Cmp = Builder.CreateOr(Loaded_MaskOut, Cmp_Shifted);
+  AtomicCmpXchgInst *NewCI = Builder.CreateAtomicCmpXchg(
+      PMV.AlignedAddr, FullWord_Cmp, FullWord_NewVal, CI->getSuccessOrdering(),
+      CI->getFailureOrdering(), CI->getSynchScope());
+  NewCI->setVolatile(CI->isVolatile());
+  // When we're building a strong cmpxchg, we need a loop, so you
+  // might think we could use a weak cmpxchg inside. But, using strong
+  // allows the below comparison for ShouldContinue, and we're
+  // expecting the underlying cmpxchg to be a machine instruction,
+  // which is strong anyways.
+  NewCI->setWeak(CI->isWeak());
+
+  Value *OldVal = Builder.CreateExtractValue(NewCI, 0);
+  Value *Success = Builder.CreateExtractValue(NewCI, 1);
+
+  if (CI->isWeak())
+    Builder.CreateBr(EndBB);
+  else
+    Builder.CreateCondBr(Success, EndBB, FailureBB);
+
+  // partword.cmpxchg.failure:
+  Builder.SetInsertPoint(FailureBB);
+  // Upon failure, verify that the masked-out part of the loaded value
+  // has been modified.  If it didn't, abort the cmpxchg, since the
+  // masked-in part must've.
+  Value *OldVal_MaskOut = Builder.CreateAnd(OldVal, PMV.Inv_Mask);
+  Value *ShouldContinue = Builder.CreateICmpNE(Loaded_MaskOut, OldVal_MaskOut);
+  Builder.CreateCondBr(ShouldContinue, LoopBB, EndBB);
+
+  // Add the second value to the phi from above
+  Loaded_MaskOut->addIncoming(OldVal_MaskOut, FailureBB);
+
+  // partword.cmpxchg.end:
+  Builder.SetInsertPoint(CI);
+
+  Value *FinalOldVal = Builder.CreateTrunc(
+      Builder.CreateLShr(OldVal, PMV.ShiftAmt), PMV.ValueType);
+  Value *Res = UndefValue::get(CI->getType());
+  Res = Builder.CreateInsertValue(Res, FinalOldVal, 0);
+  Res = Builder.CreateInsertValue(Res, Success, 1);
+
+  CI->replaceAllUsesWith(Res);
+  CI->eraseFromParent();
+}
+
+void AtomicExpand::expandAtomicOpToLLSC(
+    Instruction *I, Type *ResultType, Value *Addr, AtomicOrdering MemOpOrder,
+    function_ref<Value *(IRBuilder<> &, Value *)> PerformOp) {
+  IRBuilder<> Builder(I);
+  Value *Loaded =
+      insertRMWLLSCLoop(Builder, ResultType, Addr, MemOpOrder, PerformOp);
+
+  I->replaceAllUsesWith(Loaded);
+  I->eraseFromParent();
+}
+
+Value *AtomicExpand::insertRMWLLSCLoop(
+    IRBuilder<> &Builder, Type *ResultTy, Value *Addr,
+    AtomicOrdering MemOpOrder,
+    function_ref<Value *(IRBuilder<> &, Value *)> PerformOp) {
+  LLVMContext &Ctx = Builder.getContext();
+  BasicBlock *BB = Builder.GetInsertBlock();
+  Function *F = BB->getParent();
 
   // Given: atomicrmw some_op iN* %addr, iN %incr ordering
   //
   // The standard expansion we produce is:
   //     [...]
-  //     fence?
   // atomicrmw.start:
   //     %loaded = @load.linked(%addr)
   //     %new = some_op iN %loaded, %incr
   //     %stored = @store_conditional(%new, %addr)
   //     %try_again = icmp i32 ne %stored, 0
   //     br i1 %try_again, label %loop, label %atomicrmw.end
   // atomicrmw.end:
-  //     fence?
   //     [...]
-  BasicBlock *ExitBB = BB->splitBasicBlock(I->getIterator(), "atomicrmw.end");
+  BasicBlock *ExitBB =
+      BB->splitBasicBlock(Builder.GetInsertPoint(), "atomicrmw.end");
   BasicBlock *LoopBB =  BasicBlock::Create(Ctx, "atomicrmw.start", F, ExitBB);
 
-  // This grabs the DebugLoc from I.
-  IRBuilder<> Builder(I);
-
   // The split call above "helpfully" added a branch at the end of BB (to the
-  // wrong place), but we might want a fence too. It's easiest to just remove
-  // the branch entirely.
+  // wrong place).
   std::prev(BB->end())->eraseFromParent();
   Builder.SetInsertPoint(BB);
   Builder.CreateBr(LoopBB);
 
   // Start the main loop block now that we've taken care of the preliminaries.
   Builder.SetInsertPoint(LoopBB);
   Value *Loaded = TLI->emitLoadLinked(Builder, Addr, MemOpOrder);
 
   Value *NewVal = PerformOp(Builder, Loaded);
 
   Value *StoreSuccess =
       TLI->emitStoreConditional(Builder, NewVal, Addr, MemOpOrder);
   Value *TryAgain = Builder.CreateICmpNE(
       StoreSuccess, ConstantInt::get(IntegerType::get(Ctx, 32), 0), "tryagain");
   Builder.CreateCondBr(TryAgain, LoopBB, ExitBB);
 
   Builder.SetInsertPoint(ExitBB, ExitBB->begin());
-
-  I->replaceAllUsesWith(Loaded);
-  I->eraseFromParent();
-
-  return true;
+  return Loaded;
 }
 
 /// Convert an atomic cmpxchg of a non-integral type to an integer cmpxchg of
 /// the equivalent bitwidth.  We used to not support pointer cmpxchg in the
 /// IR.  As a migration step, we convert back to what use to be the standard
 /// way to represent a pointer cmpxchg so that we can update backends one by
 /// one. 
 AtomicCmpXchgInst *AtomicExpand::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) {

 bool AtomicExpand::simplifyIdempotentRMW(AtomicRMWInst* RMWI) {
   if (auto ResultingLoad = TLI->lowerIdempotentRMWIntoFencedLoad(RMWI)) {
     tryExpandAtomicLoad(ResultingLoad);
     return true;
   }
   return false;
 }
 
-bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
-                                    CreateCmpXchgInstFun CreateCmpXchg) {
-  assert(AI);
-
-  AtomicOrdering MemOpOrder = AI->getOrdering() == AtomicOrdering::Unordered
-                                  ? AtomicOrdering::Monotonic
-                                  : AI->getOrdering();
-  Value *Addr = AI->getPointerOperand();
-  BasicBlock *BB = AI->getParent();
+Value *AtomicExpand::insertRMWCmpXchgLoop(
+    IRBuilder<> &Builder, Type *ResultTy, Value *Addr,
+    AtomicOrdering MemOpOrder,
+    function_ref<Value *(IRBuilder<> &, Value *)> PerformOp,
+  LLVMContext &Ctx = Builder.getContext();
+  BasicBlock *BB = Builder.GetInsertBlock();
   Function *F = BB->getParent();
-  LLVMContext &Ctx = F->getContext();
 
   // Given: atomicrmw some_op iN* %addr, iN %incr ordering
   //
   // The standard expansion we produce is:
   //     [...]
   //     %init_loaded = load atomic iN* %addr
   //     br label %loop
   // loop:
   //     %loaded = phi iN [ %init_loaded, %entry ], [ %new_loaded, %loop ]
   //     %new = some_op iN %loaded, %incr
   //     %pair = cmpxchg iN* %addr, iN %loaded, iN %new
   //     %new_loaded = extractvalue { iN, i1 } %pair, 0
   //     %success = extractvalue { iN, i1 } %pair, 1
   //     br i1 %success, label %atomicrmw.end, label %loop
   // atomicrmw.end:
   //     [...]
-  BasicBlock *ExitBB = BB->splitBasicBlock(AI->getIterator(), "atomicrmw.end");
+  BasicBlock *ExitBB =
+      BB->splitBasicBlock(Builder.GetInsertPoint(), "atomicrmw.end");
   BasicBlock *LoopBB = BasicBlock::Create(Ctx, "atomicrmw.start", F, ExitBB);
 
-  // This grabs the DebugLoc from AI.
-  IRBuilder<> Builder(AI);
-
   // The split call above "helpfully" added a branch at the end of BB (to the
   // wrong place), but we want a load. It's easiest to just remove
   // the branch entirely.
   std::prev(BB->end())->eraseFromParent();
   Builder.SetInsertPoint(BB);
-  LoadInst *InitLoaded = Builder.CreateLoad(Addr);
+  LoadInst *InitLoaded = Builder.CreateLoad(ResultTy, Addr);
   // Atomics require at least natural alignment.
-  InitLoaded->setAlignment(AI->getType()->getPrimitiveSizeInBits() / 8);
+  InitLoaded->setAlignment(ResultTy->getPrimitiveSizeInBits() / 8);
   Builder.CreateBr(LoopBB);
 
   // Start the main loop block now that we've taken care of the preliminaries.
   Builder.SetInsertPoint(LoopBB);
-  PHINode *Loaded = Builder.CreatePHI(AI->getType(), 2, "loaded");
+  PHINode *Loaded = Builder.CreatePHI(ResultTy, 2, "loaded");
   Loaded->addIncoming(InitLoaded, BB);
 
-  Value *NewVal =
-      performAtomicOp(AI->getOperation(), Builder, Loaded, AI->getValOperand());
+  Value *NewVal = PerformOp(Builder, Loaded);
 
   Value *NewLoaded = nullptr;
   Value *Success = nullptr;
 
-  CreateCmpXchg(Builder, Addr, Loaded, NewVal, MemOpOrder,
+  CreateCmpXchg(Builder, Addr, Loaded, NewVal,
+                MemOpOrder == AtomicOrdering::Unordered
+                    ? AtomicOrdering::Monotonic
+                    : MemOpOrder,
                 Success, NewLoaded);
   assert(Success && NewLoaded);
 
   Loaded->addIncoming(NewLoaded, LoopBB);
 
   Builder.CreateCondBr(Success, ExitBB, LoopBB);
 
   Builder.SetInsertPoint(ExitBB, ExitBB->begin());
+  return NewLoaded;
+}
 
-  AI->replaceAllUsesWith(NewLoaded);
-  AI->eraseFromParent();
+// Note: This function is exposed externally by AtomicExpandUtils.h
+bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
+                                    CreateCmpXchgInstFun CreateCmpXchg) {
+  IRBuilder<> Builder(AI);
+  Value *Loaded = AtomicExpand::insertRMWCmpXchgLoop(
+      Builder, AI->getType(), AI->getPointerOperand(), AI->getOrdering(),
+      [&](IRBuilder<> &Builder, Value *Loaded) {
+        return performAtomicOp(AI->getOperation(), Builder, Loaded,
+                               AI->getValOperand());
+      },
+      CreateCmpXchg);
 
+  AI->replaceAllUsesWith(Loaded);
+  AI->eraseFromParent();
   return true;
 }
 
 // In order to use one of the sized library calls such as
 // __atomic_fetch_add_4, the alignment must be sufficient, the size
 // must be one of the potentially-specialized sizes, and the value
 // type must actually exist in C on the target (otherwise, the
 // function wouldn't actually be defined.)

llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp

TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) {
   PrefLoopAlignment = 0;
   GatherAllAliasesMaxDepth = 6;
   MinStackArgumentAlignment = 1;
   MinimumJumpTableEntries = 4;
   // TODO: the default will be switched to 0 in the next commit, along
   // with the Target-specific changes necessary.
   MaxAtomicSizeInBitsSupported = 1024;
 
+  MinCmpXchgSizeInBits = 0;
+
   std::fill(std::begin(LibcallRoutineNames), std::end(LibcallRoutineNames), nullptr);
 
   InitLibcallNames(LibcallRoutineNames, TM.getTargetTriple());
   InitCmpLibcallCCs(CmpLibcallCCs);
   InitLibcallCallingConvs(LibcallCallingConvs);
 }
 
 void TargetLoweringBase::initActions() {

llvm/trunk/lib/Target/Sparc/SparcISelLowering.cpp

if (Subtarget->isV9())
     setMaxAtomicSizeInBitsSupported(64);
   else if (false && Subtarget->hasLeonCasa())
     // Test made to fail pending completion of AtomicExpandPass,
     // as this will cause a regression until that work is completed.
     setMaxAtomicSizeInBitsSupported(32);
   else
     setMaxAtomicSizeInBitsSupported(0);
 
+  setMinCmpXchgSizeInBits(32);
+
   setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Legal);
 
   setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Legal);
 
   // Custom Lower Atomic LOAD/STORE
   setOperationAction(ISD::ATOMIC_LOAD, MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Custom);
 

llvm/trunk/test/CodeGen/SPARC/atomics.ll

 entry:
   %0 = load atomic i64, i64* %ptr1 acquire, align 8
   %1 = load atomic i64, i64* %ptr2 acquire, align 8
   %2 = add i64 %0, %1
   store atomic i64 %2, i64* %ptr3 release, align 8
   ret i64 %2
 }
 
+;; TODO: the "move %icc" and related instructions are totally
+;; redundant here. There's something weird happening in optimization
+;; of the success value of cmpxchg.
+
+; CHECK-LABEL: test_cmpxchg_i8
+; CHECK:       and %o1, -4, %o2
+; CHECK:       mov  3, %o3
+; CHECK:       andn %o3, %o1, %o1
+; CHECK:       sll %o1, 3, %o1
+; CHECK:       mov  255, %o3
+; CHECK:       sll %o3, %o1, %o5
+; CHECK:       xor %o5, -1, %o3
+; CHECK:       mov  123, %o4
+; CHECK:       ld [%o2], %g2
+; CHECK:       sll %o4, %o1, %o4
+; CHECK:       and %o0, 255, %o0
+; CHECK:       sll %o0, %o1, %o0
+; CHECK:       andn %g2, %o5, %g2
+; CHECK:       sethi 0, %o5
+; CHECK:      [[LABEL1:\.L.*]]:
+; CHECK:       or %g2, %o4, %g3
+; CHECK:       or %g2, %o0, %g4
+; CHECK:       cas [%o2], %g4, %g3
+; CHECK:       cmp %g3, %g4
+; CHECK:       mov  %o5, %g4
+; CHECK:       move %icc, 1, %g4
+; CHECK:       cmp %g4, 0
+; CHECK:       bne  [[LABEL2:\.L.*]]
+; CHECK:       nop
+; CHECK:       and %g3, %o3, %g4
+; CHECK:       cmp %g2, %g4
+; CHECK:       bne  [[LABEL1]]
+; CHECK:       mov  %g4, %g2
+; CHECK:      [[LABEL2]]:
+; CHECK:       retl
+; CHECK:       srl %g3, %o1, %o0
+define i8 @test_cmpxchg_i8(i8 %a, i8* %ptr) {
+entry:
+  %pair = cmpxchg i8* %ptr, i8 %a, i8 123 monotonic monotonic
+  %b = extractvalue { i8, i1 } %pair, 0
+  ret i8 %b
+}
+
+; CHECK-LABEL: test_cmpxchg_i16
+
+; CHECK:       and %o1, -4, %o2
+; CHECK:       and %o1, 3, %o1
+; CHECK:       xor %o1, 2, %o1
+; CHECK:       sll %o1, 3, %o1
+; CHECK:       sethi 63, %o3
+; CHECK:       or %o3, 1023, %o4
+; CHECK:       sll %o4, %o1, %o5
+; CHECK:       xor %o5, -1, %o3
+; CHECK:       and %o0, %o4, %o4
+; CHECK:       ld [%o2], %g2
+; CHECK:       mov  123, %o0
+; CHECK:       sll %o0, %o1, %o0
+; CHECK:       sll %o4, %o1, %o4
+; CHECK:       andn %g2, %o5, %g2
+; CHECK:       sethi 0, %o5
+; CHECK:      [[LABEL1:\.L.*]]:
+; CHECK:       or %g2, %o0, %g3
+; CHECK:       or %g2, %o4, %g4
+; CHECK:       cas [%o2], %g4, %g3
+; CHECK:       cmp %g3, %g4
+; CHECK:       mov  %o5, %g4
+; CHECK:       move %icc, 1, %g4
+; CHECK:       cmp %g4, 0
+; CHECK:       bne  [[LABEL2:\.L.*]]
+; CHECK:       nop
+; CHECK:       and %g3, %o3, %g4
+; CHECK:       cmp %g2, %g4
+; CHECK:       bne  [[LABEL1]]
+; CHECK:       mov  %g4, %g2
+; CHECK:      [[LABEL2]]:
+; CHECK:       retl
+; CHECK:       srl %g3, %o1, %o0
+define i16 @test_cmpxchg_i16(i16 %a, i16* %ptr) {
+entry:
+  %pair = cmpxchg i16* %ptr, i16 %a, i16 123 monotonic monotonic
+  %b = extractvalue { i16, i1 } %pair, 0
+  ret i16 %b
+}
+
 ; CHECK-LABEL: test_cmpxchg_i32
 ; CHECK:       mov 123, [[R:%[gilo][0-7]]]
 ; CHECK:       cas [%o1], %o0, [[R]]
 
 define i32 @test_cmpxchg_i32(i32 %a, i32* %ptr) {
 entry:
   %pair = cmpxchg i32* %ptr, i32 %a, i32 123 monotonic monotonic
   %b = extractvalue { i32, i1 } %pair, 0
   ret i32 %b
 }
 
 ; CHECK-LABEL: test_cmpxchg_i64
 ; CHECK:       mov 123, [[R:%[gilo][0-7]]]
 ; CHECK:       casx [%o1], %o0, [[R]]
 
 define i64 @test_cmpxchg_i64(i64 %a, i64* %ptr) {
 entry:
   %pair = cmpxchg i64* %ptr, i64 %a, i64 123 monotonic monotonic
   %b = extractvalue { i64, i1 } %pair, 0
   ret i64 %b
 }
 
+; CHECK-LABEL: test_swap_i8
+; CHECK:       mov 42, [[R:%[gilo][0-7]]]
+; CHECK:       cas
+
+define i8 @test_swap_i8(i8 %a, i8* %ptr) {
+entry:
+  %b = atomicrmw xchg i8* %ptr, i8 42 monotonic
+  ret i8 %b
+}
+
+; CHECK-LABEL: test_swap_i16
+; CHECK:       mov 42, [[R:%[gilo][0-7]]]
+; CHECK:       cas
+
+define i16 @test_swap_i16(i16 %a, i16* %ptr) {
+entry:
+  %b = atomicrmw xchg i16* %ptr, i16 42 monotonic
+  ret i16 %b
+}
+
 ; CHECK-LABEL: test_swap_i32
 ; CHECK:       mov 42, [[R:%[gilo][0-7]]]
 ; CHECK:       swap [%o1], [[R]]
 
 define i32 @test_swap_i32(i32 %a, i32* %ptr) {
 entry:
   %b = atomicrmw xchg i32* %ptr, i32 42 monotonic
   ret i32 %b
 }
 
 ; CHECK-LABEL: test_swap_i64
 ; CHECK:       casx [%o1],
 
 define i64 @test_swap_i64(i64 %a, i64* %ptr) {
 entry:
   %b = atomicrmw xchg i64* %ptr, i64 42 monotonic
   ret i64 %b
 }
 
-; CHECK-LABEL: test_load_add_32
+; CHECK-LABEL: test_load_sub_i8
+; CHECK: membar
+; CHECK: .L{{.*}}:
+; CHECK: sub
+; CHECK: cas [{{%[gilo][0-7]}}]
+; CHECK: membar
+define zeroext i8 @test_load_sub_i8(i8* %p, i8 zeroext %v) {
+entry:
+  %0 = atomicrmw sub i8* %p, i8 %v seq_cst
+  ret i8 %0
+}
+
+; CHECK-LABEL: test_load_sub_i16
+; CHECK: membar
+; CHECK: .L{{.*}}:
+; CHECK: sub
+; CHECK: cas [{{%[gilo][0-7]}}]
+; CHECK: membar
+define zeroext i16 @test_load_sub_i16(i16* %p, i16 zeroext %v) {
+entry:
+  %0 = atomicrmw sub i16* %p, i16 %v seq_cst
+  ret i16 %0
+}
+
+; CHECK-LABEL: test_load_add_i32
 ; CHECK: membar
 ; CHECK: add [[V:%[gilo][0-7]]], %o1, [[U:%[gilo][0-7]]]
 ; CHECK: cas [%o0], [[V]], [[U]]
 ; CHECK: membar
-define zeroext i32 @test_load_add_32(i32* %p, i32 zeroext %v) {
+define zeroext i32 @test_load_add_i32(i32* %p, i32 zeroext %v) {
 entry:
   %0 = atomicrmw add i32* %p, i32 %v seq_cst
   ret i32 %0
 }
 
 ; CHECK-LABEL: test_load_sub_64
 ; CHECK: membar
 ; CHECK: sub

llvm/trunk/test/Transforms/AtomicExpand/SPARC/partword.ll

+; RUN: opt -S %s -atomic-expand | FileCheck %s
+
+;; Verify the cmpxchg and atomicrmw expansions where sub-word-size
+;; instructions are not available.
+
+;;; NOTE: this test is mostly target-independent -- any target which
+;;; doesn't support cmpxchg of sub-word sizes would do.
+target datalayout = "E-m:e-i64:64-n32:64-S128"
+target triple = "sparcv9-unknown-unknown"
+
+; CHECK-LABEL: @test_cmpxchg_i8(
+; CHECK:  fence seq_cst
+; CHECK:  %0 = ptrtoint i8* %arg to i64
+; CHECK:  %1 = and i64 %0, -4
+; CHECK:  %AlignedAddr = inttoptr i64 %1 to i32*
+; CHECK:  %PtrLSB = and i64 %0, 3
+; CHECK:  %2 = xor i64 %PtrLSB, 3
+; CHECK:  %3 = shl i64 %2, 3
+; CHECK:  %ShiftAmt = trunc i64 %3 to i32
+; CHECK:  %Mask = shl i32 255, %ShiftAmt
+; CHECK:  %Inv_Mask = xor i32 %Mask, -1
+; CHECK:  %4 = zext i8 %new to i32
+; CHECK:  %5 = shl i32 %4, %ShiftAmt
+; CHECK:  %6 = zext i8 %old to i32
+; CHECK:  %7 = shl i32 %6, %ShiftAmt
+; CHECK:  %8 = load i32, i32* %AlignedAddr
+; CHECK:  %9 = and i32 %8, %Inv_Mask
+; CHECK:  br label %partword.cmpxchg.loop
+; CHECK:partword.cmpxchg.loop:
+; CHECK:  %10 = phi i32 [ %9, %entry ], [ %16, %partword.cmpxchg.failure ]
+; CHECK:  %11 = or i32 %10, %5
+; CHECK:  %12 = or i32 %10, %7
+; CHECK:  %13 = cmpxchg i32* %AlignedAddr, i32 %12, i32 %11 monotonic monotonic
+; CHECK:  %14 = extractvalue { i32, i1 } %13, 0
+; CHECK:  %15 = extractvalue { i32, i1 } %13, 1
+; CHECK:  br i1 %15, label %partword.cmpxchg.end, label %partword.cmpxchg.failure
+; CHECK:partword.cmpxchg.failure:
+; CHECK:  %16 = and i32 %14, %Inv_Mask
+; CHECK:  %17 = icmp ne i32 %10, %16
+; CHECK:  br i1 %17, label %partword.cmpxchg.loop, label %partword.cmpxchg.end
+; CHECK:partword.cmpxchg.end:
+; CHECK:  %18 = lshr i32 %14, %ShiftAmt
+; CHECK:  %19 = trunc i32 %18 to i8
+; CHECK:  %20 = insertvalue { i8, i1 } undef, i8 %19, 0
+; CHECK:  %21 = insertvalue { i8, i1 } %20, i1 %15, 1
+; CHECK:  fence seq_cst
+; CHECK:  %ret = extractvalue { i8, i1 } %21, 0
+; CHECK:  ret i8 %ret
+define i8 @test_cmpxchg_i8(i8* %arg, i8 %old, i8 %new) {
+entry:
+  %ret_succ = cmpxchg i8* %arg, i8 %old, i8 %new seq_cst monotonic
+  %ret = extractvalue { i8, i1 } %ret_succ, 0
+  ret i8 %ret
+}
+
+; CHECK-LABEL: @test_cmpxchg_i16(
+; CHECK:  fence seq_cst
+; CHECK:  %0 = ptrtoint i16* %arg to i64
+; CHECK:  %1 = and i64 %0, -4
+; CHECK:  %AlignedAddr = inttoptr i64 %1 to i32*
+; CHECK:  %PtrLSB = and i64 %0, 3
+; CHECK:  %2 = xor i64 %PtrLSB, 2
+; CHECK:  %3 = shl i64 %2, 3
+; CHECK:  %ShiftAmt = trunc i64 %3 to i32
+; CHECK:  %Mask = shl i32 65535, %ShiftAmt
+; CHECK:  %Inv_Mask = xor i32 %Mask, -1
+; CHECK:  %4 = zext i16 %new to i32
+; CHECK:  %5 = shl i32 %4, %ShiftAmt
+; CHECK:  %6 = zext i16 %old to i32
+; CHECK:  %7 = shl i32 %6, %ShiftAmt
+; CHECK:  %8 = load i32, i32* %AlignedAddr
+; CHECK:  %9 = and i32 %8, %Inv_Mask
+; CHECK:  br label %partword.cmpxchg.loop
+; CHECK:partword.cmpxchg.loop:
+; CHECK:  %10 = phi i32 [ %9, %entry ], [ %16, %partword.cmpxchg.failure ]
+; CHECK:  %11 = or i32 %10, %5
+; CHECK:  %12 = or i32 %10, %7
+; CHECK:  %13 = cmpxchg i32* %AlignedAddr, i32 %12, i32 %11 monotonic monotonic
+; CHECK:  %14 = extractvalue { i32, i1 } %13, 0
+; CHECK:  %15 = extractvalue { i32, i1 } %13, 1
+; CHECK:  br i1 %15, label %partword.cmpxchg.end, label %partword.cmpxchg.failure
+; CHECK:partword.cmpxchg.failure:
+; CHECK:  %16 = and i32 %14, %Inv_Mask
+; CHECK:  %17 = icmp ne i32 %10, %16
+; CHECK:  br i1 %17, label %partword.cmpxchg.loop, label %partword.cmpxchg.end
+; CHECK:partword.cmpxchg.end:
+; CHECK:  %18 = lshr i32 %14, %ShiftAmt
+; CHECK:  %19 = trunc i32 %18 to i16
+; CHECK:  %20 = insertvalue { i16, i1 } undef, i16 %19, 0
+; CHECK:  %21 = insertvalue { i16, i1 } %20, i1 %15, 1
+; CHECK:  fence seq_cst
+; CHECK:  %ret = extractvalue { i16, i1 } %21, 0
+; CHECK:  ret i16 %ret
+define i16 @test_cmpxchg_i16(i16* %arg, i16 %old, i16 %new) {
+entry:
+  %ret_succ = cmpxchg i16* %arg, i16 %old, i16 %new seq_cst monotonic
+  %ret = extractvalue { i16, i1 } %ret_succ, 0
+  ret i16 %ret
+}
+
+
+; CHECK-LABEL: @test_add_i16(
+; CHECK:  fence seq_cst
+; CHECK:  %0 = ptrtoint i16* %arg to i64
+; CHECK:  %1 = and i64 %0, -4
+; CHECK:  %AlignedAddr = inttoptr i64 %1 to i32*
+; CHECK:  %PtrLSB = and i64 %0, 3
+; CHECK:  %2 = xor i64 %PtrLSB, 2
+; CHECK:  %3 = shl i64 %2, 3
+; CHECK:  %ShiftAmt = trunc i64 %3 to i32
+; CHECK:  %Mask = shl i32 65535, %ShiftAmt
+; CHECK:  %Inv_Mask = xor i32 %Mask, -1
+; CHECK:  %4 = zext i16 %val to i32
+; CHECK:  %ValOperand_Shifted = shl i32 %4, %ShiftAmt
+; CHECK:  %5 = load i32, i32* %AlignedAddr, align 4
+; CHECK:  br label %atomicrmw.start
+; CHECK:atomicrmw.start:
+; CHECK:  %loaded = phi i32 [ %5, %entry ], [ %newloaded, %atomicrmw.start ]
+; CHECK:  %new = add i32 %loaded, %ValOperand_Shifted
+; CHECK:  %6 = and i32 %new, %Mask
+; CHECK:  %7 = and i32 %loaded, %Inv_Mask
+; CHECK:  %8 = or i32 %7, %6
+; CHECK:  %9 = cmpxchg i32* %AlignedAddr, i32 %loaded, i32 %8 monotonic monotonic
+; CHECK:  %success = extractvalue { i32, i1 } %9, 1
+; CHECK:  %newloaded = extractvalue { i32, i1 } %9, 0
+; CHECK:  br i1 %success, label %atomicrmw.end, label %atomicrmw.start
+; CHECK:atomicrmw.end:
+; CHECK:  %10 = lshr i32 %newloaded, %ShiftAmt
+; CHECK:  %11 = trunc i32 %10 to i16
+; CHECK:  fence seq_cst
+; CHECK:  ret i16 %11
+define i16 @test_add_i16(i16* %arg, i16 %val) {
+entry:
+  %ret = atomicrmw add i16* %arg, i16 %val seq_cst
+  ret i16 %ret
+}
+
+; CHECK-LABEL: @test_xor_i16(
+; (I'm going to just assert on the bits that differ from add, above.)
+; CHECK:atomicrmw.start:
+; CHECK:  %new = xor i32 %loaded, %ValOperand_Shifted
+; CHECK:  %6 = cmpxchg i32* %AlignedAddr, i32 %loaded, i32 %new monotonic monotonic
+; CHECK:atomicrmw.end:
+define i16 @test_xor_i16(i16* %arg, i16 %val) {
+entry:
+  %ret = atomicrmw xor i16* %arg, i16 %val seq_cst
+  ret i16 %ret
+}
+
+; CHECK-LABEL: @test_min_i16(
+; CHECK:atomicrmw.start:
+; CHECK:  %6 = lshr i32 %loaded, %ShiftAmt
+; CHECK:  %7 = trunc i32 %6 to i16
+; CHECK:  %8 = icmp sle i16 %7, %val
+; CHECK:  %new = select i1 %8, i16 %7, i16 %val
+; CHECK:  %9 = zext i16 %new to i32
+; CHECK:  %10 = shl i32 %9, %ShiftAmt
+; CHECK:  %11 = and i32 %loaded, %Inv_Mask
+; CHECK:  %12 = or i32 %11, %10
+; CHECK:  %13 = cmpxchg i32* %AlignedAddr, i32 %loaded, i32 %12 monotonic monotonic
+; CHECK:atomicrmw.end:
+define i16 @test_min_i16(i16* %arg, i16 %val) {
+entry:
+  %ret = atomicrmw min i16* %arg, i16 %val seq_cst
+  ret i16 %ret
+}