Index: clang/lib/Basic/Targets/SystemZ.h
===================================================================
--- clang/lib/Basic/Targets/SystemZ.h
+++ clang/lib/Basic/Targets/SystemZ.h
@@ -57,7 +57,7 @@
     } else
       resetDataLayout("E-m:e-i1:8:16-i8:8:16-i64:64-f128:64"
                       "-v128:64-a:8:16-n32:64");
-    MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 64;
+    MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 128;
     HasStrictFP = true;
   }
 
Index: clang/test/CodeGen/SystemZ/gnu-atomic-builtins-i128-16Al.c
===================================================================
--- /dev/null
+++ clang/test/CodeGen/SystemZ/gnu-atomic-builtins-i128-16Al.c
@@ -0,0 +1,275 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple s390x-linux-gnu -O1 -emit-llvm %s -o - \
+// RUN:   | FileCheck %s
+//
+// Test GNU atomic builtins for __int128 aligned to 16 bytes, which should be
+// expanded to LLVM I/R by the front end.
+
+#include <stdatomic.h>
+#include <stdint.h>
+
+__int128 Ptr __attribute__((aligned(16)));
+__int128 Ret __attribute__((aligned(16)));
+__int128 Val __attribute__((aligned(16)));
+__int128 Exp __attribute__((aligned(16)));
+__int128 Des __attribute__((aligned(16)));
+
+// CHECK-LABEL: @f1(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load atomic i128, ptr @Ptr seq_cst, align 16
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2:![0-9]+]]
+// CHECK-NEXT:    ret void
+//
+__int128 f1() {
+  return __atomic_load_n(&Ptr, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load atomic i128, ptr @Ptr seq_cst, align 16
+// CHECK-NEXT:    store i128 [[TMP0]], ptr @Ret, align 16
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f2() {
+  __atomic_load(&Ptr, &Ret, memory_order_seq_cst);
+  return Ret;
+}
+
+// CHECK-LABEL: @f3(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store atomic i128 [[TMP0]], ptr @Ptr seq_cst, align 16
+// CHECK-NEXT:    ret void
+//
+void f3() {
+  __atomic_store_n(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 16
+// CHECK-NEXT:    store atomic i128 [[TMP0]], ptr @Ptr seq_cst, align 16
+// CHECK-NEXT:    ret void
+//
+void f4() {
+  __atomic_store(&Ptr, &Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f5(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw xchg ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
+// CHECK-NEXT:    store i128 [[TMP1]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f5() {
+  return __atomic_exchange_n(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f6(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 16
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw xchg ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
+// CHECK-NEXT:    store i128 [[TMP1]], ptr @Ret, align 16
+// CHECK-NEXT:    store i128 [[TMP1]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f6() {
+  __atomic_exchange(&Ptr, &Val, &Ret, memory_order_seq_cst);
+  return Ret;
+}
+
+// CHECK-LABEL: @f7(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Des, align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP1:%.*]] = load i128, ptr @Exp, align 16
+// CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg ptr @Ptr, i128 [[TMP1]], i128 [[TMP0]] seq_cst seq_cst, align 16
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i128, i1 } [[TMP2]], 1
+// CHECK-NEXT:    br i1 [[TMP3]], label [[CMPXCHG_CONTINUE:%.*]], label [[CMPXCHG_STORE_EXPECTED:%.*]]
+// CHECK:       cmpxchg.store_expected:
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i128, i1 } [[TMP2]], 0
+// CHECK-NEXT:    store i128 [[TMP4]], ptr @Exp, align 16
+// CHECK-NEXT:    br label [[CMPXCHG_CONTINUE]]
+// CHECK:       cmpxchg.continue:
+// CHECK-NEXT:    ret i1 [[TMP3]]
+//
+_Bool f7() {
+  return __atomic_compare_exchange_n(&Ptr, &Exp, Des, 0,
+                                     memory_order_seq_cst, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Exp, align 16
+// CHECK-NEXT:    [[TMP1:%.*]] = load i128, ptr @Des, align 16
+// CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg ptr @Ptr, i128 [[TMP0]], i128 [[TMP1]] seq_cst seq_cst, align 16
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i128, i1 } [[TMP2]], 1
+// CHECK-NEXT:    br i1 [[TMP3]], label [[CMPXCHG_CONTINUE:%.*]], label [[CMPXCHG_STORE_EXPECTED:%.*]]
+// CHECK:       cmpxchg.store_expected:
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i128, i1 } [[TMP2]], 0
+// CHECK-NEXT:    store i128 [[TMP4]], ptr @Exp, align 16
+// CHECK-NEXT:    br label [[CMPXCHG_CONTINUE]]
+// CHECK:       cmpxchg.continue:
+// CHECK-NEXT:    ret i1 [[TMP3]]
+//
+_Bool f8() {
+  return __atomic_compare_exchange(&Ptr, &Exp, &Des, 0,
+                                   memory_order_seq_cst, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f9(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw add ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
+// CHECK-NEXT:    [[TMP2:%.*]] = add i128 [[TMP1]], [[TMP0]]
+// CHECK-NEXT:    store i128 [[TMP2]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f9() {
+  return __atomic_add_fetch(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f10(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw sub ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
+// CHECK-NEXT:    [[TMP2:%.*]] = sub i128 [[TMP1]], [[TMP0]]
+// CHECK-NEXT:    store i128 [[TMP2]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f10() {
+  return __atomic_sub_fetch(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f11(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw and ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
+// CHECK-NEXT:    [[TMP2:%.*]] = and i128 [[TMP1]], [[TMP0]]
+// CHECK-NEXT:    store i128 [[TMP2]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f11() {
+  return __atomic_and_fetch(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f12(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw xor ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
+// CHECK-NEXT:    [[TMP2:%.*]] = xor i128 [[TMP1]], [[TMP0]]
+// CHECK-NEXT:    store i128 [[TMP2]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f12() {
+  return __atomic_xor_fetch(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f13(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw or ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
+// CHECK-NEXT:    [[TMP2:%.*]] = or i128 [[TMP1]], [[TMP0]]
+// CHECK-NEXT:    store i128 [[TMP2]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f13() {
+  return __atomic_or_fetch(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f14(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw nand ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
+// CHECK-NEXT:    [[TMP2:%.*]] = and i128 [[TMP1]], [[TMP0]]
+// CHECK-NEXT:    [[TMP3:%.*]] = xor i128 [[TMP2]], -1
+// CHECK-NEXT:    store i128 [[TMP3]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f14() {
+  return __atomic_nand_fetch(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f15(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw add ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
+// CHECK-NEXT:    store i128 [[TMP1]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f15() {
+  return __atomic_fetch_add(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw sub ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
+// CHECK-NEXT:    store i128 [[TMP1]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f16() {
+  return __atomic_fetch_sub(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f17(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw and ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
+// CHECK-NEXT:    store i128 [[TMP1]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f17() {
+  return __atomic_fetch_and(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f18(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw xor ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
+// CHECK-NEXT:    store i128 [[TMP1]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f18() {
+  return __atomic_fetch_xor(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f19(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw or ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
+// CHECK-NEXT:    store i128 [[TMP1]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f19() {
+  return __atomic_fetch_or(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f20(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw nand ptr @Ptr, i128 [[TMP0]] seq_cst, align 16
+// CHECK-NEXT:    store i128 [[TMP1]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f20() {
+  return __atomic_fetch_nand(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f21(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i1 false
+//
+_Bool f21() {
+  return __atomic_always_lock_free(16, &Ptr);
+}
+
+// CHECK-LABEL: @f22(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CALL:%.*]] = tail call zeroext i1 @__atomic_is_lock_free(i64 noundef 16, ptr noundef nonnull @Ptr) #[[ATTR4:[0-9]+]]
+// CHECK-NEXT:    ret i1 [[CALL]]
+//
+_Bool f22() {
+  return __atomic_is_lock_free(16, &Ptr);
+}
Index: clang/test/CodeGen/SystemZ/gnu-atomic-builtins-i128-8Al.c
===================================================================
--- /dev/null
+++ clang/test/CodeGen/SystemZ/gnu-atomic-builtins-i128-8Al.c
@@ -0,0 +1,289 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple s390x-linux-gnu -O1 -emit-llvm %s -o - \
+// RUN:   | FileCheck %s
+//
+// Test GNU atomic builtins for __int128 aligned to 8 bytes only, which should result in libcalls.
+
+#include <stdatomic.h>
+#include <stdint.h>
+
+__int128 Ptr __attribute__((aligned(8)));
+__int128 Ret __attribute__((aligned(8)));
+__int128 Val __attribute__((aligned(8)));
+__int128 Exp __attribute__((aligned(8)));
+__int128 Des __attribute__((aligned(8)));
+
+// CHECK-LABEL: @f1(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @__atomic_load(i64 noundef 16, ptr noundef nonnull @Ptr, ptr noundef nonnull [[AGG_RESULT:%.*]], i32 noundef signext 5)
+// CHECK-NEXT:    ret void
+//
+__int128 f1() {
+  return __atomic_load_n(&Ptr, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @__atomic_load(i64 noundef 16, ptr noundef nonnull @Ptr, ptr noundef nonnull @Ret, i32 noundef signext 5)
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Ret, align 8, !tbaa [[TBAA2:![0-9]+]]
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f2() {
+  __atomic_load(&Ptr, &Ret, memory_order_seq_cst);
+  return Ret;
+}
+
+// CHECK-LABEL: @f3(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DOTATOMICTMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[DOTATOMICTMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    call void @__atomic_store(i64 noundef 16, ptr noundef nonnull @Ptr, ptr noundef nonnull [[DOTATOMICTMP]], i32 noundef signext 5)
+// CHECK-NEXT:    ret void
+//
+void f3() {
+  __atomic_store_n(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @__atomic_store(i64 noundef 16, ptr noundef nonnull @Ptr, ptr noundef nonnull @Val, i32 noundef signext 5)
+// CHECK-NEXT:    ret void
+//
+void f4() {
+  __atomic_store(&Ptr, &Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f5(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DOTATOMICTMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[DOTATOMICTMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    call void @__atomic_exchange(i64 noundef 16, ptr noundef nonnull @Ptr, ptr noundef nonnull [[DOTATOMICTMP]], ptr noundef nonnull [[AGG_RESULT:%.*]], i32 noundef signext 5)
+// CHECK-NEXT:    ret void
+//
+__int128 f5() {
+  return __atomic_exchange_n(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f6(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @__atomic_exchange(i64 noundef 16, ptr noundef nonnull @Ptr, ptr noundef nonnull @Val, ptr noundef nonnull @Ret, i32 noundef signext 5)
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Ret, align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f6() {
+  __atomic_exchange(&Ptr, &Val, &Ret, memory_order_seq_cst);
+  return Ret;
+}
+
+// CHECK-LABEL: @f7(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DOTATOMICTMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Des, align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[DOTATOMICTMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[CALL:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 noundef 16, ptr noundef nonnull @Ptr, ptr noundef nonnull @Exp, ptr noundef nonnull [[DOTATOMICTMP]], i32 noundef signext 5, i32 noundef signext 5)
+// CHECK-NEXT:    ret i1 [[CALL]]
+//
+_Bool f7() {
+  return __atomic_compare_exchange_n(&Ptr, &Exp, Des, 0,
+                                     memory_order_seq_cst, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CALL:%.*]] = tail call zeroext i1 @__atomic_compare_exchange(i64 noundef 16, ptr noundef nonnull @Ptr, ptr noundef nonnull @Exp, ptr noundef nonnull @Des, i32 noundef signext 5, i32 noundef signext 5)
+// CHECK-NEXT:    ret i1 [[CALL]]
+//
+_Bool f8() {
+  return __atomic_compare_exchange(&Ptr, &Exp, &Des, 0,
+                                   memory_order_seq_cst, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f9(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[INDIRECT_ARG_TEMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[INDIRECT_ARG_TEMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    call void @__atomic_fetch_add_16(ptr nonnull sret(i128) align 8 [[TMP]], ptr noundef nonnull @Ptr, ptr noundef nonnull [[INDIRECT_ARG_TEMP]], i32 noundef signext 5)
+// CHECK-NEXT:    [[TMP1:%.*]] = load i128, ptr [[TMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP2:%.*]] = add i128 [[TMP1]], [[TMP0]]
+// CHECK-NEXT:    store i128 [[TMP2]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f9() {
+  return __atomic_add_fetch(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f10(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[INDIRECT_ARG_TEMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[INDIRECT_ARG_TEMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    call void @__atomic_fetch_sub_16(ptr nonnull sret(i128) align 8 [[TMP]], ptr noundef nonnull @Ptr, ptr noundef nonnull [[INDIRECT_ARG_TEMP]], i32 noundef signext 5)
+// CHECK-NEXT:    [[TMP1:%.*]] = load i128, ptr [[TMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP2:%.*]] = sub i128 [[TMP1]], [[TMP0]]
+// CHECK-NEXT:    store i128 [[TMP2]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f10() {
+  return __atomic_sub_fetch(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f11(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[INDIRECT_ARG_TEMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[INDIRECT_ARG_TEMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    call void @__atomic_fetch_and_16(ptr nonnull sret(i128) align 8 [[TMP]], ptr noundef nonnull @Ptr, ptr noundef nonnull [[INDIRECT_ARG_TEMP]], i32 noundef signext 5)
+// CHECK-NEXT:    [[TMP1:%.*]] = load i128, ptr [[TMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP2:%.*]] = and i128 [[TMP1]], [[TMP0]]
+// CHECK-NEXT:    store i128 [[TMP2]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f11() {
+  return __atomic_and_fetch(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f12(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[INDIRECT_ARG_TEMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[INDIRECT_ARG_TEMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    call void @__atomic_fetch_xor_16(ptr nonnull sret(i128) align 8 [[TMP]], ptr noundef nonnull @Ptr, ptr noundef nonnull [[INDIRECT_ARG_TEMP]], i32 noundef signext 5)
+// CHECK-NEXT:    [[TMP1:%.*]] = load i128, ptr [[TMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP2:%.*]] = xor i128 [[TMP1]], [[TMP0]]
+// CHECK-NEXT:    store i128 [[TMP2]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f12() {
+  return __atomic_xor_fetch(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f13(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[INDIRECT_ARG_TEMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[INDIRECT_ARG_TEMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    call void @__atomic_fetch_or_16(ptr nonnull sret(i128) align 8 [[TMP]], ptr noundef nonnull @Ptr, ptr noundef nonnull [[INDIRECT_ARG_TEMP]], i32 noundef signext 5)
+// CHECK-NEXT:    [[TMP1:%.*]] = load i128, ptr [[TMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP2:%.*]] = or i128 [[TMP1]], [[TMP0]]
+// CHECK-NEXT:    store i128 [[TMP2]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f13() {
+  return __atomic_or_fetch(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f14(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[INDIRECT_ARG_TEMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[INDIRECT_ARG_TEMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    call void @__atomic_fetch_nand_16(ptr nonnull sret(i128) align 8 [[TMP]], ptr noundef nonnull @Ptr, ptr noundef nonnull [[INDIRECT_ARG_TEMP]], i32 noundef signext 5)
+// CHECK-NEXT:    [[TMP1:%.*]] = load i128, ptr [[TMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP2:%.*]] = and i128 [[TMP1]], [[TMP0]]
+// CHECK-NEXT:    [[TMP3:%.*]] = xor i128 [[TMP2]], -1
+// CHECK-NEXT:    store i128 [[TMP3]], ptr [[AGG_RESULT:%.*]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+__int128 f14() {
+  return __atomic_nand_fetch(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f15(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[INDIRECT_ARG_TEMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[INDIRECT_ARG_TEMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    call void @__atomic_fetch_add_16(ptr nonnull sret(i128) align 8 [[AGG_RESULT:%.*]], ptr noundef nonnull @Ptr, ptr noundef nonnull [[INDIRECT_ARG_TEMP]], i32 noundef signext 5)
+// CHECK-NEXT:    ret void
+//
+__int128 f15() {
+  return __atomic_fetch_add(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[INDIRECT_ARG_TEMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[INDIRECT_ARG_TEMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    call void @__atomic_fetch_sub_16(ptr nonnull sret(i128) align 8 [[AGG_RESULT:%.*]], ptr noundef nonnull @Ptr, ptr noundef nonnull [[INDIRECT_ARG_TEMP]], i32 noundef signext 5)
+// CHECK-NEXT:    ret void
+//
+__int128 f16() {
+  return __atomic_fetch_sub(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f17(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[INDIRECT_ARG_TEMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[INDIRECT_ARG_TEMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    call void @__atomic_fetch_and_16(ptr nonnull sret(i128) align 8 [[AGG_RESULT:%.*]], ptr noundef nonnull @Ptr, ptr noundef nonnull [[INDIRECT_ARG_TEMP]], i32 noundef signext 5)
+// CHECK-NEXT:    ret void
+//
+__int128 f17() {
+  return __atomic_fetch_and(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f18(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[INDIRECT_ARG_TEMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[INDIRECT_ARG_TEMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    call void @__atomic_fetch_xor_16(ptr nonnull sret(i128) align 8 [[AGG_RESULT:%.*]], ptr noundef nonnull @Ptr, ptr noundef nonnull [[INDIRECT_ARG_TEMP]], i32 noundef signext 5)
+// CHECK-NEXT:    ret void
+//
+__int128 f18() {
+  return __atomic_fetch_xor(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f19(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[INDIRECT_ARG_TEMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[INDIRECT_ARG_TEMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    call void @__atomic_fetch_or_16(ptr nonnull sret(i128) align 8 [[AGG_RESULT:%.*]], ptr noundef nonnull @Ptr, ptr noundef nonnull [[INDIRECT_ARG_TEMP]], i32 noundef signext 5)
+// CHECK-NEXT:    ret void
+//
+__int128 f19() {
+  return __atomic_fetch_or(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f20(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[INDIRECT_ARG_TEMP:%.*]] = alloca i128, align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i128, ptr @Val, align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i128 [[TMP0]], ptr [[INDIRECT_ARG_TEMP]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    call void @__atomic_fetch_nand_16(ptr nonnull sret(i128) align 8 [[AGG_RESULT:%.*]], ptr noundef nonnull @Ptr, ptr noundef nonnull [[INDIRECT_ARG_TEMP]], i32 noundef signext 5)
+// CHECK-NEXT:    ret void
+//
+__int128 f20() {
+  return __atomic_fetch_nand(&Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f21(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i1 false
+//
+_Bool f21() {
+  return __atomic_always_lock_free(16, &Ptr);
+}
+
+// CHECK-LABEL: @f22(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CALL:%.*]] = tail call zeroext i1 @__atomic_is_lock_free(i64 noundef 16, ptr noundef nonnull @Ptr) #[[ATTR6:[0-9]+]]
+// CHECK-NEXT:    ret i1 [[CALL]]
+//
+_Bool f22() {
+  return __atomic_is_lock_free(16, &Ptr);
+}
Index: clang/test/CodeGen/SystemZ/gnu-atomic-builtins-i16.c
===================================================================
--- /dev/null
+++ clang/test/CodeGen/SystemZ/gnu-atomic-builtins-i16.c
@@ -0,0 +1,220 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple s390x-linux-gnu -O1 -emit-llvm %s -o - \
+// RUN:   | FileCheck %s
+//
+// Test GNU atomic builtins for int16_t.
+
+#include <stdatomic.h>
+#include <stdint.h>
+
+// CHECK-LABEL: @f1(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load atomic i16, ptr [[PTR:%.*]] seq_cst, align 2
+// CHECK-NEXT:    ret i16 [[TMP0]]
+//
+int16_t f1(int16_t *Ptr) {
+  return __atomic_load_n(Ptr, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load atomic i16, ptr [[PTR:%.*]] seq_cst, align 2
+// CHECK-NEXT:    store i16 [[TMP0]], ptr [[RET:%.*]], align 2
+// CHECK-NEXT:    ret i16 [[TMP0]]
+//
+int16_t f2(int16_t *Ptr, int16_t *Ret) {
+  __atomic_load(Ptr, Ret, memory_order_seq_cst);
+  return *Ret;
+}
+
+// CHECK-LABEL: @f3(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    store atomic i16 [[VAL:%.*]], ptr [[PTR:%.*]] seq_cst, align 2
+// CHECK-NEXT:    ret void
+//
+void f3(int16_t *Ptr, int16_t Val) {
+  __atomic_store_n(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[VAL:%.*]], align 2
+// CHECK-NEXT:    store atomic i16 [[TMP0]], ptr [[PTR:%.*]] seq_cst, align 2
+// CHECK-NEXT:    ret void
+//
+void f4(int16_t *Ptr, int16_t *Val) {
+  __atomic_store(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f5(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR:%.*]], i16 [[VAL:%.*]] seq_cst, align 2
+// CHECK-NEXT:    ret i16 [[TMP0]]
+//
+int16_t f5(int16_t *Ptr, int16_t Val) {
+  return __atomic_exchange_n(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f6(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[VAL:%.*]], align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw xchg ptr [[PTR:%.*]], i16 [[TMP0]] seq_cst, align 2
+// CHECK-NEXT:    store i16 [[TMP1]], ptr [[RET:%.*]], align 2
+// CHECK-NEXT:    ret i16 [[TMP1]]
+//
+int16_t f6(int16_t *Ptr, int16_t *Val, int16_t *Ret) {
+  __atomic_exchange(Ptr, Val, Ret, memory_order_seq_cst);
+  return *Ret;
+}
+
+// CHECK-LABEL: @f7(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[EXP:%.*]], align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg ptr [[PTR:%.*]], i16 [[TMP0]], i16 [[DES:%.*]] seq_cst seq_cst, align 2
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1
+// CHECK-NEXT:    br i1 [[TMP2]], label [[CMPXCHG_CONTINUE:%.*]], label [[CMPXCHG_STORE_EXPECTED:%.*]]
+// CHECK:       cmpxchg.store_expected:
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i16, i1 } [[TMP1]], 0
+// CHECK-NEXT:    store i16 [[TMP3]], ptr [[EXP]], align 2
+// CHECK-NEXT:    br label [[CMPXCHG_CONTINUE]]
+// CHECK:       cmpxchg.continue:
+// CHECK-NEXT:    ret i1 [[TMP2]]
+//
+_Bool f7(int16_t *Ptr, int16_t *Exp, int16_t Des) {
+  return __atomic_compare_exchange_n(Ptr, Exp, Des, 0,
+                                     memory_order_seq_cst, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[EXP:%.*]], align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr [[DES:%.*]], align 2
+// CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR:%.*]], i16 [[TMP0]], i16 [[TMP1]] seq_cst seq_cst, align 2
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i16, i1 } [[TMP2]], 1
+// CHECK-NEXT:    br i1 [[TMP3]], label [[CMPXCHG_CONTINUE:%.*]], label [[CMPXCHG_STORE_EXPECTED:%.*]]
+// CHECK:       cmpxchg.store_expected:
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i16, i1 } [[TMP2]], 0
+// CHECK-NEXT:    store i16 [[TMP4]], ptr [[EXP]], align 2
+// CHECK-NEXT:    br label [[CMPXCHG_CONTINUE]]
+// CHECK:       cmpxchg.continue:
+// CHECK-NEXT:    ret i1 [[TMP3]]
+//
+_Bool f8(int16_t *Ptr, int16_t *Exp, int16_t *Des) {
+  return __atomic_compare_exchange(Ptr, Exp, Des, 0,
+                                   memory_order_seq_cst, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f9(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw add ptr [[PTR:%.*]], i16 [[VAL:%.*]] seq_cst, align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = add i16 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i16 [[TMP1]]
+//
+int16_t f9(int16_t *Ptr, int16_t Val) {
+  return __atomic_add_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f10(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw sub ptr [[PTR:%.*]], i16 [[VAL:%.*]] seq_cst, align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = sub i16 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i16 [[TMP1]]
+//
+int16_t f10(int16_t *Ptr, int16_t Val) {
+  return __atomic_sub_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f11(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw and ptr [[PTR:%.*]], i16 [[VAL:%.*]] seq_cst, align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i16 [[TMP1]]
+//
+int16_t f11(int16_t *Ptr, int16_t Val) {
+  return __atomic_and_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f12(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw xor ptr [[PTR:%.*]], i16 [[VAL:%.*]] seq_cst, align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = xor i16 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i16 [[TMP1]]
+//
+int16_t f12(int16_t *Ptr, int16_t Val) {
+  return __atomic_xor_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f13(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw or ptr [[PTR:%.*]], i16 [[VAL:%.*]] seq_cst, align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = or i16 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i16 [[TMP1]]
+//
+int16_t f13(int16_t *Ptr, int16_t Val) {
+  return __atomic_or_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f14(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw nand ptr [[PTR:%.*]], i16 [[VAL:%.*]] seq_cst, align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    [[TMP2:%.*]] = xor i16 [[TMP1]], -1
+// CHECK-NEXT:    ret i16 [[TMP2]]
+//
+int16_t f14(int16_t *Ptr, int16_t Val) {
+  return __atomic_nand_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f15(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw add ptr [[PTR:%.*]], i16 [[VAL:%.*]] seq_cst, align 2
+// CHECK-NEXT:    ret i16 [[TMP0]]
+//
+int16_t f15(int16_t *Ptr, int16_t Val) {
+  return __atomic_fetch_add(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw sub ptr [[PTR:%.*]], i16 [[VAL:%.*]] seq_cst, align 2
+// CHECK-NEXT:    ret i16 [[TMP0]]
+//
+int16_t f16(int16_t *Ptr, int16_t Val) {
+  return __atomic_fetch_sub(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f17(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw and ptr [[PTR:%.*]], i16 [[VAL:%.*]] seq_cst, align 2
+// CHECK-NEXT:    ret i16 [[TMP0]]
+//
+int16_t f17(int16_t *Ptr, int16_t Val) {
+  return __atomic_fetch_and(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f18(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw xor ptr [[PTR:%.*]], i16 [[VAL:%.*]] seq_cst, align 2
+// CHECK-NEXT:    ret i16 [[TMP0]]
+//
+int16_t f18(int16_t *Ptr, int16_t Val) {
+  return __atomic_fetch_xor(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f19(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw or ptr [[PTR:%.*]], i16 [[VAL:%.*]] seq_cst, align 2
+// CHECK-NEXT:    ret i16 [[TMP0]]
+//
+int16_t f19(int16_t *Ptr, int16_t Val) {
+  return __atomic_fetch_or(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f20(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw nand ptr [[PTR:%.*]], i16 [[VAL:%.*]] seq_cst, align 2
+// CHECK-NEXT:    ret i16 [[TMP0]]
+//
+int16_t f20(int16_t *Ptr, int16_t Val) {
+  return __atomic_fetch_nand(Ptr, Val, memory_order_seq_cst);
+}
Index: clang/test/CodeGen/SystemZ/gnu-atomic-builtins-i32.c
===================================================================
--- /dev/null
+++ clang/test/CodeGen/SystemZ/gnu-atomic-builtins-i32.c
@@ -0,0 +1,220 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple s390x-linux-gnu -O1 -emit-llvm %s -o - \
+// RUN:   | FileCheck %s
+//
+// Test GNU atomic builtins for int32_t.
+
+#include <stdatomic.h>
+#include <stdint.h>
+
+// CHECK-LABEL: @f1(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load atomic i32, ptr [[PTR:%.*]] seq_cst, align 4
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+int32_t f1(int32_t *Ptr) {
+  return __atomic_load_n(Ptr, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load atomic i32, ptr [[PTR:%.*]] seq_cst, align 4
+// CHECK-NEXT:    store i32 [[TMP0]], ptr [[RET:%.*]], align 4
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+int32_t f2(int32_t *Ptr, int32_t *Ret) {
+  __atomic_load(Ptr, Ret, memory_order_seq_cst);
+  return *Ret;
+}
+
+// CHECK-LABEL: @f3(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    store atomic i32 [[VAL:%.*]], ptr [[PTR:%.*]] seq_cst, align 4
+// CHECK-NEXT:    ret void
+//
+void f3(int32_t *Ptr, int32_t Val) {
+  __atomic_store_n(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[VAL:%.*]], align 4
+// CHECK-NEXT:    store atomic i32 [[TMP0]], ptr [[PTR:%.*]] seq_cst, align 4
+// CHECK-NEXT:    ret void
+//
+void f4(int32_t *Ptr, int32_t *Val) {
+  __atomic_store(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f5(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR:%.*]], i32 [[VAL:%.*]] seq_cst, align 4
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+int32_t f5(int32_t *Ptr, int32_t Val) {
+  return __atomic_exchange_n(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f6(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[VAL:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw xchg ptr [[PTR:%.*]], i32 [[TMP0]] seq_cst, align 4
+// CHECK-NEXT:    store i32 [[TMP1]], ptr [[RET:%.*]], align 4
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
+int32_t f6(int32_t *Ptr, int32_t *Val, int32_t *Ret) {
+  __atomic_exchange(Ptr, Val, Ret, memory_order_seq_cst);
+  return *Ret;
+}
+
+// CHECK-LABEL: @f7(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[EXP:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg ptr [[PTR:%.*]], i32 [[TMP0]], i32 [[DES:%.*]] seq_cst seq_cst, align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+// CHECK-NEXT:    br i1 [[TMP2]], label [[CMPXCHG_CONTINUE:%.*]], label [[CMPXCHG_STORE_EXPECTED:%.*]]
+// CHECK:       cmpxchg.store_expected:
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+// CHECK-NEXT:    store i32 [[TMP3]], ptr [[EXP]], align 4
+// CHECK-NEXT:    br label [[CMPXCHG_CONTINUE]]
+// CHECK:       cmpxchg.continue:
+// CHECK-NEXT:    ret i1 [[TMP2]]
+//
+_Bool f7(int32_t *Ptr, int32_t *Exp, int32_t Des) {
+  return __atomic_compare_exchange_n(Ptr, Exp, Des, 0,
+                                     memory_order_seq_cst, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[EXP:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DES:%.*]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR:%.*]], i32 [[TMP0]], i32 [[TMP1]] seq_cst seq_cst, align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+// CHECK-NEXT:    br i1 [[TMP3]], label [[CMPXCHG_CONTINUE:%.*]], label [[CMPXCHG_STORE_EXPECTED:%.*]]
+// CHECK:       cmpxchg.store_expected:
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
+// CHECK-NEXT:    store i32 [[TMP4]], ptr [[EXP]], align 4
+// CHECK-NEXT:    br label [[CMPXCHG_CONTINUE]]
+// CHECK:       cmpxchg.continue:
+// CHECK-NEXT:    ret i1 [[TMP3]]
+//
+_Bool f8(int32_t *Ptr, int32_t *Exp, int32_t *Des) {
+  return __atomic_compare_exchange(Ptr, Exp, Des, 0,
+                                   memory_order_seq_cst, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f9(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw add ptr [[PTR:%.*]], i32 [[VAL:%.*]] seq_cst, align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
+int32_t f9(int32_t *Ptr, int32_t Val) {
+  return __atomic_add_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f10(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw sub ptr [[PTR:%.*]], i32 [[VAL:%.*]] seq_cst, align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
+int32_t f10(int32_t *Ptr, int32_t Val) {
+  return __atomic_sub_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f11(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw and ptr [[PTR:%.*]], i32 [[VAL:%.*]] seq_cst, align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
+int32_t f11(int32_t *Ptr, int32_t Val) {
+  return __atomic_and_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f12(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw xor ptr [[PTR:%.*]], i32 [[VAL:%.*]] seq_cst, align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
+int32_t f12(int32_t *Ptr, int32_t Val) {
+  return __atomic_xor_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f13(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw or ptr [[PTR:%.*]], i32 [[VAL:%.*]] seq_cst, align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
+int32_t f13(int32_t *Ptr, int32_t Val) {
+  return __atomic_or_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f14(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw nand ptr [[PTR:%.*]], i32 [[VAL:%.*]] seq_cst, align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP1]], -1
+// CHECK-NEXT:    ret i32 [[TMP2]]
+//
+int32_t f14(int32_t *Ptr, int32_t Val) {
+  return __atomic_nand_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f15(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw add ptr [[PTR:%.*]], i32 [[VAL:%.*]] seq_cst, align 4
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+int32_t f15(int32_t *Ptr, int32_t Val) {
+  return __atomic_fetch_add(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw sub ptr [[PTR:%.*]], i32 [[VAL:%.*]] seq_cst, align 4
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+int32_t f16(int32_t *Ptr, int32_t Val) {
+  return __atomic_fetch_sub(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f17(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw and ptr [[PTR:%.*]], i32 [[VAL:%.*]] seq_cst, align 4
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+int32_t f17(int32_t *Ptr, int32_t Val) {
+  return __atomic_fetch_and(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f18(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw xor ptr [[PTR:%.*]], i32 [[VAL:%.*]] seq_cst, align 4
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+int32_t f18(int32_t *Ptr, int32_t Val) {
+  return __atomic_fetch_xor(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f19(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw or ptr [[PTR:%.*]], i32 [[VAL:%.*]] seq_cst, align 4
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+int32_t f19(int32_t *Ptr, int32_t Val) {
+  return __atomic_fetch_or(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f20(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw nand ptr [[PTR:%.*]], i32 [[VAL:%.*]] seq_cst, align 4
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+int32_t f20(int32_t *Ptr, int32_t Val) {
+  return __atomic_fetch_nand(Ptr, Val, memory_order_seq_cst);
+}
Index: clang/test/CodeGen/SystemZ/gnu-atomic-builtins-i64.c
===================================================================
--- /dev/null
+++ clang/test/CodeGen/SystemZ/gnu-atomic-builtins-i64.c
@@ -0,0 +1,220 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple s390x-linux-gnu -O1 -emit-llvm %s -o - \
+// RUN:   | FileCheck %s
+//
+// Test GNU atomic builtins for int64_t.
+
+#include <stdatomic.h>
+#include <stdint.h>
+
+// CHECK-LABEL: @f1(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load atomic i64, ptr [[PTR:%.*]] seq_cst, align 8
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+int64_t f1(int64_t *Ptr) {
+  return __atomic_load_n(Ptr, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load atomic i64, ptr [[PTR:%.*]] seq_cst, align 8
+// CHECK-NEXT:    store i64 [[TMP0]], ptr [[RET:%.*]], align 8
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+int64_t f2(int64_t *Ptr, int64_t *Ret) {
+  __atomic_load(Ptr, Ret, memory_order_seq_cst);
+  return *Ret;
+}
+
+// CHECK-LABEL: @f3(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    store atomic i64 [[VAL:%.*]], ptr [[PTR:%.*]] seq_cst, align 8
+// CHECK-NEXT:    ret void
+//
+void f3(int64_t *Ptr, int64_t Val) {
+  __atomic_store_n(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[VAL:%.*]], align 8
+// CHECK-NEXT:    store atomic i64 [[TMP0]], ptr [[PTR:%.*]] seq_cst, align 8
+// CHECK-NEXT:    ret void
+//
+void f4(int64_t *Ptr, int64_t *Val) {
+  __atomic_store(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f5(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR:%.*]], i64 [[VAL:%.*]] seq_cst, align 8
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+int64_t f5(int64_t *Ptr, int64_t Val) {
+  return __atomic_exchange_n(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f6(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[VAL:%.*]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw xchg ptr [[PTR:%.*]], i64 [[TMP0]] seq_cst, align 8
+// CHECK-NEXT:    store i64 [[TMP1]], ptr [[RET:%.*]], align 8
+// CHECK-NEXT:    ret i64 [[TMP1]]
+//
+int64_t f6(int64_t *Ptr, int64_t *Val, int64_t *Ret) {
+  __atomic_exchange(Ptr, Val, Ret, memory_order_seq_cst);
+  return *Ret;
+}
+
+// CHECK-LABEL: @f7(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[EXP:%.*]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg ptr [[PTR:%.*]], i64 [[TMP0]], i64 [[DES:%.*]] seq_cst seq_cst, align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+// CHECK-NEXT:    br i1 [[TMP2]], label [[CMPXCHG_CONTINUE:%.*]], label [[CMPXCHG_STORE_EXPECTED:%.*]]
+// CHECK:       cmpxchg.store_expected:
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+// CHECK-NEXT:    store i64 [[TMP3]], ptr [[EXP]], align 8
+// CHECK-NEXT:    br label [[CMPXCHG_CONTINUE]]
+// CHECK:       cmpxchg.continue:
+// CHECK-NEXT:    ret i1 [[TMP2]]
+//
+_Bool f7(int64_t *Ptr, int64_t *Exp, int64_t Des) {
+  return __atomic_compare_exchange_n(Ptr, Exp, Des, 0,
+                                     memory_order_seq_cst, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[EXP:%.*]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr [[DES:%.*]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR:%.*]], i64 [[TMP0]], i64 [[TMP1]] seq_cst seq_cst, align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+// CHECK-NEXT:    br i1 [[TMP3]], label [[CMPXCHG_CONTINUE:%.*]], label [[CMPXCHG_STORE_EXPECTED:%.*]]
+// CHECK:       cmpxchg.store_expected:
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
+// CHECK-NEXT:    store i64 [[TMP4]], ptr [[EXP]], align 8
+// CHECK-NEXT:    br label [[CMPXCHG_CONTINUE]]
+// CHECK:       cmpxchg.continue:
+// CHECK-NEXT:    ret i1 [[TMP3]]
+//
+_Bool f8(int64_t *Ptr, int64_t *Exp, int64_t *Des) {
+  return __atomic_compare_exchange(Ptr, Exp, Des, 0,
+                                   memory_order_seq_cst, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f9(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw add ptr [[PTR:%.*]], i64 [[VAL:%.*]] seq_cst, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i64 [[TMP1]]
+//
+int64_t f9(int64_t *Ptr, int64_t Val) {
+  return __atomic_add_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f10(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw sub ptr [[PTR:%.*]], i64 [[VAL:%.*]] seq_cst, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i64 [[TMP1]]
+//
+int64_t f10(int64_t *Ptr, int64_t Val) {
+  return __atomic_sub_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f11(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw and ptr [[PTR:%.*]], i64 [[VAL:%.*]] seq_cst, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i64 [[TMP1]]
+//
+int64_t f11(int64_t *Ptr, int64_t Val) {
+  return __atomic_and_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f12(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw xor ptr [[PTR:%.*]], i64 [[VAL:%.*]] seq_cst, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = xor i64 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i64 [[TMP1]]
+//
+int64_t f12(int64_t *Ptr, int64_t Val) {
+  return __atomic_xor_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f13(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw or ptr [[PTR:%.*]], i64 [[VAL:%.*]] seq_cst, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = or i64 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i64 [[TMP1]]
+//
+int64_t f13(int64_t *Ptr, int64_t Val) {
+  return __atomic_or_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f14(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw nand ptr [[PTR:%.*]], i64 [[VAL:%.*]] seq_cst, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP1]], -1
+// CHECK-NEXT:    ret i64 [[TMP2]]
+//
+int64_t f14(int64_t *Ptr, int64_t Val) {
+  return __atomic_nand_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f15(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw add ptr [[PTR:%.*]], i64 [[VAL:%.*]] seq_cst, align 8
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+int64_t f15(int64_t *Ptr, int64_t Val) {
+  return __atomic_fetch_add(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw sub ptr [[PTR:%.*]], i64 [[VAL:%.*]] seq_cst, align 8
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+int64_t f16(int64_t *Ptr, int64_t Val) {
+  return __atomic_fetch_sub(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f17(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw and ptr [[PTR:%.*]], i64 [[VAL:%.*]] seq_cst, align 8
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+int64_t f17(int64_t *Ptr, int64_t Val) {
+  return __atomic_fetch_and(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f18(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw xor ptr [[PTR:%.*]], i64 [[VAL:%.*]] seq_cst, align 8
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+int64_t f18(int64_t *Ptr, int64_t Val) {
+  return __atomic_fetch_xor(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f19(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw or ptr [[PTR:%.*]], i64 [[VAL:%.*]] seq_cst, align 8
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+int64_t f19(int64_t *Ptr, int64_t Val) {
+  return __atomic_fetch_or(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f20(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw nand ptr [[PTR:%.*]], i64 [[VAL:%.*]] seq_cst, align 8
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+int64_t f20(int64_t *Ptr, int64_t Val) {
+  return __atomic_fetch_nand(Ptr, Val, memory_order_seq_cst);
+}
Index: clang/test/CodeGen/SystemZ/gnu-atomic-builtins-i8.c
===================================================================
--- /dev/null
+++ clang/test/CodeGen/SystemZ/gnu-atomic-builtins-i8.c
@@ -0,0 +1,220 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple s390x-linux-gnu -O1 -emit-llvm %s -o - \
+// RUN:   | FileCheck %s
+//
+// Test GNU atomic builtins for int8_t.
+
+#include <stdatomic.h>
+#include <stdint.h>
+
+// CHECK-LABEL: @f1(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load atomic i8, ptr [[PTR:%.*]] seq_cst, align 1
+// CHECK-NEXT:    ret i8 [[TMP0]]
+//
+int8_t f1(int8_t *Ptr) {
+  return __atomic_load_n(Ptr, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load atomic i8, ptr [[PTR:%.*]] seq_cst, align 1
+// CHECK-NEXT:    store i8 [[TMP0]], ptr [[RET:%.*]], align 1
+// CHECK-NEXT:    ret i8 [[TMP0]]
+//
+int8_t f2(int8_t *Ptr, int8_t *Ret) {
+  __atomic_load(Ptr, Ret, memory_order_seq_cst);
+  return *Ret;
+}
+
+// CHECK-LABEL: @f3(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    store atomic i8 [[VAL:%.*]], ptr [[PTR:%.*]] seq_cst, align 1
+// CHECK-NEXT:    ret void
+//
+void f3(int8_t *Ptr, int8_t Val) {
+  __atomic_store_n(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[VAL:%.*]], align 1
+// CHECK-NEXT:    store atomic i8 [[TMP0]], ptr [[PTR:%.*]] seq_cst, align 1
+// CHECK-NEXT:    ret void
+//
+void f4(int8_t *Ptr, int8_t *Val) {
+  __atomic_store(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f5(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR:%.*]], i8 [[VAL:%.*]] seq_cst, align 1
+// CHECK-NEXT:    ret i8 [[TMP0]]
+//
+int8_t f5(int8_t *Ptr, int8_t Val) {
+  return __atomic_exchange_n(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f6(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[VAL:%.*]], align 1
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw xchg ptr [[PTR:%.*]], i8 [[TMP0]] seq_cst, align 1
+// CHECK-NEXT:    store i8 [[TMP1]], ptr [[RET:%.*]], align 1
+// CHECK-NEXT:    ret i8 [[TMP1]]
+//
+int8_t f6(int8_t *Ptr, int8_t *Val, int8_t *Ret) {
+  __atomic_exchange(Ptr, Val, Ret, memory_order_seq_cst);
+  return *Ret;
+}
+
+// CHECK-LABEL: @f7(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[EXP:%.*]], align 1
+// CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg ptr [[PTR:%.*]], i8 [[TMP0]], i8 [[DES:%.*]] seq_cst seq_cst, align 1
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1
+// CHECK-NEXT:    br i1 [[TMP2]], label [[CMPXCHG_CONTINUE:%.*]], label [[CMPXCHG_STORE_EXPECTED:%.*]]
+// CHECK:       cmpxchg.store_expected:
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i8, i1 } [[TMP1]], 0
+// CHECK-NEXT:    store i8 [[TMP3]], ptr [[EXP]], align 1
+// CHECK-NEXT:    br label [[CMPXCHG_CONTINUE]]
+// CHECK:       cmpxchg.continue:
+// CHECK-NEXT:    ret i1 [[TMP2]]
+//
+_Bool f7(int8_t *Ptr, int8_t *Exp, int8_t Des) {
+  return __atomic_compare_exchange_n(Ptr, Exp, Des, 0,
+                                     memory_order_seq_cst, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[EXP:%.*]], align 1
+// CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr [[DES:%.*]], align 1
+// CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR:%.*]], i8 [[TMP0]], i8 [[TMP1]] seq_cst seq_cst, align 1
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i8, i1 } [[TMP2]], 1
+// CHECK-NEXT:    br i1 [[TMP3]], label [[CMPXCHG_CONTINUE:%.*]], label [[CMPXCHG_STORE_EXPECTED:%.*]]
+// CHECK:       cmpxchg.store_expected:
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i8, i1 } [[TMP2]], 0
+// CHECK-NEXT:    store i8 [[TMP4]], ptr [[EXP]], align 1
+// CHECK-NEXT:    br label [[CMPXCHG_CONTINUE]]
+// CHECK:       cmpxchg.continue:
+// CHECK-NEXT:    ret i1 [[TMP3]]
+//
+_Bool f8(int8_t *Ptr, int8_t *Exp, int8_t *Des) {
+  return __atomic_compare_exchange(Ptr, Exp, Des, 0,
+                                   memory_order_seq_cst, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f9(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw add ptr [[PTR:%.*]], i8 [[VAL:%.*]] seq_cst, align 1
+// CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i8 [[TMP1]]
+//
+int8_t f9(int8_t *Ptr, int8_t Val) {
+  return __atomic_add_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f10(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw sub ptr [[PTR:%.*]], i8 [[VAL:%.*]] seq_cst, align 1
+// CHECK-NEXT:    [[TMP1:%.*]] = sub i8 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i8 [[TMP1]]
+//
+int8_t f10(int8_t *Ptr, int8_t Val) {
+  return __atomic_sub_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f11(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw and ptr [[PTR:%.*]], i8 [[VAL:%.*]] seq_cst, align 1
+// CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i8 [[TMP1]]
+//
+int8_t f11(int8_t *Ptr, int8_t Val) {
+  return __atomic_and_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f12(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw xor ptr [[PTR:%.*]], i8 [[VAL:%.*]] seq_cst, align 1
+// CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i8 [[TMP1]]
+//
+int8_t f12(int8_t *Ptr, int8_t Val) {
+  return __atomic_xor_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f13(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw or ptr [[PTR:%.*]], i8 [[VAL:%.*]] seq_cst, align 1
+// CHECK-NEXT:    [[TMP1:%.*]] = or i8 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    ret i8 [[TMP1]]
+//
+int8_t f13(int8_t *Ptr, int8_t Val) {
+  return __atomic_or_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f14(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw nand ptr [[PTR:%.*]], i8 [[VAL:%.*]] seq_cst, align 1
+// CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP0]], [[VAL]]
+// CHECK-NEXT:    [[TMP2:%.*]] = xor i8 [[TMP1]], -1
+// CHECK-NEXT:    ret i8 [[TMP2]]
+//
+int8_t f14(int8_t *Ptr, int8_t Val) {
+  return __atomic_nand_fetch(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f15(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw add ptr [[PTR:%.*]], i8 [[VAL:%.*]] seq_cst, align 1
+// CHECK-NEXT:    ret i8 [[TMP0]]
+//
+int8_t f15(int8_t *Ptr, int8_t Val) {
+  return __atomic_fetch_add(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw sub ptr [[PTR:%.*]], i8 [[VAL:%.*]] seq_cst, align 1
+// CHECK-NEXT:    ret i8 [[TMP0]]
+//
+int8_t f16(int8_t *Ptr, int8_t Val) {
+  return __atomic_fetch_sub(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f17(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw and ptr [[PTR:%.*]], i8 [[VAL:%.*]] seq_cst, align 1
+// CHECK-NEXT:    ret i8 [[TMP0]]
+//
+int8_t f17(int8_t *Ptr, int8_t Val) {
+  return __atomic_fetch_and(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f18(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw xor ptr [[PTR:%.*]], i8 [[VAL:%.*]] seq_cst, align 1
+// CHECK-NEXT:    ret i8 [[TMP0]]
+//
+int8_t f18(int8_t *Ptr, int8_t Val) {
+  return __atomic_fetch_xor(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f19(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw or ptr [[PTR:%.*]], i8 [[VAL:%.*]] seq_cst, align 1
+// CHECK-NEXT:    ret i8 [[TMP0]]
+//
+int8_t f19(int8_t *Ptr, int8_t Val) {
+  return __atomic_fetch_or(Ptr, Val, memory_order_seq_cst);
+}
+
+// CHECK-LABEL: @f20(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw nand ptr [[PTR:%.*]], i8 [[VAL:%.*]] seq_cst, align 1
+// CHECK-NEXT:    ret i8 [[TMP0]]
+//
+int8_t f20(int8_t *Ptr, int8_t Val) {
+  return __atomic_fetch_nand(Ptr, Val, memory_order_seq_cst);
+}
Index: llvm/lib/Target/SystemZ/SystemZISelLowering.h
===================================================================
--- llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -601,6 +601,26 @@
       SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
       bool PoisonOnly, unsigned Depth) const override;
 
+  AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const override {
+    return AtomicExpansionKind::None;
+  }
+
+  AtomicExpansionKind shouldCastAtomicStoreInIR(StoreInst *SI) const override {
+    return AtomicExpansionKind::None;
+  }
+
+  AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const override {
+    // Expand i128 only as backend currently expands the rest on its
+    // own. TODO: expand them all here instead of in backend.
+    if (RMW->getType()->isIntegerTy(128))
+      return AtomicExpansionKind::CmpXChg;
+    return AtomicExpansionKind::None;
+  }
+
+  AtomicExpansionKind shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const override {
+    return AtomicExpansionKind::None;
+  }
+
   ISD::NodeType getExtendForAtomicOps() const override {
     return ISD::ANY_EXTEND;
   }
Index: llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
===================================================================
--- llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -129,6 +129,8 @@
   setBooleanContents(ZeroOrOneBooleanContent);
   setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
+  setMaxAtomicSizeInBitsSupported(128);
+
   // Instructions are strings of 2-byte aligned 2-byte values.
   setMinFunctionAlignment(Align(2));
   // For performance reasons we prefer 16-byte alignment.
Index: llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
===================================================================
--- llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
+++ llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
@@ -390,6 +390,32 @@
         (getRegSizeInBits(*SrcRC) <= 64 || getRegSizeInBits(*DstRC) <= 64)))
     return true;
 
+  // Allow coalescing inside an AtomicRMW/CmpXchg loop, which has a CDSG
+  // surrounded by COPYs and maybe some other needed instructions.
+  // XXX Better to search def/use lists?
+  MachineBasicBlock *MBB = MI->getParent();
+  if (MBB->isPredecessor(MBB) && MBB->size() < 20) {
+    bool SeenMI = false;
+    bool SeenCDSG = false;
+    for (auto &mi : *MBB) {
+      if (mi.isCopy() && &mi == MI)
+        SeenMI = true;
+      else if (mi.getOpcode() == SystemZ::CDSG)
+        SeenCDSG = true;
+    }
+    if (SeenMI && SeenCDSG)
+      return true;
+  }
+
+  // Allow coalescing into an AtomicRMW/CmpXchg loop.
+  Register DstReg = MI->getOperand(0).getReg();
+  const MachineRegisterInfo *MRI = &MBB->getParent()->getRegInfo();
+  for (const MachineInstr &Use : MRI->use_nodbg_instructions(DstReg)) {
+    if (Use.getOpcode() == SystemZ::CDSG &&
+        Use.getParent()->isPredecessor(Use.getParent()))
+      return true;
+  }
+
   // Allow coalescing of a GR128 subreg COPY only if the live ranges are small
   // and local to one MBB with not too much interferring registers. Otherwise
   // regalloc may run out of registers.
@@ -401,7 +427,6 @@
   LiveInterval &IntGRNar = LIS.getInterval(GRNarReg);
 
   // Check that the two virtual registers are local to MBB.
-  MachineBasicBlock *MBB = MI->getParent();
   MachineInstr *FirstMI_GR128 =
     LIS.getInstructionFromIndex(IntGR128.beginIndex());
   MachineInstr *FirstMI_GRNar =
Index: llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
===================================================================
--- llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
+++ llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
@@ -221,6 +221,8 @@
 } // end anonymous namespace
 
 void SystemZPassConfig::addIRPasses() {
+  addPass(createAtomicExpandPass());
+
   if (getOptLevel() != CodeGenOpt::None) {
     addPass(createSystemZTDCPass());
     addPass(createLoopDataPrefetchPass());
Index: llvm/test/CodeGen/SystemZ/atomicrmw-ops-i128.ll
===================================================================
--- llvm/test/CodeGen/SystemZ/atomicrmw-ops-i128.ll
+++ llvm/test/CodeGen/SystemZ/atomicrmw-ops-i128.ll
@@ -1,102 +1,361 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; Test i128 atomicrmw operations.
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
+;
+; Test expansion of AtomicRMW instructions. The atomicrmw instructions
+; intrinsically always have a natural alignment, so it is up to the front end
+; to generate a libcall in cases of underalignment (AtomicExpandPass would
+; also generate the call in these cases if the AtomicRMW held the actual
+; (insufficient) alignment value.).
 
 ; Check register exchange.
-define i128 @f1(i128 %dummy, ptr %src, i128 %b) {
-; CHECK-LABEL: f1:
-; CHECK: brasl %r14, __sync_lock_test_and_set_16@PLT
-; CHECK: br %r14
+define i128 @atomicrmw_xchg(i128 %dummy, ptr %src, i128 %b) {
+; CHECK-LABEL: atomicrmw_xchg:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r12, %r15, 96(%r15)
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    lg %r1, 8(%r5)
+; CHECK-NEXT:    lg %r0, 0(%r5)
+; CHECK-NEXT:    lg %r13, 8(%r4)
+; CHECK-NEXT:    lg %r12, 0(%r4)
+; CHECK-NEXT:  .LBB0_1: # %atomicrmw.start
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    cdsg %r12, %r0, 0(%r4)
+; CHECK-NEXT:    jl .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
+; CHECK-NEXT:    stg %r12, 0(%r2)
+; CHECK-NEXT:    stg %r13, 8(%r2)
+; CHECK-NEXT:    lmg %r12, %r15, 96(%r15)
+; CHECK-NEXT:    br %r14
   %res = atomicrmw xchg ptr %src, i128 %b seq_cst
   ret i128 %res
 }
 
 ; Check addition of a variable.
-define i128 @f2(i128 %dummy, ptr %src, i128 %b) {
-; CHECK-LABEL: f2:
-; CHECK: brasl %r14, __sync_fetch_and_add_16@PLT
-; CHECK: br %r14
+define i128 @atomicrmw_add(i128 %dummy, ptr %src, i128 %b) {
+; CHECK-LABEL: atomicrmw_add:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    .cfi_offset %r10, -80
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    lg %r0, 8(%r5)
+; CHECK-NEXT:    lg %r1, 0(%r5)
+; CHECK-NEXT:    lg %r13, 8(%r4)
+; CHECK-NEXT:    lg %r12, 0(%r4)
+; CHECK-NEXT:  .LBB1_1: # %atomicrmw.start
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    algrk %r11, %r13, %r0
+; CHECK-NEXT:    lgr %r10, %r12
+; CHECK-NEXT:    alcgr %r10, %r1
+; CHECK-NEXT:    cdsg %r12, %r10, 0(%r4)
+; CHECK-NEXT:    jl .LBB1_1
+; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
+; CHECK-NEXT:    stg %r12, 0(%r2)
+; CHECK-NEXT:    stg %r13, 8(%r2)
+; CHECK-NEXT:    lmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    br %r14
   %res = atomicrmw add ptr %src, i128 %b seq_cst
   ret i128 %res
 }
 
 ; Check subtraction of a variable.
-define i128 @f3(i128 %dummy, ptr %src, i128 %b) {
-; CHECK-LABEL: f3:
-; CHECK: brasl %r14, __sync_fetch_and_sub_16@PLT
-; CHECK: br %r14
+define i128 @atomicrmw_sub(i128 %dummy, ptr %src, i128 %b) {
+; CHECK-LABEL: atomicrmw_sub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    .cfi_offset %r10, -80
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    lg %r0, 8(%r5)
+; CHECK-NEXT:    lg %r1, 0(%r5)
+; CHECK-NEXT:    lg %r13, 8(%r4)
+; CHECK-NEXT:    lg %r12, 0(%r4)
+; CHECK-NEXT:  .LBB2_1: # %atomicrmw.start
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    slgrk %r11, %r13, %r0
+; CHECK-NEXT:    lgr %r10, %r12
+; CHECK-NEXT:    slbgr %r10, %r1
+; CHECK-NEXT:    cdsg %r12, %r10, 0(%r4)
+; CHECK-NEXT:    jl .LBB2_1
+; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
+; CHECK-NEXT:    stg %r12, 0(%r2)
+; CHECK-NEXT:    stg %r13, 8(%r2)
+; CHECK-NEXT:    lmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    br %r14
   %res = atomicrmw sub ptr %src, i128 %b seq_cst
   ret i128 %res
 }
 
 ; Check AND of a variable.
-define i128 @f4(i128 %dummy, ptr %src, i128 %b) {
-; CHECK-LABEL: f4:
-; CHECK: brasl %r14, __sync_fetch_and_and_16@PLT
-; CHECK: br %r14
+define i128 @atomicrmw_and(i128 %dummy, ptr %src, i128 %b) {
+; CHECK-LABEL: atomicrmw_and:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    .cfi_offset %r10, -80
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    lg %r0, 8(%r5)
+; CHECK-NEXT:    lg %r1, 0(%r5)
+; CHECK-NEXT:    lg %r13, 8(%r4)
+; CHECK-NEXT:    lg %r12, 0(%r4)
+; CHECK-NEXT:  .LBB3_1: # %atomicrmw.start
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ngrk %r10, %r12, %r1
+; CHECK-NEXT:    ngrk %r11, %r13, %r0
+; CHECK-NEXT:    cdsg %r12, %r10, 0(%r4)
+; CHECK-NEXT:    jl .LBB3_1
+; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
+; CHECK-NEXT:    stg %r12, 0(%r2)
+; CHECK-NEXT:    stg %r13, 8(%r2)
+; CHECK-NEXT:    lmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    br %r14
   %res = atomicrmw and ptr %src, i128 %b seq_cst
   ret i128 %res
 }
 
 ; Check NAND of a variable.
-define i128 @f5(i128 %dummy, ptr %src, i128 %b) {
-; CHECK-LABEL: f5:
-; CHECK: brasl %r14, __sync_fetch_and_nand_16@PLT
-; CHECK: br %r14
+define i128 @atomicrmw_nand(i128 %dummy, ptr %src, i128 %b) {
+; CHECK-LABEL: atomicrmw_nand:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    .cfi_offset %r10, -80
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    lg %r0, 8(%r5)
+; CHECK-NEXT:    lg %r1, 0(%r5)
+; CHECK-NEXT:    lg %r13, 8(%r4)
+; CHECK-NEXT:    lg %r12, 0(%r4)
+; CHECK-NEXT:  .LBB4_1: # %atomicrmw.start
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    nngrk %r10, %r12, %r1
+; CHECK-NEXT:    nngrk %r11, %r13, %r0
+; CHECK-NEXT:    cdsg %r12, %r10, 0(%r4)
+; CHECK-NEXT:    jl .LBB4_1
+; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
+; CHECK-NEXT:    stg %r12, 0(%r2)
+; CHECK-NEXT:    stg %r13, 8(%r2)
+; CHECK-NEXT:    lmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    br %r14
   %res = atomicrmw nand ptr %src, i128 %b seq_cst
   ret i128 %res
 }
 
 ; Check OR of a variable.
-define i128 @f6(i128 %dummy, ptr %src, i128 %b) {
-; CHECK-LABEL: f6:
-; CHECK: brasl %r14, __sync_fetch_and_or_16@PLT
-; CHECK: br %r14
+define i128 @atomicrmw_or(i128 %dummy, ptr %src, i128 %b) {
+; CHECK-LABEL: atomicrmw_or:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    .cfi_offset %r10, -80
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    lg %r0, 8(%r5)
+; CHECK-NEXT:    lg %r1, 0(%r5)
+; CHECK-NEXT:    lg %r13, 8(%r4)
+; CHECK-NEXT:    lg %r12, 0(%r4)
+; CHECK-NEXT:  .LBB5_1: # %atomicrmw.start
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ogrk %r10, %r12, %r1
+; CHECK-NEXT:    ogrk %r11, %r13, %r0
+; CHECK-NEXT:    cdsg %r12, %r10, 0(%r4)
+; CHECK-NEXT:    jl .LBB5_1
+; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
+; CHECK-NEXT:    stg %r12, 0(%r2)
+; CHECK-NEXT:    stg %r13, 8(%r2)
+; CHECK-NEXT:    lmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    br %r14
   %res = atomicrmw or ptr %src, i128 %b seq_cst
   ret i128 %res
 }
 
 ; Check XOR of a variable.
-define i128 @f7(i128 %dummy, ptr %src, i128 %b) {
-; CHECK-LABEL: f7:
-; CHECK: brasl %r14, __sync_fetch_and_xor_16@PLT
-; CHECK: br %r14
+define i128 @atomicrmw_xor(i128 %dummy, ptr %src, i128 %b) {
+; CHECK-LABEL: atomicrmw_xor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    .cfi_offset %r10, -80
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    lg %r0, 8(%r5)
+; CHECK-NEXT:    lg %r1, 0(%r5)
+; CHECK-NEXT:    lg %r13, 8(%r4)
+; CHECK-NEXT:    lg %r12, 0(%r4)
+; CHECK-NEXT:  .LBB6_1: # %atomicrmw.start
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    xgrk %r10, %r12, %r1
+; CHECK-NEXT:    xgrk %r11, %r13, %r0
+; CHECK-NEXT:    cdsg %r12, %r10, 0(%r4)
+; CHECK-NEXT:    jl .LBB6_1
+; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
+; CHECK-NEXT:    stg %r12, 0(%r2)
+; CHECK-NEXT:    stg %r13, 8(%r2)
+; CHECK-NEXT:    lmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    br %r14
   %res = atomicrmw xor ptr %src, i128 %b seq_cst
   ret i128 %res
 }
 
 ; Check signed minimum.
-define i128 @f8(i128 %dummy, ptr %src, i128 %b) {
-; CHECK-LABEL: f8:
-; CHECK: brasl %r14, __sync_fetch_and_min_16@PLT
-; CHECK: br %r14
+define i128 @atomicrmw_min(i128 %dummy, ptr %src, i128 %b) {
+; CHECK-LABEL: atomicrmw_min:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    .cfi_offset %r10, -80
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    lg %r0, 8(%r5)
+; CHECK-NEXT:    lg %r1, 0(%r5)
+; CHECK-NEXT:    lg %r13, 8(%r4)
+; CHECK-NEXT:    lg %r12, 0(%r4)
+; CHECK-NEXT:  .LBB7_1: # %atomicrmw.start
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    clgr %r13, %r0
+; CHECK-NEXT:    lhi %r3, 0
+; CHECK-NEXT:    lochile %r3, 1
+; CHECK-NEXT:    cgr %r12, %r1
+; CHECK-NEXT:    lhi %r5, 0
+; CHECK-NEXT:    lochile %r5, 1
+; CHECK-NEXT:    locrlh %r3, %r5
+; CHECK-NEXT:    chi %r3, 0
+; CHECK-NEXT:    selgrlh %r11, %r13, %r0
+; CHECK-NEXT:    selgrlh %r10, %r12, %r1
+; CHECK-NEXT:    cdsg %r12, %r10, 0(%r4)
+; CHECK-NEXT:    jl .LBB7_1
+; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
+; CHECK-NEXT:    stg %r12, 0(%r2)
+; CHECK-NEXT:    stg %r13, 8(%r2)
+; CHECK-NEXT:    lmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    br %r14
   %res = atomicrmw min ptr %src, i128 %b seq_cst
   ret i128 %res
 }
 
 ; Check signed maximum.
-define i128 @f9(i128 %dummy, ptr %src, i128 %b) {
-; CHECK-LABEL: f9:
-; CHECK: brasl %r14, __sync_fetch_and_max_16@PLT
-; CHECK: br %r14
+define i128 @atomicrmw_max(i128 %dummy, ptr %src, i128 %b) {
+; CHECK-LABEL: atomicrmw_max:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    .cfi_offset %r10, -80
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    lg %r0, 8(%r5)
+; CHECK-NEXT:    lg %r1, 0(%r5)
+; CHECK-NEXT:    lg %r13, 8(%r4)
+; CHECK-NEXT:    lg %r12, 0(%r4)
+; CHECK-NEXT:  .LBB8_1: # %atomicrmw.start
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    clgr %r13, %r0
+; CHECK-NEXT:    lhi %r3, 0
+; CHECK-NEXT:    lochih %r3, 1
+; CHECK-NEXT:    cgr %r12, %r1
+; CHECK-NEXT:    lhi %r5, 0
+; CHECK-NEXT:    lochih %r5, 1
+; CHECK-NEXT:    locrlh %r3, %r5
+; CHECK-NEXT:    chi %r3, 0
+; CHECK-NEXT:    selgrlh %r11, %r13, %r0
+; CHECK-NEXT:    selgrlh %r10, %r12, %r1
+; CHECK-NEXT:    cdsg %r12, %r10, 0(%r4)
+; CHECK-NEXT:    jl .LBB8_1
+; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
+; CHECK-NEXT:    stg %r12, 0(%r2)
+; CHECK-NEXT:    stg %r13, 8(%r2)
+; CHECK-NEXT:    lmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    br %r14
   %res = atomicrmw max ptr %src, i128 %b seq_cst
   ret i128 %res
 }
 
 ; Check unsigned minimum.
-define i128 @f10(i128 %dummy, ptr %src, i128 %b) {
-; CHECK-LABEL: f10:
-; CHECK: brasl %r14, __sync_fetch_and_umin_16@PLT
-; CHECK: br %r14
+define i128 @atomicrmw_umin(i128 %dummy, ptr %src, i128 %b) {
+; CHECK-LABEL: atomicrmw_umin:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    .cfi_offset %r10, -80
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    lg %r0, 8(%r5)
+; CHECK-NEXT:    lg %r1, 0(%r5)
+; CHECK-NEXT:    lg %r13, 8(%r4)
+; CHECK-NEXT:    lg %r12, 0(%r4)
+; CHECK-NEXT:  .LBB9_1: # %atomicrmw.start
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    clgr %r12, %r1
+; CHECK-NEXT:    lhi %r3, 0
+; CHECK-NEXT:    lochile %r3, 1
+; CHECK-NEXT:    clgr %r13, %r0
+; CHECK-NEXT:    lhi %r5, 0
+; CHECK-NEXT:    lochile %r5, 1
+; CHECK-NEXT:    cgr %r12, %r1
+; CHECK-NEXT:    locre %r3, %r5
+; CHECK-NEXT:    chi %r3, 0
+; CHECK-NEXT:    selgrlh %r11, %r13, %r0
+; CHECK-NEXT:    selgrlh %r10, %r12, %r1
+; CHECK-NEXT:    cdsg %r12, %r10, 0(%r4)
+; CHECK-NEXT:    jl .LBB9_1
+; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
+; CHECK-NEXT:    stg %r12, 0(%r2)
+; CHECK-NEXT:    stg %r13, 8(%r2)
+; CHECK-NEXT:    lmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    br %r14
   %res = atomicrmw umin ptr %src, i128 %b seq_cst
   ret i128 %res
 }
 
 ; Check unsigned maximum.
-define i128 @f11(i128 %dummy, ptr %src, i128 %b) {
-; CHECK-LABEL: f11:
-; CHECK: brasl %r14, __sync_fetch_and_umax_16@PLT
-; CHECK: br %r14
+define i128 @atomicrmw_umax(i128 %dummy, ptr %src, i128 %b) {
+; CHECK-LABEL: atomicrmw_umax:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    .cfi_offset %r10, -80
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    lg %r0, 8(%r5)
+; CHECK-NEXT:    lg %r1, 0(%r5)
+; CHECK-NEXT:    lg %r13, 8(%r4)
+; CHECK-NEXT:    lg %r12, 0(%r4)
+; CHECK-NEXT:  .LBB10_1: # %atomicrmw.start
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    clgr %r12, %r1
+; CHECK-NEXT:    lhi %r3, 0
+; CHECK-NEXT:    lochih %r3, 1
+; CHECK-NEXT:    clgr %r13, %r0
+; CHECK-NEXT:    lhi %r5, 0
+; CHECK-NEXT:    lochih %r5, 1
+; CHECK-NEXT:    cgr %r12, %r1
+; CHECK-NEXT:    locre %r3, %r5
+; CHECK-NEXT:    chi %r3, 0
+; CHECK-NEXT:    selgrlh %r11, %r13, %r0
+; CHECK-NEXT:    selgrlh %r10, %r12, %r1
+; CHECK-NEXT:    cdsg %r12, %r10, 0(%r4)
+; CHECK-NEXT:    jl .LBB10_1
+; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
+; CHECK-NEXT:    stg %r12, 0(%r2)
+; CHECK-NEXT:    stg %r13, 8(%r2)
+; CHECK-NEXT:    lmg %r10, %r15, 80(%r15)
+; CHECK-NEXT:    br %r14
   %res = atomicrmw umax ptr %src, i128 %b seq_cst
   ret i128 %res
 }