Index: CodeGen/CGBuiltin.cpp =================================================================== --- CodeGen/CGBuiltin.cpp +++ CodeGen/CGBuiltin.cpp @@ -281,6 +281,19 @@ return CGF.Builder.CreateAdd(Result, ConstantInt::get(IntTy, 1)); } +static Value *EmitAtomicDecrementValue(CodeGenFunction &CGF, const CallExpr *E, + AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) { + assert(E->getArg(0)->getType()->isPointerType()); + + auto *IntTy = CGF.ConvertType(E->getType()); + auto *Result = CGF.Builder.CreateAtomicRMW( + AtomicRMWInst::Sub, + CGF.EmitScalarExpr(E->getArg(0)), + ConstantInt::get(IntTy, 1), + Ordering); + return CGF.Builder.CreateSub(Result, ConstantInt::get(IntTy, 1)); +} + // Emit a simple mangled intrinsic that has 1 argument and a return type // matching the argument type. static Value *emitUnaryBuiltin(CodeGenFunction &CGF, @@ -817,6 +830,9 @@ _InterlockedIncrement_acq, _InterlockedIncrement_rel, _InterlockedIncrement_nf, + _InterlockedDecrement_acq, + _InterlockedDecrement_rel, + _InterlockedDecrement_nf, __fastfail, }; @@ -940,16 +956,15 @@ return EmitAtomicIncrementValue(*this, E, AtomicOrdering::Release); case MSVCIntrin::_InterlockedIncrement_nf: return EmitAtomicIncrementValue(*this, E, AtomicOrdering::Monotonic); - - case MSVCIntrin::_InterlockedDecrement: { - llvm::Type *IntTy = ConvertType(E->getType()); - AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( - AtomicRMWInst::Sub, - EmitScalarExpr(E->getArg(0)), - ConstantInt::get(IntTy, 1), - llvm::AtomicOrdering::SequentiallyConsistent); - return Builder.CreateSub(RMWI, ConstantInt::get(IntTy, 1)); - } + case MSVCIntrin::_InterlockedDecrement_acq: + return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Acquire); + case MSVCIntrin::_InterlockedDecrement_rel: + return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Release); + case MSVCIntrin::_InterlockedDecrement_nf: + return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Monotonic); + + case MSVCIntrin::_InterlockedDecrement: + return EmitAtomicDecrementValue(*this, E); case MSVCIntrin::_InterlockedIncrement: return EmitAtomicIncrementValue(*this, E); @@ -6318,6 +6333,18 @@ case ARM::BI_InterlockedIncrement_nf: case ARM::BI_InterlockedIncrement64_nf: return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_nf, E); + case ARM::BI_InterlockedDecrement16_acq: + case ARM::BI_InterlockedDecrement_acq: + case ARM::BI_InterlockedDecrement64_acq: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_acq, E); + case ARM::BI_InterlockedDecrement16_rel: + case ARM::BI_InterlockedDecrement_rel: + case ARM::BI_InterlockedDecrement64_rel: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_rel, E); + case ARM::BI_InterlockedDecrement16_nf: + case ARM::BI_InterlockedDecrement_nf: + case ARM::BI_InterlockedDecrement64_nf: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_nf, E); } // Get the last argument, which specifies the vector type. @@ -8906,6 +8933,18 @@ case AArch64::BI_InterlockedIncrement_nf: case AArch64::BI_InterlockedIncrement64_nf: return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_nf, E); + case AArch64::BI_InterlockedDecrement16_acq: + case AArch64::BI_InterlockedDecrement_acq: + case AArch64::BI_InterlockedDecrement64_acq: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_acq, E); + case AArch64::BI_InterlockedDecrement16_rel: + case AArch64::BI_InterlockedDecrement_rel: + case AArch64::BI_InterlockedDecrement64_rel: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_rel, E); + case AArch64::BI_InterlockedDecrement16_nf: + case AArch64::BI_InterlockedDecrement_nf: + case AArch64::BI_InterlockedDecrement64_nf: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_nf, E); case AArch64::BI_InterlockedAdd: { Value *Arg0 = EmitScalarExpr(E->getArg(0)); Index: CodeGen/ms-intrinsics.c =================================================================== --- CodeGen/ms-intrinsics.c +++ CodeGen/ms-intrinsics.c @@ -1259,6 +1259,87 @@ // CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i64 [[TMP]], 1 // CHECK-ARM-ARM64: ret i64 [[RESULT]] // CHECK-ARM-ARM64: } + +short test_InterlockedDecrement16_acq(short volatile *Addend) { + return _InterlockedDecrement16_acq(Addend); +} +// CHECK-ARM-ARM64: define{{.*}}i16 @test_InterlockedDecrement16_acq(i16*{{[a-z_ ]*}}%Addend){{.*}}{ +// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub i16* %Addend, i16 1 acquire +// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i16 [[TMP]], -1 +// CHECK-ARM-ARM64: ret i16 [[RESULT]] +// CHECK-ARM-ARM64: } + +short test_InterlockedDecrement16_rel(short volatile *Addend) { + return _InterlockedDecrement16_rel(Addend); +} +// CHECK-ARM-ARM64: define{{.*}}i16 @test_InterlockedDecrement16_rel(i16*{{[a-z_ ]*}}%Addend){{.*}}{ +// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub i16* %Addend, i16 1 release +// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i16 [[TMP]], -1 +// CHECK-ARM-ARM64: ret i16 [[RESULT]] +// CHECK-ARM-ARM64: } + +short test_InterlockedDecrement16_nf(short volatile *Addend) { + return _InterlockedDecrement16_nf(Addend); +} +// CHECK-ARM-ARM64: define{{.*}}i16 @test_InterlockedDecrement16_nf(i16*{{[a-z_ ]*}}%Addend){{.*}}{ +// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub i16* %Addend, i16 1 monotonic +// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i16 [[TMP]], -1 +// CHECK-ARM-ARM64: ret i16 [[RESULT]] +// CHECK-ARM-ARM64: } + +long test_InterlockedDecrement_acq(long volatile *Addend) { + return _InterlockedDecrement_acq(Addend); +} +// CHECK-ARM-ARM64: define{{.*}}i32 @test_InterlockedDecrement_acq(i32*{{[a-z_ ]*}}%Addend){{.*}}{ +// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub i32* %Addend, i32 1 acquire +// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i32 [[TMP]], -1 +// CHECK-ARM-ARM64: ret i32 [[RESULT]] +// CHECK-ARM-ARM64: } + +long test_InterlockedDecrement_rel(long volatile *Addend) { + return _InterlockedDecrement_rel(Addend); +} +// CHECK-ARM-ARM64: define{{.*}}i32 @test_InterlockedDecrement_rel(i32*{{[a-z_ ]*}}%Addend){{.*}}{ +// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub i32* %Addend, i32 1 release +// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i32 [[TMP]], -1 +// CHECK-ARM-ARM64: ret i32 [[RESULT]] +// CHECK-ARM-ARM64: } + +long test_InterlockedDecrement_nf(long volatile *Addend) { + return _InterlockedDecrement_nf(Addend); +} +// CHECK-ARM-ARM64: define{{.*}}i32 @test_InterlockedDecrement_nf(i32*{{[a-z_ ]*}}%Addend){{.*}}{ +// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub i32* %Addend, i32 1 monotonic +// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i32 [[TMP]], -1 +// CHECK-ARM-ARM64: ret i32 [[RESULT]] +// CHECK-ARM-ARM64: } + +__int64 test_InterlockedDecrement64_acq(__int64 volatile *Addend) { + return _InterlockedDecrement64_acq(Addend); +} +// CHECK-ARM-ARM64: define{{.*}}i64 @test_InterlockedDecrement64_acq(i64*{{[a-z_ ]*}}%Addend){{.*}}{ +// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub i64* %Addend, i64 1 acquire +// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i64 [[TMP]], -1 +// CHECK-ARM-ARM64: ret i64 [[RESULT]] +// CHECK-ARM-ARM64: } + +__int64 test_InterlockedDecrement64_rel(__int64 volatile *Addend) { + return _InterlockedDecrement64_rel(Addend); +} +// CHECK-ARM-ARM64: define{{.*}}i64 @test_InterlockedDecrement64_rel(i64*{{[a-z_ ]*}}%Addend){{.*}}{ +// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub i64* %Addend, i64 1 release +// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i64 [[TMP]], -1 +// CHECK-ARM-ARM64: ret i64 [[RESULT]] +// CHECK-ARM-ARM64: } + +__int64 test_InterlockedDecrement64_nf(__int64 volatile *Addend) { + return _InterlockedDecrement64_nf(Addend); +} +// CHECK-ARM-ARM64: define{{.*}}i64 @test_InterlockedDecrement64_nf(i64*{{[a-z_ ]*}}%Addend){{.*}}{ +// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub i64* %Addend, i64 1 monotonic +// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i64 [[TMP]], -1 +// CHECK-ARM-ARM64: ret i64 [[RESULT]] +// CHECK-ARM-ARM64: } #endif #if !defined(__aarch64__) Index: Headers/intrin.h =================================================================== --- Headers/intrin.h +++ Headers/intrin.h @@ -360,42 +360,15 @@ |* Interlocked Decrement \*----------------------------------------------------------------------------*/ #if defined(__arm__) || defined(__aarch64__) -static __inline__ short __DEFAULT_FN_ATTRS -_InterlockedDecrement16_acq(short volatile *_Value) { - return __atomic_sub_fetch(_Value, 1, __ATOMIC_ACQUIRE); -} -static __inline__ short __DEFAULT_FN_ATTRS -_InterlockedDecrement16_nf(short volatile *_Value) { - return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELAXED); -} -static __inline__ short __DEFAULT_FN_ATTRS -_InterlockedDecrement16_rel(short volatile *_Value) { - return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELEASE); -} -static __inline__ long __DEFAULT_FN_ATTRS -_InterlockedDecrement_acq(long volatile *_Value) { - return __atomic_sub_fetch(_Value, 1, __ATOMIC_ACQUIRE); -} -static __inline__ long __DEFAULT_FN_ATTRS -_InterlockedDecrement_nf(long volatile *_Value) { - return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELAXED); -} -static __inline__ long __DEFAULT_FN_ATTRS -_InterlockedDecrement_rel(long volatile *_Value) { - return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELEASE); -} -static __inline__ __int64 __DEFAULT_FN_ATTRS -_InterlockedDecrement64_acq(__int64 volatile *_Value) { - return __atomic_sub_fetch(_Value, 1, __ATOMIC_ACQUIRE); -} -static __inline__ __int64 __DEFAULT_FN_ATTRS -_InterlockedDecrement64_nf(__int64 volatile *_Value) { - return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELAXED); -} -static __inline__ __int64 __DEFAULT_FN_ATTRS -_InterlockedDecrement64_rel(__int64 volatile *_Value) { - return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELEASE); -} +short _InterlockedDecrement16_acq(short volatile *_Value); +short _InterlockedDecrement16_nf(short volatile *_Value); +short _InterlockedDecrement16_rel(short volatile *_Value); +long _InterlockedDecrement_acq(long volatile *_Value); +long _InterlockedDecrement_nf(long volatile *_Value); +long _InterlockedDecrement_rel(long volatile *_Value); +__int64 _InterlockedDecrement64_acq(__int64 volatile *_Value); +__int64 _InterlockedDecrement64_nf(__int64 volatile *_Value); +__int64 _InterlockedDecrement64_rel(__int64 volatile *_Value); #endif /*----------------------------------------------------------------------------*\ |* Interlocked And Index: clang/Basic/BuiltinsAArch64.def =================================================================== --- clang/Basic/BuiltinsAArch64.def +++ clang/Basic/BuiltinsAArch64.def @@ -192,6 +192,16 @@ TARGET_HEADER_BUILTIN(_InterlockedIncrement64_nf, "LLiLLiD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") TARGET_HEADER_BUILTIN(_InterlockedIncrement64_rel, "LLiLLiD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement16_acq, "ssD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement16_nf, "ssD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement16_rel, "ssD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement_acq, "LiLiD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement_nf, "LiLiD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement_rel, "LiLiD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement64_acq, "LLiLLiD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement64_nf, "LLiLLiD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement64_rel, "LLiLLiD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") + TARGET_HEADER_BUILTIN(_ReadWriteBarrier, "v", "nh", "intrin.h", ALL_MS_LANGUAGES, "") TARGET_HEADER_BUILTIN(__getReg, "ULLii", "nh", "intrin.h", ALL_MS_LANGUAGES, "") TARGET_HEADER_BUILTIN(_ReadStatusReg, "ii", "nh", "intrin.h", ALL_MS_LANGUAGES, "") Index: clang/Basic/BuiltinsARM.def =================================================================== --- clang/Basic/BuiltinsARM.def +++ clang/Basic/BuiltinsARM.def @@ -318,6 +318,16 @@ TARGET_HEADER_BUILTIN(_InterlockedIncrement64_nf, "LLiLLiD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") TARGET_HEADER_BUILTIN(_InterlockedIncrement64_rel, "LLiLLiD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement16_acq, "ssD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement16_nf, "ssD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement16_rel, "ssD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement_acq, "LiLiD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement_nf, "LiLiD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement_rel, "LiLiD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement64_acq, "LLiLLiD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement64_nf, "LLiLLiD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") +TARGET_HEADER_BUILTIN(_InterlockedDecrement64_rel, "LLiLLiD*", "nh", "intrin.h", ALL_MS_LANGUAGES, "") + #undef BUILTIN #undef LANGBUILTIN #undef TARGET_HEADER_BUILTIN