diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -1011,16 +1011,342 @@ __fastfail, }; +static Optional +translateArmToMsvcIntrin(unsigned BuiltinID) { + using MSVCIntrin = CodeGenFunction::MSVCIntrin; + switch (BuiltinID) { + default: + break; + case ARM::BI_BitScanForward: + case ARM::BI_BitScanForward64: + return MSVCIntrin::_BitScanForward; + case ARM::BI_BitScanReverse: + case ARM::BI_BitScanReverse64: + return MSVCIntrin::_BitScanReverse; + case ARM::BI_InterlockedAnd64: + return MSVCIntrin::_InterlockedAnd; + case ARM::BI_InterlockedExchange64: + return MSVCIntrin::_InterlockedExchange; + case ARM::BI_InterlockedExchangeAdd64: + return MSVCIntrin::_InterlockedExchangeAdd; + case ARM::BI_InterlockedExchangeSub64: + return MSVCIntrin::_InterlockedExchangeSub; + case ARM::BI_InterlockedOr64: + return MSVCIntrin::_InterlockedOr; + case ARM::BI_InterlockedXor64: + return MSVCIntrin::_InterlockedXor; + case ARM::BI_InterlockedDecrement64: + return MSVCIntrin::_InterlockedDecrement; + case ARM::BI_InterlockedIncrement64: + return MSVCIntrin::_InterlockedIncrement; + case ARM::BI_InterlockedExchangeAdd8_acq: + case ARM::BI_InterlockedExchangeAdd16_acq: + case ARM::BI_InterlockedExchangeAdd_acq: + case ARM::BI_InterlockedExchangeAdd64_acq: + return MSVCIntrin::_InterlockedExchangeAdd_acq; + case ARM::BI_InterlockedExchangeAdd8_rel: + case ARM::BI_InterlockedExchangeAdd16_rel: + case ARM::BI_InterlockedExchangeAdd_rel: + case ARM::BI_InterlockedExchangeAdd64_rel: + return MSVCIntrin::_InterlockedExchangeAdd_rel; + case ARM::BI_InterlockedExchangeAdd8_nf: + case ARM::BI_InterlockedExchangeAdd16_nf: + case ARM::BI_InterlockedExchangeAdd_nf: + case ARM::BI_InterlockedExchangeAdd64_nf: + return MSVCIntrin::_InterlockedExchangeAdd_nf; + case ARM::BI_InterlockedExchange8_acq: + case ARM::BI_InterlockedExchange16_acq: + case ARM::BI_InterlockedExchange_acq: + case ARM::BI_InterlockedExchange64_acq: + return MSVCIntrin::_InterlockedExchange_acq; + case ARM::BI_InterlockedExchange8_rel: + case ARM::BI_InterlockedExchange16_rel: + case ARM::BI_InterlockedExchange_rel: + case ARM::BI_InterlockedExchange64_rel: + return MSVCIntrin::_InterlockedExchange_rel; + case ARM::BI_InterlockedExchange8_nf: + case ARM::BI_InterlockedExchange16_nf: + case ARM::BI_InterlockedExchange_nf: + case ARM::BI_InterlockedExchange64_nf: + return MSVCIntrin::_InterlockedExchange_nf; + case ARM::BI_InterlockedCompareExchange8_acq: + case ARM::BI_InterlockedCompareExchange16_acq: + case ARM::BI_InterlockedCompareExchange_acq: + case ARM::BI_InterlockedCompareExchange64_acq: + return MSVCIntrin::_InterlockedCompareExchange_acq; + case ARM::BI_InterlockedCompareExchange8_rel: + case ARM::BI_InterlockedCompareExchange16_rel: + case ARM::BI_InterlockedCompareExchange_rel: + case ARM::BI_InterlockedCompareExchange64_rel: + return MSVCIntrin::_InterlockedCompareExchange_rel; + case ARM::BI_InterlockedCompareExchange8_nf: + case ARM::BI_InterlockedCompareExchange16_nf: + case ARM::BI_InterlockedCompareExchange_nf: + case ARM::BI_InterlockedCompareExchange64_nf: + return MSVCIntrin::_InterlockedCompareExchange_nf; + case ARM::BI_InterlockedOr8_acq: + case ARM::BI_InterlockedOr16_acq: + case ARM::BI_InterlockedOr_acq: + case ARM::BI_InterlockedOr64_acq: + return MSVCIntrin::_InterlockedOr_acq; + case ARM::BI_InterlockedOr8_rel: + case ARM::BI_InterlockedOr16_rel: + case ARM::BI_InterlockedOr_rel: + case ARM::BI_InterlockedOr64_rel: + return MSVCIntrin::_InterlockedOr_rel; + case ARM::BI_InterlockedOr8_nf: + case ARM::BI_InterlockedOr16_nf: + case ARM::BI_InterlockedOr_nf: + case ARM::BI_InterlockedOr64_nf: + return MSVCIntrin::_InterlockedOr_nf; + case ARM::BI_InterlockedXor8_acq: + case ARM::BI_InterlockedXor16_acq: + case ARM::BI_InterlockedXor_acq: + case ARM::BI_InterlockedXor64_acq: + return MSVCIntrin::_InterlockedXor_acq; + case ARM::BI_InterlockedXor8_rel: + case ARM::BI_InterlockedXor16_rel: + case ARM::BI_InterlockedXor_rel: + case ARM::BI_InterlockedXor64_rel: + return MSVCIntrin::_InterlockedXor_rel; + case ARM::BI_InterlockedXor8_nf: + case ARM::BI_InterlockedXor16_nf: + case ARM::BI_InterlockedXor_nf: + case ARM::BI_InterlockedXor64_nf: + return MSVCIntrin::_InterlockedXor_nf; + case ARM::BI_InterlockedAnd8_acq: + case ARM::BI_InterlockedAnd16_acq: + case ARM::BI_InterlockedAnd_acq: + case ARM::BI_InterlockedAnd64_acq: + return MSVCIntrin::_InterlockedAnd_acq; + case ARM::BI_InterlockedAnd8_rel: + case ARM::BI_InterlockedAnd16_rel: + case ARM::BI_InterlockedAnd_rel: + case ARM::BI_InterlockedAnd64_rel: + return MSVCIntrin::_InterlockedAnd_rel; + case ARM::BI_InterlockedAnd8_nf: + case ARM::BI_InterlockedAnd16_nf: + case ARM::BI_InterlockedAnd_nf: + case ARM::BI_InterlockedAnd64_nf: + return MSVCIntrin::_InterlockedAnd_nf; + case ARM::BI_InterlockedIncrement16_acq: + case ARM::BI_InterlockedIncrement_acq: + case ARM::BI_InterlockedIncrement64_acq: + return MSVCIntrin::_InterlockedIncrement_acq; + case ARM::BI_InterlockedIncrement16_rel: + case ARM::BI_InterlockedIncrement_rel: + case ARM::BI_InterlockedIncrement64_rel: + return MSVCIntrin::_InterlockedIncrement_rel; + case ARM::BI_InterlockedIncrement16_nf: + case ARM::BI_InterlockedIncrement_nf: + case ARM::BI_InterlockedIncrement64_nf: + return MSVCIntrin::_InterlockedIncrement_nf; + case ARM::BI_InterlockedDecrement16_acq: + case ARM::BI_InterlockedDecrement_acq: + case ARM::BI_InterlockedDecrement64_acq: + return MSVCIntrin::_InterlockedDecrement_acq; + case ARM::BI_InterlockedDecrement16_rel: + case ARM::BI_InterlockedDecrement_rel: + case ARM::BI_InterlockedDecrement64_rel: + return MSVCIntrin::_InterlockedDecrement_rel; + case ARM::BI_InterlockedDecrement16_nf: + case ARM::BI_InterlockedDecrement_nf: + case ARM::BI_InterlockedDecrement64_nf: + return MSVCIntrin::_InterlockedDecrement_nf; + } + return None; +} + +static Optional +translateAarch64ToMsvcIntrin(unsigned BuiltinID) { + using MSVCIntrin = CodeGenFunction::MSVCIntrin; + switch (BuiltinID) { + default: + break; + case AArch64::BI_BitScanForward: + case AArch64::BI_BitScanForward64: + return MSVCIntrin::_BitScanForward; + case AArch64::BI_BitScanReverse: + case AArch64::BI_BitScanReverse64: + return MSVCIntrin::_BitScanReverse; + case AArch64::BI_InterlockedAnd64: + return MSVCIntrin::_InterlockedAnd; + case AArch64::BI_InterlockedExchange64: + return MSVCIntrin::_InterlockedExchange; + case AArch64::BI_InterlockedExchangeAdd64: + return MSVCIntrin::_InterlockedExchangeAdd; + case AArch64::BI_InterlockedExchangeSub64: + return MSVCIntrin::_InterlockedExchangeSub; + case AArch64::BI_InterlockedOr64: + return MSVCIntrin::_InterlockedOr; + case AArch64::BI_InterlockedXor64: + return MSVCIntrin::_InterlockedXor; + case AArch64::BI_InterlockedDecrement64: + return MSVCIntrin::_InterlockedDecrement; + case AArch64::BI_InterlockedIncrement64: + return MSVCIntrin::_InterlockedIncrement; + case AArch64::BI_InterlockedExchangeAdd8_acq: + case AArch64::BI_InterlockedExchangeAdd16_acq: + case AArch64::BI_InterlockedExchangeAdd_acq: + case AArch64::BI_InterlockedExchangeAdd64_acq: + return MSVCIntrin::_InterlockedExchangeAdd_acq; + case AArch64::BI_InterlockedExchangeAdd8_rel: + case AArch64::BI_InterlockedExchangeAdd16_rel: + case AArch64::BI_InterlockedExchangeAdd_rel: + case AArch64::BI_InterlockedExchangeAdd64_rel: + return MSVCIntrin::_InterlockedExchangeAdd_rel; + case AArch64::BI_InterlockedExchangeAdd8_nf: + case AArch64::BI_InterlockedExchangeAdd16_nf: + case AArch64::BI_InterlockedExchangeAdd_nf: + case AArch64::BI_InterlockedExchangeAdd64_nf: + return MSVCIntrin::_InterlockedExchangeAdd_nf; + case AArch64::BI_InterlockedExchange8_acq: + case AArch64::BI_InterlockedExchange16_acq: + case AArch64::BI_InterlockedExchange_acq: + case AArch64::BI_InterlockedExchange64_acq: + return MSVCIntrin::_InterlockedExchange_acq; + case AArch64::BI_InterlockedExchange8_rel: + case AArch64::BI_InterlockedExchange16_rel: + case AArch64::BI_InterlockedExchange_rel: + case AArch64::BI_InterlockedExchange64_rel: + return MSVCIntrin::_InterlockedExchange_rel; + case AArch64::BI_InterlockedExchange8_nf: + case AArch64::BI_InterlockedExchange16_nf: + case AArch64::BI_InterlockedExchange_nf: + case AArch64::BI_InterlockedExchange64_nf: + return MSVCIntrin::_InterlockedExchange_nf; + case AArch64::BI_InterlockedCompareExchange8_acq: + case AArch64::BI_InterlockedCompareExchange16_acq: + case AArch64::BI_InterlockedCompareExchange_acq: + case AArch64::BI_InterlockedCompareExchange64_acq: + return MSVCIntrin::_InterlockedCompareExchange_acq; + case AArch64::BI_InterlockedCompareExchange8_rel: + case AArch64::BI_InterlockedCompareExchange16_rel: + case AArch64::BI_InterlockedCompareExchange_rel: + case AArch64::BI_InterlockedCompareExchange64_rel: + return MSVCIntrin::_InterlockedCompareExchange_rel; + case AArch64::BI_InterlockedCompareExchange8_nf: + case AArch64::BI_InterlockedCompareExchange16_nf: + case AArch64::BI_InterlockedCompareExchange_nf: + case AArch64::BI_InterlockedCompareExchange64_nf: + return MSVCIntrin::_InterlockedCompareExchange_nf; + case AArch64::BI_InterlockedOr8_acq: + case AArch64::BI_InterlockedOr16_acq: + case AArch64::BI_InterlockedOr_acq: + case AArch64::BI_InterlockedOr64_acq: + return MSVCIntrin::_InterlockedOr_acq; + case AArch64::BI_InterlockedOr8_rel: + case AArch64::BI_InterlockedOr16_rel: + case AArch64::BI_InterlockedOr_rel: + case AArch64::BI_InterlockedOr64_rel: + return MSVCIntrin::_InterlockedOr_rel; + case AArch64::BI_InterlockedOr8_nf: + case AArch64::BI_InterlockedOr16_nf: + case AArch64::BI_InterlockedOr_nf: + case AArch64::BI_InterlockedOr64_nf: + return MSVCIntrin::_InterlockedOr_nf; + case AArch64::BI_InterlockedXor8_acq: + case AArch64::BI_InterlockedXor16_acq: + case AArch64::BI_InterlockedXor_acq: + case AArch64::BI_InterlockedXor64_acq: + return MSVCIntrin::_InterlockedXor_acq; + case AArch64::BI_InterlockedXor8_rel: + case AArch64::BI_InterlockedXor16_rel: + case AArch64::BI_InterlockedXor_rel: + case AArch64::BI_InterlockedXor64_rel: + return MSVCIntrin::_InterlockedXor_rel; + case AArch64::BI_InterlockedXor8_nf: + case AArch64::BI_InterlockedXor16_nf: + case AArch64::BI_InterlockedXor_nf: + case AArch64::BI_InterlockedXor64_nf: + return MSVCIntrin::_InterlockedXor_nf; + case AArch64::BI_InterlockedAnd8_acq: + case AArch64::BI_InterlockedAnd16_acq: + case AArch64::BI_InterlockedAnd_acq: + case AArch64::BI_InterlockedAnd64_acq: + return MSVCIntrin::_InterlockedAnd_acq; + case AArch64::BI_InterlockedAnd8_rel: + case AArch64::BI_InterlockedAnd16_rel: + case AArch64::BI_InterlockedAnd_rel: + case AArch64::BI_InterlockedAnd64_rel: + return MSVCIntrin::_InterlockedAnd_rel; + case AArch64::BI_InterlockedAnd8_nf: + case AArch64::BI_InterlockedAnd16_nf: + case AArch64::BI_InterlockedAnd_nf: + case AArch64::BI_InterlockedAnd64_nf: + return MSVCIntrin::_InterlockedAnd_nf; + case AArch64::BI_InterlockedIncrement16_acq: + case AArch64::BI_InterlockedIncrement_acq: + case AArch64::BI_InterlockedIncrement64_acq: + return MSVCIntrin::_InterlockedIncrement_acq; + case AArch64::BI_InterlockedIncrement16_rel: + case AArch64::BI_InterlockedIncrement_rel: + case AArch64::BI_InterlockedIncrement64_rel: + return MSVCIntrin::_InterlockedIncrement_rel; + case AArch64::BI_InterlockedIncrement16_nf: + case AArch64::BI_InterlockedIncrement_nf: + case AArch64::BI_InterlockedIncrement64_nf: + return MSVCIntrin::_InterlockedIncrement_nf; + case AArch64::BI_InterlockedDecrement16_acq: + case AArch64::BI_InterlockedDecrement_acq: + case AArch64::BI_InterlockedDecrement64_acq: + return MSVCIntrin::_InterlockedDecrement_acq; + case AArch64::BI_InterlockedDecrement16_rel: + case AArch64::BI_InterlockedDecrement_rel: + case AArch64::BI_InterlockedDecrement64_rel: + return MSVCIntrin::_InterlockedDecrement_rel; + case AArch64::BI_InterlockedDecrement16_nf: + case AArch64::BI_InterlockedDecrement_nf: + case AArch64::BI_InterlockedDecrement64_nf: + return MSVCIntrin::_InterlockedDecrement_nf; + } + return None; +} + +static Optional +translateX86ToMsvcIntrin(unsigned BuiltinID) { + using MSVCIntrin = CodeGenFunction::MSVCIntrin; + switch (BuiltinID) { + default: + break; + case clang::X86::BI_BitScanForward: + case clang::X86::BI_BitScanForward64: + return MSVCIntrin::_BitScanForward; + case clang::X86::BI_BitScanReverse: + case clang::X86::BI_BitScanReverse64: + return MSVCIntrin::_BitScanReverse; + case clang::X86::BI_InterlockedAnd64: + return MSVCIntrin::_InterlockedAnd; + case clang::X86::BI_InterlockedExchange64: + return MSVCIntrin::_InterlockedExchange; + case clang::X86::BI_InterlockedExchangeAdd64: + return MSVCIntrin::_InterlockedExchangeAdd; + case clang::X86::BI_InterlockedExchangeSub64: + return MSVCIntrin::_InterlockedExchangeSub; + case clang::X86::BI_InterlockedOr64: + return MSVCIntrin::_InterlockedOr; + case clang::X86::BI_InterlockedXor64: + return MSVCIntrin::_InterlockedXor; + case clang::X86::BI_InterlockedDecrement64: + return MSVCIntrin::_InterlockedDecrement; + case clang::X86::BI_InterlockedIncrement64: + return MSVCIntrin::_InterlockedIncrement; + } + return None; +} + +// Emit an MSVC intrinsic. Assumes that arguments have *not* been evaluated. Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E) { switch (BuiltinID) { case MSVCIntrin::_BitScanForward: case MSVCIntrin::_BitScanReverse: { + Address IndexAddress(EmitPointerWithAlignment(E->getArg(0))); Value *ArgValue = EmitScalarExpr(E->getArg(1)); llvm::Type *ArgType = ArgValue->getType(); llvm::Type *IndexType = - EmitScalarExpr(E->getArg(0))->getType()->getPointerElementType(); + IndexAddress.getPointer()->getType()->getPointerElementType(); llvm::Type *ResultType = ConvertType(E->getType()); Value *ArgZero = llvm::Constant::getNullValue(ArgType); @@ -1039,7 +1365,6 @@ Result->addIncoming(ResZero, Begin); Builder.SetInsertPoint(NotZero); - Address IndexAddress = EmitPointerWithAlignment(E->getArg(0)); if (BuiltinID == MSVCIntrin::_BitScanForward) { Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); @@ -6847,6 +7172,10 @@ AccessKind); } + // Handle MSVC intrinsics before argument evaluation. + if (Optional MsvcIntId = translateArmToMsvcIntrin(BuiltinID)) + return EmitMSVCBuiltinExpr(*MsvcIntId, E); + // Deal with MVE builtins if (Value *Result = EmitARMMVEBuiltinExpr(BuiltinID, E, ReturnValue, Arch)) return Result; @@ -7007,143 +7336,6 @@ return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0], Ops[3], Ops[4], Ops[5]}); } - case ARM::BI_BitScanForward: - case ARM::BI_BitScanForward64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E); - case ARM::BI_BitScanReverse: - case ARM::BI_BitScanReverse64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E); - - case ARM::BI_InterlockedAnd64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E); - case ARM::BI_InterlockedExchange64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E); - case ARM::BI_InterlockedExchangeAdd64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E); - case ARM::BI_InterlockedExchangeSub64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E); - case ARM::BI_InterlockedOr64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E); - case ARM::BI_InterlockedXor64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E); - case ARM::BI_InterlockedDecrement64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E); - case ARM::BI_InterlockedIncrement64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E); - case ARM::BI_InterlockedExchangeAdd8_acq: - case ARM::BI_InterlockedExchangeAdd16_acq: - case ARM::BI_InterlockedExchangeAdd_acq: - case ARM::BI_InterlockedExchangeAdd64_acq: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_acq, E); - case ARM::BI_InterlockedExchangeAdd8_rel: - case ARM::BI_InterlockedExchangeAdd16_rel: - case ARM::BI_InterlockedExchangeAdd_rel: - case ARM::BI_InterlockedExchangeAdd64_rel: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_rel, E); - case ARM::BI_InterlockedExchangeAdd8_nf: - case ARM::BI_InterlockedExchangeAdd16_nf: - case ARM::BI_InterlockedExchangeAdd_nf: - case ARM::BI_InterlockedExchangeAdd64_nf: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_nf, E); - case ARM::BI_InterlockedExchange8_acq: - case ARM::BI_InterlockedExchange16_acq: - case ARM::BI_InterlockedExchange_acq: - case ARM::BI_InterlockedExchange64_acq: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_acq, E); - case ARM::BI_InterlockedExchange8_rel: - case ARM::BI_InterlockedExchange16_rel: - case ARM::BI_InterlockedExchange_rel: - case ARM::BI_InterlockedExchange64_rel: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_rel, E); - case ARM::BI_InterlockedExchange8_nf: - case ARM::BI_InterlockedExchange16_nf: - case ARM::BI_InterlockedExchange_nf: - case ARM::BI_InterlockedExchange64_nf: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_nf, E); - case ARM::BI_InterlockedCompareExchange8_acq: - case ARM::BI_InterlockedCompareExchange16_acq: - case ARM::BI_InterlockedCompareExchange_acq: - case ARM::BI_InterlockedCompareExchange64_acq: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_acq, E); - case ARM::BI_InterlockedCompareExchange8_rel: - case ARM::BI_InterlockedCompareExchange16_rel: - case ARM::BI_InterlockedCompareExchange_rel: - case ARM::BI_InterlockedCompareExchange64_rel: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_rel, E); - case ARM::BI_InterlockedCompareExchange8_nf: - case ARM::BI_InterlockedCompareExchange16_nf: - case ARM::BI_InterlockedCompareExchange_nf: - case ARM::BI_InterlockedCompareExchange64_nf: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_nf, E); - case ARM::BI_InterlockedOr8_acq: - case ARM::BI_InterlockedOr16_acq: - case ARM::BI_InterlockedOr_acq: - case ARM::BI_InterlockedOr64_acq: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_acq, E); - case ARM::BI_InterlockedOr8_rel: - case ARM::BI_InterlockedOr16_rel: - case ARM::BI_InterlockedOr_rel: - case ARM::BI_InterlockedOr64_rel: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_rel, E); - case ARM::BI_InterlockedOr8_nf: - case ARM::BI_InterlockedOr16_nf: - case ARM::BI_InterlockedOr_nf: - case ARM::BI_InterlockedOr64_nf: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_nf, E); - case ARM::BI_InterlockedXor8_acq: - case ARM::BI_InterlockedXor16_acq: - case ARM::BI_InterlockedXor_acq: - case ARM::BI_InterlockedXor64_acq: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_acq, E); - case ARM::BI_InterlockedXor8_rel: - case ARM::BI_InterlockedXor16_rel: - case ARM::BI_InterlockedXor_rel: - case ARM::BI_InterlockedXor64_rel: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_rel, E); - case ARM::BI_InterlockedXor8_nf: - case ARM::BI_InterlockedXor16_nf: - case ARM::BI_InterlockedXor_nf: - case ARM::BI_InterlockedXor64_nf: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_nf, E); - case ARM::BI_InterlockedAnd8_acq: - case ARM::BI_InterlockedAnd16_acq: - case ARM::BI_InterlockedAnd_acq: - case ARM::BI_InterlockedAnd64_acq: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_acq, E); - case ARM::BI_InterlockedAnd8_rel: - case ARM::BI_InterlockedAnd16_rel: - case ARM::BI_InterlockedAnd_rel: - case ARM::BI_InterlockedAnd64_rel: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_rel, E); - case ARM::BI_InterlockedAnd8_nf: - case ARM::BI_InterlockedAnd16_nf: - case ARM::BI_InterlockedAnd_nf: - case ARM::BI_InterlockedAnd64_nf: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_nf, E); - case ARM::BI_InterlockedIncrement16_acq: - case ARM::BI_InterlockedIncrement_acq: - case ARM::BI_InterlockedIncrement64_acq: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_acq, E); - case ARM::BI_InterlockedIncrement16_rel: - case ARM::BI_InterlockedIncrement_rel: - case ARM::BI_InterlockedIncrement64_rel: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_rel, E); - case ARM::BI_InterlockedIncrement16_nf: - case ARM::BI_InterlockedIncrement_nf: - case ARM::BI_InterlockedIncrement64_nf: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_nf, E); - case ARM::BI_InterlockedDecrement16_acq: - case ARM::BI_InterlockedDecrement_acq: - case ARM::BI_InterlockedDecrement64_acq: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_acq, E); - case ARM::BI_InterlockedDecrement16_rel: - case ARM::BI_InterlockedDecrement_rel: - case ARM::BI_InterlockedDecrement64_rel: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_rel, E); - case ARM::BI_InterlockedDecrement16_nf: - case ARM::BI_InterlockedDecrement_nf: - case ARM::BI_InterlockedDecrement64_nf: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_nf, E); } // Get the last argument, which specifies the vector type. @@ -8949,6 +9141,10 @@ return Builder.CreateCall(F); } + // Handle MSVC intrinsics before argument evaluation. + if (Optional MsvcIntId = translateAarch64ToMsvcIntrin(BuiltinID)) + return EmitMSVCBuiltinExpr(*MsvcIntId, E); + // Find out if any arguments are required to be integer constant // expressions. unsigned ICEArguments = 0; @@ -9650,142 +9846,6 @@ return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), "vgetq_lane"); } - case AArch64::BI_BitScanForward: - case AArch64::BI_BitScanForward64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E); - case AArch64::BI_BitScanReverse: - case AArch64::BI_BitScanReverse64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E); - case AArch64::BI_InterlockedAnd64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E); - case AArch64::BI_InterlockedExchange64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E); - case AArch64::BI_InterlockedExchangeAdd64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E); - case AArch64::BI_InterlockedExchangeSub64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E); - case AArch64::BI_InterlockedOr64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E); - case AArch64::BI_InterlockedXor64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E); - case AArch64::BI_InterlockedDecrement64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E); - case AArch64::BI_InterlockedIncrement64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E); - case AArch64::BI_InterlockedExchangeAdd8_acq: - case AArch64::BI_InterlockedExchangeAdd16_acq: - case AArch64::BI_InterlockedExchangeAdd_acq: - case AArch64::BI_InterlockedExchangeAdd64_acq: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_acq, E); - case AArch64::BI_InterlockedExchangeAdd8_rel: - case AArch64::BI_InterlockedExchangeAdd16_rel: - case AArch64::BI_InterlockedExchangeAdd_rel: - case AArch64::BI_InterlockedExchangeAdd64_rel: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_rel, E); - case AArch64::BI_InterlockedExchangeAdd8_nf: - case AArch64::BI_InterlockedExchangeAdd16_nf: - case AArch64::BI_InterlockedExchangeAdd_nf: - case AArch64::BI_InterlockedExchangeAdd64_nf: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_nf, E); - case AArch64::BI_InterlockedExchange8_acq: - case AArch64::BI_InterlockedExchange16_acq: - case AArch64::BI_InterlockedExchange_acq: - case AArch64::BI_InterlockedExchange64_acq: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_acq, E); - case AArch64::BI_InterlockedExchange8_rel: - case AArch64::BI_InterlockedExchange16_rel: - case AArch64::BI_InterlockedExchange_rel: - case AArch64::BI_InterlockedExchange64_rel: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_rel, E); - case AArch64::BI_InterlockedExchange8_nf: - case AArch64::BI_InterlockedExchange16_nf: - case AArch64::BI_InterlockedExchange_nf: - case AArch64::BI_InterlockedExchange64_nf: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_nf, E); - case AArch64::BI_InterlockedCompareExchange8_acq: - case AArch64::BI_InterlockedCompareExchange16_acq: - case AArch64::BI_InterlockedCompareExchange_acq: - case AArch64::BI_InterlockedCompareExchange64_acq: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_acq, E); - case AArch64::BI_InterlockedCompareExchange8_rel: - case AArch64::BI_InterlockedCompareExchange16_rel: - case AArch64::BI_InterlockedCompareExchange_rel: - case AArch64::BI_InterlockedCompareExchange64_rel: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_rel, E); - case AArch64::BI_InterlockedCompareExchange8_nf: - case AArch64::BI_InterlockedCompareExchange16_nf: - case AArch64::BI_InterlockedCompareExchange_nf: - case AArch64::BI_InterlockedCompareExchange64_nf: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_nf, E); - case AArch64::BI_InterlockedOr8_acq: - case AArch64::BI_InterlockedOr16_acq: - case AArch64::BI_InterlockedOr_acq: - case AArch64::BI_InterlockedOr64_acq: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_acq, E); - case AArch64::BI_InterlockedOr8_rel: - case AArch64::BI_InterlockedOr16_rel: - case AArch64::BI_InterlockedOr_rel: - case AArch64::BI_InterlockedOr64_rel: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_rel, E); - case AArch64::BI_InterlockedOr8_nf: - case AArch64::BI_InterlockedOr16_nf: - case AArch64::BI_InterlockedOr_nf: - case AArch64::BI_InterlockedOr64_nf: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_nf, E); - case AArch64::BI_InterlockedXor8_acq: - case AArch64::BI_InterlockedXor16_acq: - case AArch64::BI_InterlockedXor_acq: - case AArch64::BI_InterlockedXor64_acq: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_acq, E); - case AArch64::BI_InterlockedXor8_rel: - case AArch64::BI_InterlockedXor16_rel: - case AArch64::BI_InterlockedXor_rel: - case AArch64::BI_InterlockedXor64_rel: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_rel, E); - case AArch64::BI_InterlockedXor8_nf: - case AArch64::BI_InterlockedXor16_nf: - case AArch64::BI_InterlockedXor_nf: - case AArch64::BI_InterlockedXor64_nf: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_nf, E); - case AArch64::BI_InterlockedAnd8_acq: - case AArch64::BI_InterlockedAnd16_acq: - case AArch64::BI_InterlockedAnd_acq: - case AArch64::BI_InterlockedAnd64_acq: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_acq, E); - case AArch64::BI_InterlockedAnd8_rel: - case AArch64::BI_InterlockedAnd16_rel: - case AArch64::BI_InterlockedAnd_rel: - case AArch64::BI_InterlockedAnd64_rel: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_rel, E); - case AArch64::BI_InterlockedAnd8_nf: - case AArch64::BI_InterlockedAnd16_nf: - case AArch64::BI_InterlockedAnd_nf: - case AArch64::BI_InterlockedAnd64_nf: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_nf, E); - case AArch64::BI_InterlockedIncrement16_acq: - case AArch64::BI_InterlockedIncrement_acq: - case AArch64::BI_InterlockedIncrement64_acq: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_acq, E); - case AArch64::BI_InterlockedIncrement16_rel: - case AArch64::BI_InterlockedIncrement_rel: - case AArch64::BI_InterlockedIncrement64_rel: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_rel, E); - case AArch64::BI_InterlockedIncrement16_nf: - case AArch64::BI_InterlockedIncrement_nf: - case AArch64::BI_InterlockedIncrement64_nf: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_nf, E); - case AArch64::BI_InterlockedDecrement16_acq: - case AArch64::BI_InterlockedDecrement_acq: - case AArch64::BI_InterlockedDecrement64_acq: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_acq, E); - case AArch64::BI_InterlockedDecrement16_rel: - case AArch64::BI_InterlockedDecrement_rel: - case AArch64::BI_InterlockedDecrement64_rel: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_rel, E); - case AArch64::BI_InterlockedDecrement16_nf: - case AArch64::BI_InterlockedDecrement_nf: - case AArch64::BI_InterlockedDecrement64_nf: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_nf, E); case AArch64::BI_InterlockedAdd: { Value *Arg0 = EmitScalarExpr(E->getArg(0)); @@ -11759,6 +11819,10 @@ if (BuiltinID == X86::BI__builtin_cpu_init) return EmitX86CpuInit(); + // Handle MSVC intrinsics before argument evaluation. + if (Optional MsvcIntId = translateX86ToMsvcIntrin(BuiltinID)) + return EmitMSVCBuiltinExpr(*MsvcIntId, E); + SmallVector Ops; bool IsMaskFCmp = false; @@ -13965,29 +14029,6 @@ return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, llvm::SyncScope::SingleThread); } - case X86::BI_BitScanForward: - case X86::BI_BitScanForward64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E); - case X86::BI_BitScanReverse: - case X86::BI_BitScanReverse64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E); - - case X86::BI_InterlockedAnd64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E); - case X86::BI_InterlockedExchange64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E); - case X86::BI_InterlockedExchangeAdd64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E); - case X86::BI_InterlockedExchangeSub64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E); - case X86::BI_InterlockedOr64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E); - case X86::BI_InterlockedXor64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E); - case X86::BI_InterlockedDecrement64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E); - case X86::BI_InterlockedIncrement64: - return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E); case X86::BI_InterlockedCompareExchange128: { // InterlockedCompareExchange128 doesn't directly refer to 128bit ints, // instead it takes pointers to 64bit ints for Destination and diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -4120,11 +4120,10 @@ llvm::AtomicOrdering &AO, llvm::SyncScope::ID &SSID); -private: enum class MSVCIntrin; - -public: llvm::Value *EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E); + llvm::Value *EmitEvaluatedMSVCBuiltin(MSVCIntrin BuiltinID, const CallExpr *E, + ArrayRef Ops); llvm::Value *EmitBuiltinAvailable(const VersionTuple &Version); diff --git a/clang/test/CodeGen/ms-intrinsics.c b/clang/test/CodeGen/ms-intrinsics.c --- a/clang/test/CodeGen/ms-intrinsics.c +++ b/clang/test/CodeGen/ms-intrinsics.c @@ -147,7 +147,7 @@ #endif unsigned char test_BitScanForward(unsigned long *Index, unsigned long Mask) { - return _BitScanForward(Index, Mask); + return _BitScanForward(++Index, Mask); } // CHECK: define{{.*}}i8 @test_BitScanForward(i32* {{[a-z_ ]*}}%Index, i32 {{[a-z_ ]*}}%Mask){{.*}}{ // CHECK: [[ISNOTZERO:%[a-z0-9._]+]] = icmp eq i32 %Mask, 0 @@ -156,12 +156,13 @@ // CHECK: [[RESULT:%[a-z0-9._]+]] = phi i8 [ 0, %[[ISZERO_LABEL:[a-z0-9._]+]] ], [ 1, %[[ISNOTZERO_LABEL]] ] // CHECK: ret i8 [[RESULT]] // CHECK: [[ISNOTZERO_LABEL]]: +// CHECK: [[IDXGEP:%[a-z0-9._]+]] = getelementptr inbounds i32, i32* %Index, {{i64|i32}} 1 // CHECK: [[INDEX:%[0-9]+]] = tail call i32 @llvm.cttz.i32(i32 %Mask, i1 true) -// CHECK: store i32 [[INDEX]], i32* %Index, align 4 +// CHECK: store i32 [[INDEX]], i32* [[IDXGEP]], align 4 // CHECK: br label %[[END_LABEL]] unsigned char test_BitScanReverse(unsigned long *Index, unsigned long Mask) { - return _BitScanReverse(Index, Mask); + return _BitScanReverse(++Index, Mask); } // CHECK: define{{.*}}i8 @test_BitScanReverse(i32* {{[a-z_ ]*}}%Index, i32 {{[a-z_ ]*}}%Mask){{.*}}{ // CHECK: [[ISNOTZERO:%[0-9]+]] = icmp eq i32 %Mask, 0 @@ -170,9 +171,10 @@ // CHECK: [[RESULT:%[a-z0-9._]+]] = phi i8 [ 0, %[[ISZERO_LABEL:[a-z0-9._]+]] ], [ 1, %[[ISNOTZERO_LABEL]] ] // CHECK: ret i8 [[RESULT]] // CHECK: [[ISNOTZERO_LABEL]]: +// CHECK: [[IDXGEP:%[a-z0-9._]+]] = getelementptr inbounds i32, i32* %Index, {{i64|i32}} 1 // CHECK: [[REVINDEX:%[0-9]+]] = tail call i32 @llvm.ctlz.i32(i32 %Mask, i1 true) // CHECK: [[INDEX:%[0-9]+]] = xor i32 [[REVINDEX]], 31 -// CHECK: store i32 [[INDEX]], i32* %Index, align 4 +// CHECK: store i32 [[INDEX]], i32* [[IDXGEP]], align 4 // CHECK: br label %[[END_LABEL]] #if defined(__x86_64__) || defined(__arm__) || defined(__aarch64__) @@ -459,19 +461,21 @@ #endif short test_InterlockedIncrement16(short volatile *Addend) { - return _InterlockedIncrement16(Addend); + return _InterlockedIncrement16(++Addend); } // CHECK: define{{.*}}i16 @test_InterlockedIncrement16(i16*{{[a-z_ ]*}}%Addend){{.*}}{ -// CHECK: [[TMP:%[0-9]+]] = atomicrmw add i16* %Addend, i16 1 seq_cst +// CHECK: %incdec.ptr = getelementptr inbounds i16, i16* %Addend, {{i64|i32}} 1 +// CHECK: [[TMP:%[0-9]+]] = atomicrmw add i16* %incdec.ptr, i16 1 seq_cst // CHECK: [[RESULT:%[0-9]+]] = add i16 [[TMP]], 1 // CHECK: ret i16 [[RESULT]] // CHECK: } long test_InterlockedIncrement(long volatile *Addend) { - return _InterlockedIncrement(Addend); + return _InterlockedIncrement(++Addend); } // CHECK: define{{.*}}i32 @test_InterlockedIncrement(i32*{{[a-z_ ]*}}%Addend){{.*}}{ -// CHECK: [[TMP:%[0-9]+]] = atomicrmw add i32* %Addend, i32 1 seq_cst +// CHECK: %incdec.ptr = getelementptr inbounds i32, i32* %Addend, {{i64|i32}} 1 +// CHECK: [[TMP:%[0-9]+]] = atomicrmw add i32* %incdec.ptr, i32 1 seq_cst // CHECK: [[RESULT:%[0-9]+]] = add i32 [[TMP]], 1 // CHECK: ret i32 [[RESULT]] // CHECK: }