Index: include/clang/Basic/Builtins.def =================================================================== --- include/clang/Basic/Builtins.def +++ include/clang/Basic/Builtins.def @@ -1244,6 +1244,10 @@ BUILTIN(__builtin___get_unsafe_stack_start, "v*", "Fn") BUILTIN(__builtin___get_unsafe_stack_ptr, "v*", "Fn") +// Nontemporal loads/stores builtins +BUILTIN(__builtin_nontemporal_store, "v.", "t") +BUILTIN(__builtin_nontemporal_load, "v.", "t") + #undef BUILTIN #undef LIBBUILTIN #undef LANGBUILTIN Index: include/clang/Basic/DiagnosticSemaKinds.td =================================================================== --- include/clang/Basic/DiagnosticSemaKinds.td +++ include/clang/Basic/DiagnosticSemaKinds.td @@ -6196,6 +6196,12 @@ "atomic %select{load|store}0 requires runtime support that is not " "available for this target">; +def err_nontemporal_builtin_must_be_pointer : Error< + "address argument to nontemporal builtin must be a pointer (%0 invalid)">; +def err_nontemporal_builtin_must_be_pointer_intfltptr_or_vector : Error< + "address argument to nontemporal builtin must be a pointer to integer, float, " + "pointer, or a vector of such types (%0 invalid)">; + def err_deleted_function_use : Error<"attempt to use a deleted function">; def err_kern_type_not_void_return : Error< Index: include/clang/Sema/Sema.h =================================================================== --- include/clang/Sema/Sema.h +++ include/clang/Sema/Sema.h @@ -8838,6 +8838,7 @@ bool SemaBuiltinLongjmp(CallExpr *TheCall); bool SemaBuiltinSetjmp(CallExpr *TheCall); ExprResult SemaBuiltinAtomicOverloaded(ExprResult TheCallResult); + ExprResult SemaBuiltinNontemporalOverloaded(ExprResult TheCallResult); ExprResult SemaAtomicOpsOverloaded(ExprResult TheCallResult, AtomicExpr::AtomicOp Op); bool SemaBuiltinConstantArg(CallExpr *TheCall, int ArgNum, Index: lib/CodeGen/CGBuiltin.cpp =================================================================== --- lib/CodeGen/CGBuiltin.cpp +++ lib/CodeGen/CGBuiltin.cpp @@ -111,6 +111,43 @@ return EmitFromInt(CGF, Result, T, ValueType); } +static Value *MakeNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) { + Value *Val = CGF.EmitScalarExpr(E->getArg(0)); + Value *Address = CGF.EmitScalarExpr(E->getArg(1)); + + unsigned Align = CGF.getContext() + .getTypeAlignInChars(E->getArg(0)->getType()) + .getQuantity(); + + llvm::MDNode *Node = + llvm::MDNode::get(CGF.getLLVMContext(), + llvm::ConstantAsMetadata::get(CGF.Builder.getInt32(1))); + + // Convert the type of the pointer to a pointer to the stored type. + Val = CGF.EmitToMemory(Val, E->getArg(0)->getType()); + Value *BC = CGF.Builder.CreateBitCast( + Address, llvm::PointerType::getUnqual(Val->getType()), "cast"); + StoreInst *SI = CGF.Builder.CreateStore(Val, BC); + SI->setMetadata(CGF.CGM.getModule().getMDKindID("nontemporal"), Node); + SI->setAlignment(Align); + return nullptr; +} + +static Value *MakeNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) { + QualType T = E->getType(); + Value *Address = CGF.EmitScalarExpr(E->getArg(0)); + unsigned Align = CGF.getContext().getTypeAlignInChars(T).getQuantity(); + + llvm::MDNode *Node = + llvm::MDNode::get(CGF.getLLVMContext(), + llvm::ConstantAsMetadata::get(CGF.Builder.getInt32(1))); + + LoadInst *LI = CGF.Builder.CreateLoad(Address, "ntload"); + LI->setMetadata(CGF.CGM.getModule().getMDKindID("nontemporal"), Node); + LI->setAlignment(Align); + return LI; +} + static RValue EmitBinaryAtomic(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E) { @@ -1153,6 +1190,10 @@ return RValue::get(nullptr); } + case Builtin::BI__builtin_nontemporal_load: + return RValue::get(MakeNontemporalLoad(*this, E)); + case Builtin::BI__builtin_nontemporal_store: + return RValue::get(MakeNontemporalStore(*this, E)); case Builtin::BI__c11_atomic_is_lock_free: case Builtin::BI__atomic_is_lock_free: { // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the Index: lib/Sema/SemaChecking.cpp =================================================================== --- lib/Sema/SemaChecking.cpp +++ lib/Sema/SemaChecking.cpp @@ -441,6 +441,9 @@ case Builtin::BI__sync_swap_8: case Builtin::BI__sync_swap_16: return SemaBuiltinAtomicOverloaded(TheCallResult); + case Builtin::BI__builtin_nontemporal_load: + case Builtin::BI__builtin_nontemporal_store: + return SemaBuiltinNontemporalOverloaded(TheCallResult); #define BUILTIN(ID, TYPE, ATTRS) #define ATOMIC_BUILTIN(ID, TYPE, ATTRS) \ case Builtin::BI##ID: \ @@ -2210,6 +2213,80 @@ return TheCallResult; } +/// SemaBuiltinNontemporalOverloaded - We have a call to +/// __builtin_nontemporal_store or __builtin_nontemporal_load, which is an +/// overloaded function based on the pointer type of its last argument. +/// The main ActOnCallExpr routines have already promoted the types of +/// arguments because all of these calls are prototyped as void(...). +/// +/// This function goes through and does final semantic checking for these +/// builtins. +ExprResult Sema::SemaBuiltinNontemporalOverloaded(ExprResult TheCallResult) { + CallExpr *TheCall = (CallExpr *)TheCallResult.get(); + DeclRefExpr *DRE = + cast(TheCall->getCallee()->IgnoreParenCasts()); + FunctionDecl *FDecl = cast(DRE->getDecl()); + unsigned BuiltinID = FDecl->getBuiltinID(); + assert((BuiltinID == Builtin::BI__builtin_nontemporal_store || + BuiltinID == Builtin::BI__builtin_nontemporal_load) && + "Unexpected nontemporal load/store builtin!"); + bool isStore = BuiltinID == Builtin::BI__builtin_nontemporal_store; + unsigned numArgs = isStore ? 2 : 1; + + // Ensure that we have the proper number of arguments. + if (checkArgCount(*this, TheCall, numArgs)) + return ExprError(); + + // Inspect the last argument of the nontemporal builtin. This should always + // be a pointer type, from which we imply the type of the memory access. + // Because it is a pointer type, we don't have to worry about any implicit + // casts here. + Expr *PointerArg = TheCall->getArg(numArgs - 1); + ExprResult PointerArgResult = + DefaultFunctionArrayLvalueConversion(PointerArg); + + if (PointerArgResult.isInvalid()) + return ExprError(); + PointerArg = PointerArgResult.get(); + TheCall->setArg(numArgs - 1, PointerArg); + + const PointerType *pointerType = PointerArg->getType()->getAs(); + if (!pointerType) { + Diag(DRE->getLocStart(), diag::err_nontemporal_builtin_must_be_pointer) + << PointerArg->getType() << PointerArg->getSourceRange(); + return ExprError(); + } + + QualType ValType = pointerType->getPointeeType(); + + // Strip any qualifiers off ValType. + ValType = ValType.getUnqualifiedType(); + if (!ValType->isIntegerType() && !ValType->isAnyPointerType() && + !ValType->isBlockPointerType() && !ValType->isFloatingType() && + !ValType->isVectorType()) { + Diag(DRE->getLocStart(), + diag::err_nontemporal_builtin_must_be_pointer_intfltptr_or_vector) + << PointerArg->getType() << PointerArg->getSourceRange(); + return ExprError(); + } + + if (!isStore) { + TheCall->setType(ValType); + return TheCallResult; + } + + ExprResult ValArg = TheCall->getArg(0); + InitializedEntity Entity = InitializedEntity::InitializeParameter( + Context, ValType, /*consume*/ false); + ValArg = PerformCopyInitialization(Entity, SourceLocation(), ValArg); + if (ValArg.isInvalid()) + return ExprError(); + + TheCall->setArg(0, ValArg.get()); + TheCall->setType(Context.VoidTy); + return TheCallResult; +} + /// CheckObjCString - Checks that the argument to the builtin /// CFString constructor is correct /// Note: It might also make sense to do the UTF-16 conversion here (would Index: test/CodeGen/Nontemporal.cpp =================================================================== --- /dev/null +++ test/CodeGen/Nontemporal.cpp @@ -0,0 +1,48 @@ +// Test frontend handling of nontemporal builtins. +// RUN: %clang_cc1 -triple x86_64-apple-darwin -emit-llvm %s -o - | FileCheck %s + +signed char sc; +unsigned char uc; +signed short ss; +unsigned short us; +signed int si; +unsigned int ui; +signed long long sll; +unsigned long long ull; +float f1, f2; +double d1, d2; +float __attribute__((vector_size(16))) vf1, vf2; +char __attribute__((vector_size(8))) vc1, vc2; +bool b1, b2; + +void test_all_sizes(void) // CHECK-LABEL: test_all_sizes +{ + __builtin_nontemporal_store(true, &b1); // CHECK: store i8 1, i8* @b1, align 1, !nontemporal + __builtin_nontemporal_store(b1, &b2); // CHECK: store i8{{.*}}, align 1, !nontemporal + __builtin_nontemporal_store(1, &uc); // CHECK: store i8{{.*}}align 1, !nontemporal + __builtin_nontemporal_store(1, &sc); // CHECK: store i8{{.*}}align 1, !nontemporal + __builtin_nontemporal_store(1, &us); // CHECK: store i16{{.*}}align 2, !nontemporal + __builtin_nontemporal_store(1, &ss); // CHECK: store i16{{.*}}align 2, !nontemporal + __builtin_nontemporal_store(1, &ui); // CHECK: store i32{{.*}}align 4, !nontemporal + __builtin_nontemporal_store(1, &si); // CHECK: store i32{{.*}}align 4, !nontemporal + __builtin_nontemporal_store(1, &ull); // CHECK: store i64{{.*}}align 8, !nontemporal + __builtin_nontemporal_store(1, &sll); // CHECK: store i64{{.*}}align 8, !nontemporal + __builtin_nontemporal_store(1.0, &f1); // CHECK: store float{{.*}}align 4, !nontemporal + __builtin_nontemporal_store(1.0, &d1); // CHECK: store double{{.*}}align 8, !nontemporal + __builtin_nontemporal_store(vf1, &vf2); // CHECK: store <4 x float>{{.*}}align 16, !nontemporal + __builtin_nontemporal_store(vc1, &vc2); // CHECK: store <8 x i8>{{.*}}align 8, !nontemporal + + b1 = __builtin_nontemporal_load(&b2); // CHECK: load i8{{.*}}align 1, !nontemporal + uc = __builtin_nontemporal_load(&sc); // CHECK: load i8{{.*}}align 1, !nontemporal + sc = __builtin_nontemporal_load(&uc); // CHECK: load i8{{.*}}align 1, !nontemporal + us = __builtin_nontemporal_load(&ss); // CHECK: load i16{{.*}}align 2, !nontemporal + ss = __builtin_nontemporal_load(&us); // CHECK: load i16{{.*}}align 2, !nontemporal + ui = __builtin_nontemporal_load(&si); // CHECK: load i32{{.*}}align 4, !nontemporal + si = __builtin_nontemporal_load(&ui); // CHECK: load i32{{.*}}align 4, !nontemporal + ull = __builtin_nontemporal_load(&sll); // CHECK: load i64{{.*}}align 8, !nontemporal + sll = __builtin_nontemporal_load(&ull); // CHECK: load i64{{.*}}align 8, !nontemporal + f1 = __builtin_nontemporal_load(&f2); // CHECK: load float{{.*}}align 4, !nontemporal + d1 = __builtin_nontemporal_load(&d2); // CHECK: load double{{.*}}align 8, !nontemporal + vf2 = __builtin_nontemporal_load(&vf1); // CHECK: load <4 x float>{{.*}}align 16, !nontemporal + vc2 = __builtin_nontemporal_load(&vc1); // CHECK: load <8 x i8>{{.*}}align 8, !nontemporal +}