Index: cfe/trunk/docs/LanguageExtensions.rst =================================================================== --- cfe/trunk/docs/LanguageExtensions.rst +++ cfe/trunk/docs/LanguageExtensions.rst @@ -1950,6 +1950,35 @@ These builtins are intended for use in the implementation of ``std::allocator`` and other similar allocation libraries, and are only available in C++. +``__builtin_preserve_access_index`` +----------------------------------- + +``__builtin_preserve_access_index`` specifies a code section where +array subscript access and structure/union member access are relocatable +under bpf compile-once run-everywhere framework. Debuginfo (typically +with ``-g``) is needed, otherwise, the compiler will exit with an error. + +**Syntax**: + +.. code-block:: c + + const void * __builtin_preserve_access_index(const void * ptr) + +**Example of Use**: + +.. code-block:: c + + struct t { + int i; + int j; + union { + int a; + int b; + } c[4]; + }; + struct t *v = ...; + const void *pb =__builtin_preserve_access_index(&v->c[3].b); + Multiprecision Arithmetic Builtins ---------------------------------- Index: cfe/trunk/include/clang/Basic/Builtins.def =================================================================== --- cfe/trunk/include/clang/Basic/Builtins.def +++ cfe/trunk/include/clang/Basic/Builtins.def @@ -1449,6 +1449,7 @@ BUILTIN(__builtin_operator_delete, "vv*", "tn") BUILTIN(__builtin_char_memchr, "c*cC*iz", "n") BUILTIN(__builtin_dump_struct, "ivC*v*", "tn") +BUILTIN(__builtin_preserve_access_index, "vC*vC*", "nU") // Safestack builtins BUILTIN(__builtin___get_unsafe_stack_start, "v*", "Fn") Index: cfe/trunk/lib/CodeGen/CGBuilder.h =================================================================== --- cfe/trunk/lib/CodeGen/CGBuilder.h +++ cfe/trunk/lib/CodeGen/CGBuilder.h @@ -298,6 +298,21 @@ return CreateMemSet(Dest.getPointer(), Value, Size, Dest.getAlignment().getQuantity(), IsVolatile); } + + using CGBuilderBaseTy::CreatePreserveStructAccessIndex; + Address CreatePreserveStructAccessIndex(Address Addr, + unsigned Index, + unsigned FieldIndex, + llvm::MDNode *DbgInfo) { + llvm::StructType *ElTy = cast(Addr.getElementType()); + const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); + const llvm::StructLayout *Layout = DL.getStructLayout(ElTy); + auto Offset = CharUnits::fromQuantity(Layout->getElementOffset(Index)); + + return Address(CreatePreserveStructAccessIndex(Addr.getPointer(), + Index, FieldIndex, DbgInfo), + Addr.getAlignment().alignmentAtOffset(Offset)); + } }; } // end namespace CodeGen Index: cfe/trunk/lib/CodeGen/CGBuiltin.cpp =================================================================== --- cfe/trunk/lib/CodeGen/CGBuiltin.cpp +++ cfe/trunk/lib/CodeGen/CGBuiltin.cpp @@ -1840,6 +1840,27 @@ return RValue::get(Res); } + case Builtin::BI__builtin_preserve_access_index: { + // Only enabled preserved access index region when debuginfo + // is available as debuginfo is needed to preserve user-level + // access pattern. + if (!getDebugInfo()) { + CGM.Error(E->getExprLoc(), "using builtin_preserve_access_index() without -g"); + return RValue::get(EmitScalarExpr(E->getArg(0))); + } + + // Nested builtin_preserve_access_index() not supported + if (IsInPreservedAIRegion) { + CGM.Error(E->getExprLoc(), "nested builtin_preserve_access_index() not supported"); + return RValue::get(EmitScalarExpr(E->getArg(0))); + } + + IsInPreservedAIRegion = true; + Value *Res = EmitScalarExpr(E->getArg(0)); + IsInPreservedAIRegion = false; + return RValue::get(Res); + } + case Builtin::BI__builtin_cimag: case Builtin::BI__builtin_cimagf: case Builtin::BI__builtin_cimagl: Index: cfe/trunk/lib/CodeGen/CGExpr.cpp =================================================================== --- cfe/trunk/lib/CodeGen/CGExpr.cpp +++ cfe/trunk/lib/CodeGen/CGExpr.cpp @@ -25,6 +25,7 @@ #include "clang/AST/Attr.h" #include "clang/AST/DeclObjC.h" #include "clang/AST/NSAPI.h" +#include "clang/Basic/Builtins.h" #include "clang/Basic/CodeGenOptions.h" #include "llvm/ADT/Hashing.h" #include "llvm/ADT/StringExtras.h" @@ -3418,8 +3419,20 @@ CharUnits eltAlign = getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize); - llvm::Value *eltPtr = emitArraySubscriptGEP( - CGF, addr.getPointer(), indices, inbounds, signedIndices, loc, name); + llvm::Value *eltPtr; + auto LastIndex = dyn_cast(indices.back()); + if (!CGF.IsInPreservedAIRegion || !LastIndex) { + eltPtr = emitArraySubscriptGEP( + CGF, addr.getPointer(), indices, inbounds, signedIndices, + loc, name); + } else { + // Remember the original array subscript for bpf target + unsigned idx = LastIndex->getZExtValue(); + eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(addr.getPointer(), + indices.size() - 1, + idx); + } + return Address(eltPtr, eltAlign); } @@ -3908,6 +3921,19 @@ return CGF.Builder.CreateStructGEP(base, idx, field->getName()); } +static Address emitPreserveStructAccess(CodeGenFunction &CGF, Address base, + const FieldDecl *field) { + const RecordDecl *rec = field->getParent(); + llvm::DIType *DbgInfo = CGF.getDebugInfo()->getOrCreateRecordType( + CGF.getContext().getRecordType(rec), rec->getLocation()); + + unsigned idx = + CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field); + + return CGF.Builder.CreatePreserveStructAccessIndex( + base, idx, field->getFieldIndex(), DbgInfo); +} + static bool hasAnyVptr(const QualType Type, const ASTContext &Context) { const auto *RD = Type.getTypePtr()->getAsCXXRecordDecl(); if (!RD) @@ -4015,9 +4041,24 @@ // a barrier every time CXXRecord field with vptr is referenced. addr = Address(Builder.CreateLaunderInvariantGroup(addr.getPointer()), addr.getAlignment()); + + if (IsInPreservedAIRegion) { + // Remember the original union field index + llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType( + getContext().getRecordType(rec), rec->getLocation()); + addr = Address( + Builder.CreatePreserveUnionAccessIndex( + addr.getPointer(), field->getFieldIndex(), DbgInfo), + addr.getAlignment()); + } } else { - // For structs, we GEP to the field that the record layout suggests. - addr = emitAddrOfFieldStorage(*this, addr, field); + + if (!IsInPreservedAIRegion) + // For structs, we GEP to the field that the record layout suggests. + addr = emitAddrOfFieldStorage(*this, addr, field); + else + // Remember the original struct field index + addr = emitPreserveStructAccess(*this, addr, field); // If this is a reference field, load the reference right now. if (FieldType->isReferenceType()) { Index: cfe/trunk/lib/CodeGen/CodeGenFunction.h =================================================================== --- cfe/trunk/lib/CodeGen/CodeGenFunction.h +++ cfe/trunk/lib/CodeGen/CodeGenFunction.h @@ -480,6 +480,10 @@ /// finally block or filter expression. bool IsOutlinedSEHHelper = false; + /// True if CodeGen currently emits code inside presereved access index + /// region. + bool IsInPreservedAIRegion = false; + const CodeGen::CGBlockInfo *BlockInfo = nullptr; llvm::Value *BlockPointer = nullptr; Index: cfe/trunk/lib/Sema/SemaChecking.cpp =================================================================== --- cfe/trunk/lib/Sema/SemaChecking.cpp +++ cfe/trunk/lib/Sema/SemaChecking.cpp @@ -191,6 +191,16 @@ return false; } +/// Check the number of arguments, and set the result type to +/// the argument type. +static bool SemaBuiltinPreserveAI(Sema &S, CallExpr *TheCall) { + if (checkArgCount(S, TheCall, 1)) + return true; + + TheCall->setType(TheCall->getArg(0)->getType()); + return false; +} + static bool SemaBuiltinOverflow(Sema &S, CallExpr *TheCall) { if (checkArgCount(S, TheCall, 3)) return true; @@ -1409,6 +1419,10 @@ TheCall->setType(Context.IntTy); break; } + case Builtin::BI__builtin_preserve_access_index: + if (SemaBuiltinPreserveAI(*this, TheCall)) + return ExprError(); + break; case Builtin::BI__builtin_call_with_static_chain: if (SemaBuiltinCallWithStaticChain(*this, TheCall)) return ExprError(); Index: cfe/trunk/test/CodeGen/bpf-preserve-access-index-2.c =================================================================== --- cfe/trunk/test/CodeGen/bpf-preserve-access-index-2.c +++ cfe/trunk/test/CodeGen/bpf-preserve-access-index-2.c @@ -0,0 +1,22 @@ +// RUN: %clang %s -target bpfeb -x c -emit-llvm -S -g -O2 -o - | FileCheck %s +// RUN: %clang %s -target bpfel -x c -emit-llvm -S -g -O2 -o - | FileCheck %s + +struct t { + int i:1; + int j:2; + union { + int a; + int b; + } c[4]; +}; + +#define _(x) (x) + +const void *test(struct t *arg) { + return _(&arg->c[3].b); +} + +// CHECK-NOT: llvm.preserve.struct.access.index +// CHECK-NOT: llvm.preserve.array.access.index +// CHECK-NOT: llvm.preserve.union.access.index +// CHECK-NOT: __builtin_preserve_access_index Index: cfe/trunk/test/CodeGen/bpf-preserve-access-index.c =================================================================== --- cfe/trunk/test/CodeGen/bpf-preserve-access-index.c +++ cfe/trunk/test/CodeGen/bpf-preserve-access-index.c @@ -0,0 +1,22 @@ +// RUN: %clang %s -target bpfeb -x c -emit-llvm -S -g -O2 -o - | FileCheck --check-prefix=CHECK %s +// RUN: %clang %s -target bpfel -x c -emit-llvm -S -g -O2 -o - | FileCheck --check-prefix=CHECK %s + +struct t { + int i:1; + int j:2; + union { + int a; + int b; + } c[4]; +}; + +#define _(x) (__builtin_preserve_access_index(x)) + +const void *test(struct t *arg) { + return _(&arg->c[3].b); +} + +// CHECK: llvm.preserve.struct.access.index +// CHECK: llvm.preserve.array.access.index +// CHECK: llvm.preserve.union.access.index +// CHECK-NOT: __builtin_preserve_access_index