diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -3809,6 +3809,90 @@ DeferredReplacements.push_back(std::make_pair(Old, New)); } +namespace { + +/// Specify given \p NewAlign as the alignment of return value attribute. If +/// such attribute already exists, re-set it to the maximal one of two options. +LLVM_NODISCARD llvm::AttributeList +maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx, + const llvm::AttributeList &Attrs, + llvm::Align NewAlign) { + llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne(); + if (CurAlign >= NewAlign) + return Attrs; + llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Ctx, NewAlign); + return Attrs + .removeAttribute(Ctx, llvm::AttributeList::ReturnIndex, + llvm::Attribute::AttrKind::Alignment) + .addAttribute(Ctx, llvm::AttributeList::ReturnIndex, AlignAttr); +} + +template class AbstractAssumeAlignedAttrEmitter { +protected: + CodeGenFunction &CGF; + + /// We do nothing if this is, or becomes, nullptr. + const AlignedAttrTy *AA = nullptr; + + llvm::Value *Alignment = nullptr; // May or may not be a constant. + llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero. + + AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl) + : CGF(CGF_) { + if (!FuncDecl) + return; + AA = FuncDecl->getAttr(); + } + +public: + /// If we can, materialize the alignment as an attribute on return value. + LLVM_NODISCARD llvm::AttributeList + TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) { + if (!AA || OffsetCI || CGF.SanOpts.has(SanitizerKind::Alignment)) + return Attrs; + const auto *AlignmentCI = dyn_cast(Alignment); + if (!AlignmentCI) + return Attrs; + llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute( + CGF.getLLVMContext(), Attrs, + llvm::Align( + AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment))); + AA = nullptr; // We're done. Disallow doing anything else. + return NewAttrs; + } + + /// Emit alignment assumption. + /// This is a general fallback that we take if either there is an offset, + /// or the alignment is variable or we are sanitizing for alignment. + void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) { + if (!AA) + return; + CGF.EmitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc, + AA->getLocation(), Alignment, OffsetCI); + AA = nullptr; // We're done. Disallow doing anything else. + } +}; + +/// Helper data structure to emit `AssumeAlignedAttr`. +class AssumeAlignedAttrEmitter final + : public AbstractAssumeAlignedAttrEmitter { +public: + AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl) + : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) { + if (!AA) + return; + // It is guaranteed that the alignment/offset are constants. + Alignment = cast(CGF.EmitScalarExpr(AA->getAlignment())); + if (Expr *Offset = AA->getOffset()) { + OffsetCI = cast(CGF.EmitScalarExpr(Offset)); + if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset. + OffsetCI = nullptr; + } + } +}; + +} // namespace + RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, @@ -4400,6 +4484,9 @@ Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex, llvm::Attribute::StrictFP); + AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl); + Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs); + // Emit the actual call/invoke instruction. llvm::CallBase *CI; if (!InvokeDest) { @@ -4618,16 +4705,8 @@ // Emit the assume_aligned check on the return value. if (Ret.isScalar() && TargetDecl) { - if (const auto *AA = TargetDecl->getAttr()) { - llvm::Value *OffsetValue = nullptr; - if (const auto *Offset = AA->getOffset()) - OffsetValue = EmitScalarExpr(Offset); + AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret); - llvm::Value *Alignment = EmitScalarExpr(AA->getAlignment()); - llvm::ConstantInt *AlignmentCI = cast(Alignment); - EmitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc, AA->getLocation(), - AlignmentCI, OffsetValue); - } if (const auto *AA = TargetDecl->getAttr()) { llvm::Value *AlignmentVal = CallArgs[AA->getParamIndex().getLLVMIndex()] .getRValue(*this) diff --git a/clang/test/CodeGen/assume-aligned-and-alloc-align-attributes.c b/clang/test/CodeGen/assume-aligned-and-alloc-align-attributes.c --- a/clang/test/CodeGen/assume-aligned-and-alloc-align-attributes.c +++ b/clang/test/CodeGen/assume-aligned-and-alloc-align-attributes.c @@ -5,15 +5,11 @@ // CHECK-LABEL: @t0_immediate0( // CHECK-NEXT: entry: -// CHECK-NEXT: [[CALL:%.*]] = call i8* @my_aligned_alloc(i32 320, i32 16) +// CHECK-NEXT: [[CALL:%.*]] = call align 32 i8* @my_aligned_alloc(i32 320, i32 16) // CHECK-NEXT: [[PTRINT:%.*]] = ptrtoint i8* [[CALL]] to i64 -// CHECK-NEXT: [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31 +// CHECK-NEXT: [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 15 // CHECK-NEXT: [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0 // CHECK-NEXT: call void @llvm.assume(i1 [[MASKCOND]]) -// CHECK-NEXT: [[PTRINT1:%.*]] = ptrtoint i8* [[CALL]] to i64 -// CHECK-NEXT: [[MASKEDPTR2:%.*]] = and i64 [[PTRINT1]], 15 -// CHECK-NEXT: [[MASKCOND3:%.*]] = icmp eq i64 [[MASKEDPTR2]], 0 -// CHECK-NEXT: call void @llvm.assume(i1 [[MASKCOND3]]) // CHECK-NEXT: ret i8* [[CALL]] // void *t0_immediate0() { @@ -22,15 +18,11 @@ // CHECK-LABEL: @t1_immediate1( // CHECK-NEXT: entry: -// CHECK-NEXT: [[CALL:%.*]] = call i8* @my_aligned_alloc(i32 320, i32 32) +// CHECK-NEXT: [[CALL:%.*]] = call align 32 i8* @my_aligned_alloc(i32 320, i32 32) // CHECK-NEXT: [[PTRINT:%.*]] = ptrtoint i8* [[CALL]] to i64 // CHECK-NEXT: [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31 // CHECK-NEXT: [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0 // CHECK-NEXT: call void @llvm.assume(i1 [[MASKCOND]]) -// CHECK-NEXT: [[PTRINT1:%.*]] = ptrtoint i8* [[CALL]] to i64 -// CHECK-NEXT: [[MASKEDPTR2:%.*]] = and i64 [[PTRINT1]], 31 -// CHECK-NEXT: [[MASKCOND3:%.*]] = icmp eq i64 [[MASKEDPTR2]], 0 -// CHECK-NEXT: call void @llvm.assume(i1 [[MASKCOND3]]) // CHECK-NEXT: ret i8* [[CALL]] // void *t1_immediate1() { @@ -39,15 +31,11 @@ // CHECK-LABEL: @t2_immediate2( // CHECK-NEXT: entry: -// CHECK-NEXT: [[CALL:%.*]] = call i8* @my_aligned_alloc(i32 320, i32 64) +// CHECK-NEXT: [[CALL:%.*]] = call align 32 i8* @my_aligned_alloc(i32 320, i32 64) // CHECK-NEXT: [[PTRINT:%.*]] = ptrtoint i8* [[CALL]] to i64 -// CHECK-NEXT: [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31 +// CHECK-NEXT: [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 63 // CHECK-NEXT: [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0 // CHECK-NEXT: call void @llvm.assume(i1 [[MASKCOND]]) -// CHECK-NEXT: [[PTRINT1:%.*]] = ptrtoint i8* [[CALL]] to i64 -// CHECK-NEXT: [[MASKEDPTR2:%.*]] = and i64 [[PTRINT1]], 63 -// CHECK-NEXT: [[MASKCOND3:%.*]] = icmp eq i64 [[MASKEDPTR2]], 0 -// CHECK-NEXT: call void @llvm.assume(i1 [[MASKCOND3]]) // CHECK-NEXT: ret i8* [[CALL]] // void *t2_immediate2() { @@ -59,17 +47,13 @@ // CHECK-NEXT: [[ALIGNMENT_ADDR:%.*]] = alloca i32, align 4 // CHECK-NEXT: store i32 [[ALIGNMENT:%.*]], i32* [[ALIGNMENT_ADDR]], align 4 // CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[ALIGNMENT_ADDR]], align 4 -// CHECK-NEXT: [[CALL:%.*]] = call i8* @my_aligned_alloc(i32 320, i32 [[TMP0]]) +// CHECK-NEXT: [[CALL:%.*]] = call align 32 i8* @my_aligned_alloc(i32 320, i32 [[TMP0]]) +// CHECK-NEXT: [[ALIGNMENTCAST:%.*]] = zext i32 [[TMP0]] to i64 +// CHECK-NEXT: [[MASK:%.*]] = sub i64 [[ALIGNMENTCAST]], 1 // CHECK-NEXT: [[PTRINT:%.*]] = ptrtoint i8* [[CALL]] to i64 -// CHECK-NEXT: [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31 +// CHECK-NEXT: [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], [[MASK]] // CHECK-NEXT: [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0 // CHECK-NEXT: call void @llvm.assume(i1 [[MASKCOND]]) -// CHECK-NEXT: [[ALIGNMENTCAST:%.*]] = zext i32 [[TMP0]] to i64 -// CHECK-NEXT: [[MASK:%.*]] = sub i64 [[ALIGNMENTCAST]], 1 -// CHECK-NEXT: [[PTRINT1:%.*]] = ptrtoint i8* [[CALL]] to i64 -// CHECK-NEXT: [[MASKEDPTR2:%.*]] = and i64 [[PTRINT1]], [[MASK]] -// CHECK-NEXT: [[MASKCOND3:%.*]] = icmp eq i64 [[MASKEDPTR2]], 0 -// CHECK-NEXT: call void @llvm.assume(i1 [[MASKCOND3]]) // CHECK-NEXT: ret i8* [[CALL]] // void *t3_variable(int alignment) { diff --git a/clang/test/CodeGen/builtin-assume-aligned.c b/clang/test/CodeGen/builtin-assume-aligned.c --- a/clang/test/CodeGen/builtin-assume-aligned.c +++ b/clang/test/CodeGen/builtin-assume-aligned.c @@ -44,13 +44,14 @@ int *m1() __attribute__((assume_aligned(64))); -// CHECK-LABEL: @test5 +// CHECK-LABEL: @test5( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[CALL:%.*]] = call align 64 i32* (...) @m1() +// CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[CALL]], align 4 +// CHECK-NEXT: ret i32 [[TMP0]] +// int test5() { return *m1(); -// CHECK: [[PTRINT5:%.+]] = ptrtoint -// CHECK: [[MASKEDPTR5:%.+]] = and i64 [[PTRINT5]], 63 -// CHECK: [[MASKCOND5:%.+]] = icmp eq i64 [[MASKEDPTR5]], 0 -// CHECK: call void @llvm.assume(i1 [[MASKCOND5]]) } int *m2() __attribute__((assume_aligned(64, 12))); diff --git a/clang/test/CodeGen/catch-alignment-assumption-attribute-assume_aligned-on-function.cpp b/clang/test/CodeGen/catch-alignment-assumption-attribute-assume_aligned-on-function.cpp --- a/clang/test/CodeGen/catch-alignment-assumption-attribute-assume_aligned-on-function.cpp +++ b/clang/test/CodeGen/catch-alignment-assumption-attribute-assume_aligned-on-function.cpp @@ -6,7 +6,7 @@ // CHECK-SANITIZE-ANYRECOVER: @[[CHAR:.*]] = {{.*}} c"'char **'\00" } // CHECK-SANITIZE-ANYRECOVER: @[[LINE_100_ALIGNMENT_ASSUMPTION:.*]] = {{.*}}, i32 100, i32 10 }, {{.*}}* @[[CHAR]] } -char **__attribute__((assume_aligned(0x80000000))) passthrough(char **x) { +char **__attribute__((assume_aligned(128))) passthrough(char **x) { // CHECK: define i8** @[[PASSTHROUGH:.*]](i8** %[[X:.*]]) // CHECK-NEXT: entry: // CHECK-NEXT: %[[X_ADDR:.*]] = alloca i8**, align 8 @@ -23,19 +23,20 @@ // CHECK-NEXT: %[[X_ADDR]] = alloca i8**, align 8 // CHECK-NEXT: store i8** %[[X]], i8*** %[[X_ADDR]], align 8 // CHECK-NEXT: %[[X_RELOADED:.*]] = load i8**, i8*** %[[X_ADDR]], align 8 - // CHECK-NEXT: %[[X_RETURNED:.*]] = call i8** @[[PASSTHROUGH]](i8** %[[X_RELOADED]]) - // CHECK-NEXT: %[[PTRINT:.*]] = ptrtoint i8** %[[X_RETURNED]] to i64 - // CHECK-NEXT: %[[MASKEDPTR:.*]] = and i64 %[[PTRINT]], 2147483647 - // CHECK-NEXT: %[[MASKCOND:.*]] = icmp eq i64 %[[MASKEDPTR]], 0 + // CHECK-NOSANITIZE-NEXT: %[[X_RETURNED:.*]] = call align 128 i8** @[[PASSTHROUGH]](i8** %[[X_RELOADED]]) + // CHECK-SANITIZE-NEXT: %[[X_RETURNED:.*]] = call i8** @[[PASSTHROUGH]](i8** %[[X_RELOADED]]) + // CHECK-SANITIZE-NEXT: %[[PTRINT:.*]] = ptrtoint i8** %[[X_RETURNED]] to i64 + // CHECK-SANITIZE-NEXT: %[[MASKEDPTR:.*]] = and i64 %[[PTRINT]], 127 + // CHECK-SANITIZE-NEXT: %[[MASKCOND:.*]] = icmp eq i64 %[[MASKEDPTR]], 0 // CHECK-SANITIZE-NEXT: %[[PTRINT_DUP:.*]] = ptrtoint i8** %[[X_RETURNED]] to i64, !nosanitize // CHECK-SANITIZE-NEXT: br i1 %[[MASKCOND]], label %[[CONT:.*]], label %[[HANDLER_ALIGNMENT_ASSUMPTION:[^,]+]],{{.*}} !nosanitize // CHECK-SANITIZE: [[HANDLER_ALIGNMENT_ASSUMPTION]]: - // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 2147483648, i64 0){{.*}}, !nosanitize - // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 2147483648, i64 0){{.*}}, !nosanitize + // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 128, i64 0){{.*}}, !nosanitize + // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 128, i64 0){{.*}}, !nosanitize // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize // CHECK-SANITIZE: [[CONT]]: - // CHECK-NEXT: call void @llvm.assume(i1 %[[MASKCOND]]) + // CHECK-SANITIZE-NEXT: call void @llvm.assume(i1 %[[MASKCOND]]) // CHECK-NEXT: ret i8** %[[X_RETURNED]] // CHECK-NEXT: } #line 100