Index: mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td =================================================================== --- mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td +++ mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td @@ -186,6 +186,28 @@ let assemblyFormat = "$handle attr-dict"; } +// +// Variadic function intrinsics. +// + +def LLVM_VaStartOp : LLVM_ZeroResultIntrOp<"vastart">, + Arguments<(ins LLVM_i8Ptr:$arg_list)> { + let assemblyFormat = "$arg_list attr-dict"; + let summary = "Initializes `arg_list` for subsequent variadic argument extractions."; +} + +def LLVM_VaCopyOp : LLVM_ZeroResultIntrOp<"vacopy">, + Arguments<(ins LLVM_i8Ptr:$dest_list, LLVM_i8Ptr:$src_list)> { + let assemblyFormat = "$src_list `to` $dest_list attr-dict"; + let summary = "Copies the current argument position from `src_list` to `dest_list`."; +} + +def LLVM_VaEndOp : LLVM_ZeroResultIntrOp<"vaend">, + Arguments<(ins LLVM_i8Ptr:$arg_list)> { + let assemblyFormat = "$arg_list attr-dict"; + let summary = "Destroys `arg_list`, which has been initialized by `intr.vastart` or `intr.vacopy`."; +} + // // Exception handling intrinsics. // @@ -548,4 +570,4 @@ def LLVM_VPPtrToIntOp : LLVM_VPCastPtr<"ptrtoint">; def LLVM_VPIntToPtrOp : LLVM_VPCastI<"inttoptr">; -#endif // LLVM_INTRINSIC_OPS \ No newline at end of file +#endif // LLVM_INTRINSIC_OPS Index: mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp =================================================================== --- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -2331,9 +2331,6 @@ return success(); } - if (isVarArg()) - return emitOpError("only external functions can be variadic"); - return success(); } Index: mlir/test/Dialect/LLVMIR/func.mlir =================================================================== --- mlir/test/Dialect/LLVMIR/func.mlir +++ mlir/test/Dialect/LLVMIR/func.mlir @@ -159,6 +159,11 @@ llvm.func weak fastcc @cconv3() { llvm.return } + + // CHECK-LABEL: llvm.func @variadic_def + llvm.func @variadic_def(...) { + llvm.return + } } // ----- @@ -232,15 +237,6 @@ // ----- -module { - // expected-error@+1 {{only external functions can be variadic}} - llvm.func @variadic_def(...) { - llvm.return - } -} - -// ----- - module { // expected-error@+1 {{cannot attach result attributes to functions with a void return}} llvm.func @variadic_def() -> (!llvm.void {llvm.noalias}) Index: mlir/test/Dialect/LLVMIR/roundtrip.mlir =================================================================== --- mlir/test/Dialect/LLVMIR/roundtrip.mlir +++ mlir/test/Dialect/LLVMIR/roundtrip.mlir @@ -479,3 +479,29 @@ llvm.return } } + +// CHECK-LABEL: llvm.func @vararg_func +llvm.func @vararg_func(%arg0: i32, ...) { + // CHECK: %{{.*}} = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %{{.*}} = llvm.mlir.constant(1 : i32) : i32 + %0 = llvm.mlir.constant(1 : i32) : i32 + %1 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA0:.+]] = llvm.alloca %{{.*}} x !llvm.struct<"struct.va_list", (ptr)> {alignment = 8 : i64} : (i32) -> !llvm.ptr)>> + // CHECK: %[[CAST0:.+]] = llvm.bitcast %[[ALLOCA0]] : !llvm.ptr)>> to !llvm.ptr + %2 = llvm.alloca %1 x !llvm.struct<"struct.va_list", (ptr)> {alignment = 8 : i64} : (i32) -> !llvm.ptr)>> + %3 = llvm.bitcast %2 : !llvm.ptr)>> to !llvm.ptr + // CHECK: llvm.intr.vastart %[[CAST0]] + llvm.intr.vastart %3 + // CHECK: %[[ALLOCA1:.+]] = llvm.alloca %{{.*}} x !llvm.ptr {alignment = 8 : i64} : (i32) -> !llvm.ptr> + // CHECK: %[[CAST1:.+]] = llvm.bitcast %[[ALLOCA1]] : !llvm.ptr> to !llvm.ptr + %4 = llvm.alloca %0 x !llvm.ptr {alignment = 8 : i64} : (i32) -> !llvm.ptr> + %5 = llvm.bitcast %4 : !llvm.ptr> to !llvm.ptr + // CHECK: llvm.intr.vacopy %[[CAST0]] to %[[CAST1]] + llvm.intr.vacopy %3 to %5 + // CHECK: llvm.intr.vaend %[[CAST1]] + // CHECK: llvm.intr.vaend %[[CAST0]] + llvm.intr.vaend %5 + llvm.intr.vaend %3 + // CHECK: llvm.return + llvm.return +} Index: mlir/test/Target/LLVMIR/Import/basic.ll =================================================================== --- mlir/test/Target/LLVMIR/Import/basic.ll +++ mlir/test/Target/LLVMIR/Import/basic.ll @@ -629,3 +629,34 @@ ; CHECK: llvm.unreachable unreachable } + +; Varadic function definition +%struct.va_list = type { i8* } + +declare void @llvm.va_start(i8*) +declare void @llvm.va_copy(i8*, i8*) +declare void @llvm.va_end(i8*) + +; CHECK-LABEL: llvm.func @variadic_function +define void @variadic_function(i32 %X, ...) { + ; CHECK: %[[ALLOCA0:.+]] = llvm.alloca %{{.*}} x !llvm.struct<"struct.va_list", (ptr)> {{.*}} : (i32) -> !llvm.ptr)>> + %ap = alloca %struct.va_list + ; CHECK: %[[CAST0:.+]] = llvm.bitcast %[[ALLOCA0]] : !llvm.ptr)>> to !llvm.ptr + %ap2 = bitcast %struct.va_list* %ap to i8* + ; CHECK: llvm.intr.vastart %[[CAST0]] + call void @llvm.va_start(i8* %ap2) + + ; CHECK: %[[ALLOCA1:.+]] = llvm.alloca %{{.*}} x !llvm.ptr {{.*}} : (i32) -> !llvm.ptr> + %aq = alloca i8* + ; CHECK: %[[CAST1:.+]] = llvm.bitcast %[[ALLOCA1]] : !llvm.ptr> to !llvm.ptr + %aq2 = bitcast i8** %aq to i8* + ; CHECK: llvm.intr.vacopy %[[CAST0]] to %[[CAST1]] + call void @llvm.va_copy(i8* %aq2, i8* %ap2) + ; CHECK: llvm.intr.vaend %[[CAST1]] + call void @llvm.va_end(i8* %aq2) + + ; CHECK: llvm.intr.vaend %[[CAST0]] + call void @llvm.va_end(i8* %ap2) + ; CHECK: llvm.return + ret void +} Index: mlir/test/Target/LLVMIR/Import/intrinsic.ll =================================================================== --- mlir/test/Target/LLVMIR/Import/intrinsic.ll +++ mlir/test/Target/LLVMIR/Import/intrinsic.ll @@ -385,6 +385,17 @@ ret void } +; CHECK-LABEL: llvm.func @va_intrinsics_test +define void @va_intrinsics_test(i8* %0, i8* %1) { +; CHECK: llvm.intr.vastart %{{.*}} + call void @llvm.va_start(i8* %0) +; CHECK: llvm.intr.vacopy %{{.*}} to %{{.*}} + call void @llvm.va_copy(i8* %1, i8* %0) +; CHECK: llvm.intr.vaend %{{.*}} + call void @llvm.va_end(i8* %0) + ret void +} + ; CHECK-LABEL: llvm.func @coro_id define void @coro_id(i32 %0, i8* %1) { ; CHECK: llvm.intr.coro.id %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !llvm.token @@ -686,6 +697,9 @@ declare i32 @llvm.eh.typeid.for(i8*) declare i8* @llvm.stacksave() declare void @llvm.stackrestore(i8*) +declare void @llvm.va_start(i8*) +declare void @llvm.va_copy(i8*, i8*) +declare void @llvm.va_end(i8*) declare <8 x i32> @llvm.vp.add.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) declare <8 x i32> @llvm.vp.sub.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) declare <8 x i32> @llvm.vp.mul.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) Index: mlir/test/Target/LLVMIR/llvmir.mlir =================================================================== --- mlir/test/Target/LLVMIR/llvmir.mlir +++ mlir/test/Target/LLVMIR/llvmir.mlir @@ -1909,3 +1909,31 @@ llvm.func @bar(f32) llvm.func @baz() llvm.func @qux(f32) + +// ----- + +// Varaidic function definition + +// CHECK: %struct.va_list = type { ptr } + +// CHECK: define void @vararg_function(i32 %{{.*}}, ...) +llvm.func @vararg_function(%arg0: i32, ...) { + %0 = llvm.mlir.constant(1 : i32) : i32 + %1 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA0:.+]] = alloca %struct.va_list, align 8 + %2 = llvm.alloca %1 x !llvm.struct<"struct.va_list", (ptr)> {alignment = 8 : i64} : (i32) -> !llvm.ptr)>> + %3 = llvm.bitcast %2 : !llvm.ptr)>> to !llvm.ptr + // CHECK: call void @llvm.va_start(ptr %[[ALLOCA0]]) + llvm.intr.vastart %3 + // CHECK: %[[ALLOCA1:.+]] = alloca ptr, align 8 + %4 = llvm.alloca %0 x !llvm.ptr {alignment = 8 : i64} : (i32) -> !llvm.ptr> + %5 = llvm.bitcast %4 : !llvm.ptr> to !llvm.ptr + // CHECK: call void @llvm.va_copy(ptr %[[ALLOCA1]], ptr %[[ALLOCA0]]) + llvm.intr.vacopy %3 to %5 + // CHECK: call void @llvm.va_end(ptr %[[ALLOCA1]]) + // CHECK: call void @llvm.va_end(ptr %[[ALLOCA0]]) + llvm.intr.vaend %5 + llvm.intr.vaend %3 + // CHECK: ret void + llvm.return +} Index: mlir/test/mlir-cpu-runner/X86/lit.local.cfg =================================================================== --- /dev/null +++ mlir/test/mlir-cpu-runner/X86/lit.local.cfg @@ -0,0 +1,6 @@ +if config.root.native_target != 'X86': + config.unsupported = True + +# X86Vector tests must be enabled via build flag. +if config.mlir_run_x86vector_tests: + config.available_features.add('x86vector-tests-enabled') Index: mlir/test/mlir-cpu-runner/X86/math-polynomial-approx-avx2.mlir =================================================================== --- mlir/test/mlir-cpu-runner/X86/math-polynomial-approx-avx2.mlir +++ mlir/test/mlir-cpu-runner/X86/math-polynomial-approx-avx2.mlir @@ -9,6 +9,7 @@ // RUN: -shared-libs=%linalg_test_lib_dir/libmlir_c_runner_utils%shlibext \ // RUN: -shared-libs=%linalg_test_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s +// REQUIRES: x86vector-tests-enabled // -------------------------------------------------------------------------- // // rsqrt. Index: mlir/test/mlir-cpu-runner/X86/vararg.mlir =================================================================== --- /dev/null +++ mlir/test/mlir-cpu-runner/X86/vararg.mlir @@ -0,0 +1,66 @@ +// RUN: mlir-cpu-runner %s -e caller --entry-point-result=i32 | FileCheck %s + +// Check if variadic functions can be called and the correct variadic argument +// can be extracted. +// Note that varaidic argument list (va_list) and the extraction logics are ABI- +// specific. + +llvm.func @caller() -> i32 { + %0 = llvm.mlir.constant(3 : i32) : i32 + %1 = llvm.mlir.constant(2 : i32) : i32 + %2 = llvm.mlir.constant(1 : i32) : i32 + %3 = llvm.call @foo(%2, %1, %0) : (i32, i32, i32) -> i32 + llvm.return %3 : i32 +} + +// Equivalent C code: +// int foo(int X, ...) { +// va_list args; +// va_start(args, X); +// int num = va_arg(args, int); +// va_end(args); +// return num; +//} +llvm.func @foo(%arg0: i32, ...) -> i32 { + %0 = llvm.mlir.constant(8 : i64) : i64 + %1 = llvm.mlir.constant(2 : i32) : i32 + %2 = llvm.mlir.constant(0 : i64) : i64 + %3 = llvm.mlir.constant(0 : i64) : i64 + %4 = llvm.mlir.constant(8 : i32) : i32 + %5 = llvm.mlir.constant(3 : i32) : i32 + %6 = llvm.mlir.constant(0 : i64) : i64 + %7 = llvm.mlir.constant(0 : i64) : i64 + %8 = llvm.mlir.constant(41 : i32) : i32 + %9 = llvm.mlir.constant(0 : i32) : i32 + %10 = llvm.mlir.constant(0 : i64) : i64 + %11 = llvm.mlir.constant(0 : i64) : i64 + %12 = llvm.mlir.constant(1 : i32) : i32 + %13 = llvm.alloca %12 x !llvm.array<1 x struct<"struct.va_list", (i32, i32, ptr, ptr)>> {alignment = 8 : i64} : (i32) -> !llvm.ptr, ptr)>>> + %14 = llvm.bitcast %13 : !llvm.ptr, ptr)>>> to !llvm.ptr + llvm.intr.vastart %14 + %15 = llvm.getelementptr %13[%11, %10, 0] : (!llvm.ptr, ptr)>>>, i64, i64) -> !llvm.ptr + %16 = llvm.load %15 : !llvm.ptr + %17 = llvm.icmp "ult" %16, %8 : i32 + llvm.cond_br %17, ^bb1, ^bb2 +^bb1: // pred: ^bb0 + %18 = llvm.getelementptr %13[%7, %6, 3] : (!llvm.ptr, ptr)>>>, i64, i64) -> !llvm.ptr> + %19 = llvm.load %18 : !llvm.ptr> + %20 = llvm.zext %16 : i32 to i64 + %21 = llvm.getelementptr %19[%20] : (!llvm.ptr, i64) -> !llvm.ptr + %22 = llvm.add %16, %4 : i32 + llvm.store %22, %15 : !llvm.ptr + llvm.br ^bb3(%21 : !llvm.ptr) +^bb2: // pred: ^bb0 + %23 = llvm.getelementptr %13[%3, %2, 2] : (!llvm.ptr, ptr)>>>, i64, i64) -> !llvm.ptr> + %24 = llvm.load %23 : !llvm.ptr> + %25 = llvm.getelementptr %24[%0] : (!llvm.ptr, i64) -> !llvm.ptr + llvm.store %25, %23 : !llvm.ptr> + llvm.br ^bb3(%24 : !llvm.ptr) +^bb3(%26: !llvm.ptr): // 2 preds: ^bb1, ^bb2 + %27 = llvm.bitcast %26 : !llvm.ptr to !llvm.ptr + %28 = llvm.load %27 : !llvm.ptr + llvm.intr.vaend %14 + llvm.return %28 : i32 +} + +// CHECK: 2 Index: mlir/test/mlir-cpu-runner/X86Vector/lit.local.cfg =================================================================== --- mlir/test/mlir-cpu-runner/X86Vector/lit.local.cfg +++ /dev/null @@ -1,5 +0,0 @@ -import sys - -# X86Vector tests must be enabled via build flag. -if not config.mlir_run_x86vector_tests: - config.unsupported = True