diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -235,6 +235,19 @@ }]; } +// Code definition that is used for nontemporal metadata creation. +class MemoryOpWithAlignmentAndAttributes : MemoryOpWithAlignmentBase { + code setNonTemporalMetadataCode = [{ + if ($isNonTemporal.hasValue() && $isNonTemporal.getValue()) { + llvm::Module *module = builder.GetInsertBlock()->getModule(); + llvm::MDNode *metadata = llvm::MDNode::get( + inst->getContext(), llvm::ConstantAsMetadata::get( + builder.getInt32(1))); + inst->setMetadata(module->getMDKindID("nontemporal"), metadata); + } + }]; +} + // Memory-related operations. def LLVM_AllocaOp : MemoryOpWithAlignmentBase, @@ -266,50 +279,87 @@ }]; } def LLVM_LoadOp : - MemoryOpWithAlignmentBase, + MemoryOpWithAlignmentAndAttributes, LLVM_OneResultOp<"load">, - Arguments<(ins LLVM_Type:$addr, OptionalAttr:$alignment)> { + Arguments<(ins LLVM_Type:$addr, + OptionalAttr:$isVolatile, + OptionalAttr:$isNonTemporal, + OptionalAttr:$alignment)> { string llvmBuilder = [{ - auto *inst = builder.CreateLoad($addr); - }] # setAlignmentCode # [{ + auto *inst = + $isVolatile.hasValue() + ? builder.CreateLoad($addr, $isVolatile.getValue()) + : builder.CreateLoad($addr); + }] # setAlignmentCode # setNonTemporalMetadataCode # [{ $res = inst; }]; let builders = [OpBuilder< - "OpBuilder &b, OperationState &result, Value addr, unsigned alignment = 0", + "OpBuilder &b, OperationState &result, Value addr, unsigned alignment = 0, " + "bool isVolatile = false, bool isNonTemporal = false", [{ auto type = addr.getType().cast().getPointerElementTy(); - build(b, result, type, addr, alignment); + auto volatileAttr = isVolatile ? b.getBoolAttr(isVolatile) : BoolAttr(); + auto nonTemporalAttr = + isNonTemporal + ? b.getBoolAttr(isNonTemporal) + : BoolAttr(); + auto alignAttr = + alignment == 0 + ? IntegerAttr() + : b.getI64IntegerAttr(alignment); + build(b, result, type, addr, volatileAttr, nonTemporalAttr, alignAttr); }]>, OpBuilder< "OpBuilder &b, OperationState &result, Type t, Value addr, " - "unsigned alignment = 0", + "unsigned alignment = 0, bool isVolatile = false, " + "bool isNonTemporal = false", [{ - if (alignment == 0) - return build(b, result, t, addr, IntegerAttr()); - build(b, result, t, addr, b.getI64IntegerAttr(alignment)); + auto volatileAttr = isVolatile ? b.getBoolAttr(isVolatile) : BoolAttr(); + auto nonTemporalAttr = + isNonTemporal + ? b.getBoolAttr(isNonTemporal) + : BoolAttr(); + auto alignAttr = + alignment == 0 + ? IntegerAttr() + : b.getI64IntegerAttr(alignment); + build(b, result, t, addr, volatileAttr, nonTemporalAttr, alignAttr); }]>]; let parser = [{ return parseLoadOp(parser, result); }]; let printer = [{ printLoadOp(p, *this); }]; let verifier = alignmentVerifierCode; } def LLVM_StoreOp : - MemoryOpWithAlignmentBase, + MemoryOpWithAlignmentAndAttributes, LLVM_ZeroResultOp<"store">, Arguments<(ins LLVM_Type:$value, LLVM_Type:$addr, + OptionalAttr:$isVolatile, + OptionalAttr:$isNonTemporal, OptionalAttr:$alignment)> { string llvmBuilder = [{ - auto *inst = builder.CreateStore($value, $addr); - }] # setAlignmentCode; + auto *inst = + $isVolatile.hasValue() + ? builder.CreateStore($value, $addr, $isVolatile.getValue()) + : builder.CreateStore($value, $addr); + }] # setAlignmentCode # setNonTemporalMetadataCode; let builders = [ OpBuilder< "OpBuilder &b, OperationState &result, Value value, Value addr, " + "bool isVolatile = false, bool isNonTemporal = false, " "unsigned alignment = 0", [{ - if (alignment == 0) - return build(b, result, ArrayRef{}, value, addr, IntegerAttr()); - build(b, result, ArrayRef{}, value, addr, - b.getI64IntegerAttr(alignment)); + auto volatileAttr = isVolatile ? b.getBoolAttr(isVolatile) : BoolAttr(); + auto nonTemporalAttr = + isNonTemporal + ? b.getBoolAttr(isNonTemporal) + : BoolAttr(); + auto alignAttr = + alignment == 0 + ? IntegerAttr() + : b.getI64IntegerAttr(alignment); + build(b, result, ArrayRef{}, value, addr, volatileAttr, + nonTemporalAttr, alignAttr); }] >]; let parser = [{ return parseStoreOp(parser, result); }]; diff --git a/mlir/test/Target/llvmir.mlir b/mlir/test/Target/llvmir.mlir --- a/mlir/test/Target/llvmir.mlir +++ b/mlir/test/Target/llvmir.mlir @@ -1252,3 +1252,32 @@ %0 = llvm.mlir.addressof @address_taken : !llvm<"void()*"> llvm.return %0 : !llvm<"void()*"> } + +// ----- + +llvm.func @volatile_store_and_load() { + %val = llvm.mlir.constant(5 : i32) : !llvm.i32 + %size = llvm.mlir.constant(1 : i64) : !llvm.i64 + %0 = llvm.alloca %size x !llvm.i32 : (!llvm.i64) -> (!llvm<"i32*">) + // CHECK: store volatile i32 5, i32* %{{.*}} + llvm.store %val, %0 {isVolatile = true} : !llvm<"i32*"> + // CHECK: %{{.*}} = load volatile i32, i32* %{{.*}} + %1 = llvm.load %0 {isVolatile = true} : !llvm<"i32*"> + llvm.return +} + +// ----- + +// Check that nontemporal attribute is exported as metadata node. +llvm.func @nontemoral_store_and_load() { + %val = llvm.mlir.constant(5 : i32) : !llvm.i32 + %size = llvm.mlir.constant(1 : i64) : !llvm.i64 + %0 = llvm.alloca %size x !llvm.i32 : (!llvm.i64) -> (!llvm<"i32*">) + // CHECK: !nontemporal ![[NODE:[0-9]+]] + llvm.store %val, %0 {isNonTemporal = true} : !llvm<"i32*"> + // CHECK: !nontemporal ![[NODE]] + %1 = llvm.load %0 {isNonTemporal = true} : !llvm<"i32*"> + llvm.return +} + +// CHECK: ![[NODE]] = !{i32 1}