Page MenuHomePhabricator

Please use GitHub pull requests for new patches. Avoid migrating existing patches. Phabricator shutdown timeline

Differential D143268 Diff 494617 mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp

Changeset View

Standalone View

View Options

mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp

Show All 35 Lines
bool isStaticStrideOrOffset(int64_t strideOrOffset) { bool isStaticStrideOrOffset(int64_t strideOrOffset) {
return !ShapedType::isDynamic(strideOrOffset); return !ShapedType::isDynamic(strideOrOffset);
} }
LLVM::LLVMFuncOp getFreeFn(LLVMTypeConverter *typeConverter, ModuleOp module) { LLVM::LLVMFuncOp getFreeFn(LLVMTypeConverter *typeConverter, ModuleOp module) {
bool useGenericFn = typeConverter->getOptions().useGenericFunctions; bool useGenericFn = typeConverter->getOptions().useGenericFunctions;
if (useGenericFn) if (useGenericFn)
return LLVM::lookupOrCreateGenericFreeFn(module); return LLVM::lookupOrCreateGenericFreeFn(
module, typeConverter->useOpaquePointers());
return LLVM::lookupOrCreateFreeFn(module); return LLVM::lookupOrCreateFreeFn(module, typeConverter->useOpaquePointers());
} }
struct AllocOpLowering : public AllocLikeOpLLVMLowering { struct AllocOpLowering : public AllocLikeOpLLVMLowering {
AllocOpLowering(LLVMTypeConverter &converter) AllocOpLowering(LLVMTypeConverter &converter)
: AllocLikeOpLLVMLowering(memref::AllocOp::getOperationName(), : AllocLikeOpLLVMLowering(memref::AllocOp::getOperationName(),
converter) {} converter) {}
std::tuple<Value, Value> allocateBuffer(ConversionPatternRewriter &rewriter, std::tuple<Value, Value> allocateBuffer(ConversionPatternRewriter &rewriter,
Location loc, Value sizeBytes, Location loc, Value sizeBytes,
Show All 33 Linesstruct AllocaOpLowering : public AllocLikeOpLLVMLowering {
/// alignment is needed post allocation (for eg. in conjunction with malloc). /// alignment is needed post allocation (for eg. in conjunction with malloc).
std::tuple<Value, Value> allocateBuffer(ConversionPatternRewriter &rewriter, std::tuple<Value, Value> allocateBuffer(ConversionPatternRewriter &rewriter,
Location loc, Value sizeBytes, Location loc, Value sizeBytes,
Operation *op) const override { Operation *op) const override {
// With alloca, one gets a pointer to the element type right away. // With alloca, one gets a pointer to the element type right away.
// For stack allocations. // For stack allocations.
auto allocaOp = cast<memref::AllocaOp>(op); auto allocaOp = cast<memref::AllocaOp>(op);
auto elementPtrType = this->getElementPtrType(allocaOp.getType()); auto elementType =
typeConverter->convertType(allocaOp.getType().getElementType());
auto elementPtrType = getTypeConverter()->getPointer(
elementType, allocaOp.getType().getMemorySpaceAsInt());
auto allocatedElementPtr = rewriter.create<LLVM::AllocaOp>( auto allocatedElementPtr = rewriter.create<LLVM::AllocaOp>(
loc, elementPtrType, sizeBytes, allocaOp.getAlignment().value_or(0)); loc, elementPtrType, elementType, sizeBytes,
allocaOp.getAlignment().value_or(0));
return std::make_tuple(allocatedElementPtr, allocatedElementPtr); return std::make_tuple(allocatedElementPtr, allocatedElementPtr);
} }
}; };
/// The base class for lowering realloc op, to support the implementation of /// The base class for lowering realloc op, to support the implementation of
/// realloc via allocation methods that may or may not support alignment. /// realloc via allocation methods that may or may not support alignment.
/// A derived class should provide an implementation of allocateBuffer using /// A derived class should provide an implementation of allocateBuffer using
▲ Show 20 Lines • Show All 92 Lines • ▼ Show 20 Lines LogicalResult matchAndRewrite(memref::ReallocOp op, OpAdaptor adaptor,
// Allocate a new buffer. // Allocate a new buffer.
auto [dstRawPtr, dstAlignedPtr] = auto [dstRawPtr, dstAlignedPtr] =
allocateBuffer(rewriter, loc, dstByteSize, op); allocateBuffer(rewriter, loc, dstByteSize, op);
// Copy the data from the old buffer to the new buffer. // Copy the data from the old buffer to the new buffer.
Value srcAlignedPtr = desc.alignedPtr(rewriter, loc); Value srcAlignedPtr = desc.alignedPtr(rewriter, loc);
Value isVolatile = Value isVolatile =
rewriter.create<LLVM::ConstantOp>(loc, rewriter.getBoolAttr(false)); rewriter.create<LLVM::ConstantOp>(loc, rewriter.getBoolAttr(false));
auto toVoidPtr = [&](Value ptr) -> Value { auto toVoidPtr = [&](Value ptr) -> Value {
if (getTypeConverter()->useOpaquePointers())
return ptr;
return rewriter.create<LLVM::BitcastOp>(loc, getVoidPtrType(), ptr); return rewriter.create<LLVM::BitcastOp>(loc, getVoidPtrType(), ptr);
}; };
rewriter.create<LLVM::MemcpyOp>(loc, toVoidPtr(dstAlignedPtr), rewriter.create<LLVM::MemcpyOp>(loc, toVoidPtr(dstAlignedPtr),
toVoidPtr(srcAlignedPtr), srcByteSize, toVoidPtr(srcAlignedPtr), srcByteSize,
isVolatile); isVolatile);
// Deallocate the old buffer. // Deallocate the old buffer.
LLVM::LLVMFuncOp freeFunc = LLVM::LLVMFuncOp freeFunc =
getFreeFn(getTypeConverter(), op->getParentOfType<ModuleOp>()); getFreeFn(getTypeConverter(), op->getParentOfType<ModuleOp>());
▲ Show 20 Lines • Show All 151 Lines • ▼ Show 20 Linesstruct DeallocOpLowering : public ConvertOpToLLVMPattern<memref::DeallocOp> {
LogicalResult LogicalResult
matchAndRewrite(memref::DeallocOp op, OpAdaptor adaptor, matchAndRewrite(memref::DeallocOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override { ConversionPatternRewriter &rewriter) const override {
// Insert the `free` declaration if it is not already present. // Insert the `free` declaration if it is not already present.
LLVM::LLVMFuncOp freeFunc = LLVM::LLVMFuncOp freeFunc =
getFreeFn(getTypeConverter(), op->getParentOfType<ModuleOp>()); getFreeFn(getTypeConverter(), op->getParentOfType<ModuleOp>());
MemRefDescriptor memref(adaptor.getMemref()); MemRefDescriptor memref(adaptor.getMemref());
Value casted = rewriter.create<LLVM::BitcastOp>( Value allocatedPtr = memref.allocatedPtr(rewriter, op.getLoc());
op.getLoc(), getVoidPtrType(), Value casted = allocatedPtr;
memref.allocatedPtr(rewriter, op.getLoc())); if (!getTypeConverter()->useOpaquePointers())
casted = rewriter.create<LLVM::BitcastOp>(op.getLoc(), getVoidPtrType(),
allocatedPtr);
rewriter.replaceOpWithNewOp<LLVM::CallOp>(op, freeFunc, casted); rewriter.replaceOpWithNewOp<LLVM::CallOp>(op, freeFunc, casted);
return success(); return success();
} }
}; };
// A `dim` is converted to a constant for static sizes and to an access to the // A `dim` is converted to a constant for static sizes and to an access to the
// size stored in the memref descriptor for dynamic sizes. // size stored in the memref descriptor for dynamic sizes.
struct DimOpLowering : public ConvertOpToLLVMPattern<memref::DimOp> { struct DimOpLowering : public ConvertOpToLLVMPattern<memref::DimOp> {
Show All 30 Lines auto scalarMemRefType =
MemRefType::get({}, unrankedMemRefType.getElementType()); MemRefType::get({}, unrankedMemRefType.getElementType());
unsigned addressSpace = unrankedMemRefType.getMemorySpaceAsInt(); unsigned addressSpace = unrankedMemRefType.getMemorySpaceAsInt();
// Extract pointer to the underlying ranked descriptor and bitcast it to a // Extract pointer to the underlying ranked descriptor and bitcast it to a
// memref<element_type> descriptor pointer to minimize the number of GEP // memref<element_type> descriptor pointer to minimize the number of GEP
// operations. // operations.
UnrankedMemRefDescriptor unrankedDesc(adaptor.getSource()); UnrankedMemRefDescriptor unrankedDesc(adaptor.getSource());
Value underlyingRankedDesc = unrankedDesc.memRefDescPtr(rewriter, loc); Value underlyingRankedDesc = unrankedDesc.memRefDescPtr(rewriter, loc);
Value scalarMemRefDescPtr = rewriter.create<LLVM::BitcastOp>(
loc, Type elementType = typeConverter->convertType(scalarMemRefType);
LLVM::LLVMPointerType::get(typeConverter->convertType(scalarMemRefType), Value scalarMemRefDescPtr;
addressSpace), if (getTypeConverter()->useOpaquePointers())
scalarMemRefDescPtr = underlyingRankedDesc;
else
scalarMemRefDescPtr = rewriter.create<LLVM::BitcastOp>(
loc, LLVM::LLVMPointerType::get(elementType, addressSpace),
underlyingRankedDesc); underlyingRankedDesc);
// Get pointer to offset field of memref<element_type> descriptor. // Get pointer to offset field of memref<element_type> descriptor.
Type indexPtrTy = LLVM::LLVMPointerType::get( Type indexPtrTy = getTypeConverter()->getPointer(
getTypeConverter()->getIndexType(), addressSpace); getTypeConverter()->getIndexType(), addressSpace);
Value offsetPtr = rewriter.create<LLVM::GEPOp>( Value offsetPtr = rewriter.create<LLVM::GEPOp>(
loc, indexPtrTy, scalarMemRefDescPtr, ArrayRef<LLVM::GEPArg>{0, 2}); loc, indexPtrTy, elementType, scalarMemRefDescPtr,
ArrayRef<LLVM::GEPArg>{0, 2});
// The size value that we have to extract can be obtained using GEPop with // The size value that we have to extract can be obtained using GEPop with
// `dimOp.index() + 1` index argument. // `dimOp.index() + 1` index argument.
Value idxPlusOne = rewriter.create<LLVM::AddOp>( Value idxPlusOne = rewriter.create<LLVM::AddOp>(
loc, createIndexConstant(rewriter, loc, 1), adaptor.getIndex()); loc, createIndexConstant(rewriter, loc, 1), adaptor.getIndex());
Value sizePtr = Value sizePtr = rewriter.create<LLVM::GEPOp>(
rewriter.create<LLVM::GEPOp>(loc, indexPtrTy, offsetPtr, idxPlusOne); loc, indexPtrTy, getTypeConverter()->getIndexType(), offsetPtr,
return rewriter.create<LLVM::LoadOp>(loc, sizePtr); idxPlusOne);
return rewriter.create<LLVM::LoadOp>(
loc, getTypeConverter()->getIndexType(), sizePtr);
} }
std::optional<int64_t> getConstantDimIndex(memref::DimOp dimOp) const { std::optional<int64_t> getConstantDimIndex(memref::DimOp dimOp) const {
if (auto idx = dimOp.getConstantIndex()) if (auto idx = dimOp.getConstantIndex())
return idx; return idx;
if (auto constantOp = dimOp.getIndex().getDefiningOp<LLVM::ConstantOp>()) if (auto constantOp = dimOp.getIndex().getDefiningOp<LLVM::ConstantOp>())
return constantOp.getValue() return constantOp.getValue()
▲ Show 20 Lines • Show All 93 Lines • ▼ Show 20 Lines matchAndRewrite(memref::GenericAtomicRMWOp atomicOp, OpAdaptor adaptor,
auto opsToMoveStart = atomicOp->getIterator(); auto opsToMoveStart = atomicOp->getIterator();
auto opsToMoveEnd = initBlock->back().getIterator(); auto opsToMoveEnd = initBlock->back().getIterator();
// Compute the loaded value and branch to the loop block. // Compute the loaded value and branch to the loop block.
rewriter.setInsertionPointToEnd(initBlock); rewriter.setInsertionPointToEnd(initBlock);
auto memRefType = atomicOp.getMemref().getType().cast<MemRefType>(); auto memRefType = atomicOp.getMemref().getType().cast<MemRefType>();
auto dataPtr = getStridedElementPtr(loc, memRefType, adaptor.getMemref(), auto dataPtr = getStridedElementPtr(loc, memRefType, adaptor.getMemref(),
adaptor.getIndices(), rewriter); adaptor.getIndices(), rewriter);
Value init = rewriter.create<LLVM::LoadOp>(loc, dataPtr); Value init = rewriter.create<LLVM::LoadOp>(
loc, typeConverter->convertType(memRefType.getElementType()), dataPtr);
rewriter.create<LLVM::BrOp>(loc, init, loopBlock); rewriter.create<LLVM::BrOp>(loc, init, loopBlock);
// Prepare the body of the loop block. // Prepare the body of the loop block.
rewriter.setInsertionPointToStart(loopBlock); rewriter.setInsertionPointToStart(loopBlock);
// Clone the GenericAtomicRMWOp region and extract the result. // Clone the GenericAtomicRMWOp region and extract the result.
auto loopArgument = loopBlock->getArgument(0); auto loopArgument = loopBlock->getArgument(0);
IRMapping mapping; IRMapping mapping;
▲ Show 20 Lines • Show All 124 Lines • ▼ Show 20 Linesstruct GetGlobalMemrefOpLowering : public AllocLikeOpLLVMLowering {
std::tuple<Value, Value> allocateBuffer(ConversionPatternRewriter &rewriter, std::tuple<Value, Value> allocateBuffer(ConversionPatternRewriter &rewriter,
Location loc, Value sizeBytes, Location loc, Value sizeBytes,
Operation *op) const override { Operation *op) const override {
auto getGlobalOp = cast<memref::GetGlobalOp>(op); auto getGlobalOp = cast<memref::GetGlobalOp>(op);
MemRefType type = getGlobalOp.getResult().getType().cast<MemRefType>(); MemRefType type = getGlobalOp.getResult().getType().cast<MemRefType>();
unsigned memSpace = type.getMemorySpaceAsInt(); unsigned memSpace = type.getMemorySpaceAsInt();
Type arrayTy = convertGlobalMemrefTypeToLLVM(type, *getTypeConverter()); Type arrayTy = convertGlobalMemrefTypeToLLVM(type, *getTypeConverter());
auto addressOf = rewriter.create<LLVM::AddressOfOp>( Type resTy = getTypeConverter()->getPointer(arrayTy, memSpace);
loc, LLVM::LLVMPointerType::get(arrayTy, memSpace), auto addressOf =
getGlobalOp.getName()); rewriter.create<LLVM::AddressOfOp>(loc, resTy, getGlobalOp.getName());
// Get the address of the first element in the array by creating a GEP with // Get the address of the first element in the array by creating a GEP with
// the address of the GV as the base, and (rank + 1) number of 0 indices. // the address of the GV as the base, and (rank + 1) number of 0 indices.
Type elementType = typeConverter->convertType(type.getElementType()); Type elementType = typeConverter->convertType(type.getElementType());
Type elementPtrType = LLVM::LLVMPointerType::get(elementType, memSpace); Type elementPtrType = getTypeConverter()->getPointer(elementType, memSpace);
auto gep = rewriter.create<LLVM::GEPOp>( auto gep = rewriter.create<LLVM::GEPOp>(
loc, elementPtrType, addressOf, loc, elementPtrType, arrayTy, addressOf,
SmallVector<LLVM::GEPArg>(type.getRank() + 1, 0)); SmallVector<LLVM::GEPArg>(type.getRank() + 1, 0));
// We do not expect the memref obtained using `memref.get_global` to be // We do not expect the memref obtained using `memref.get_global` to be
// ever deallocated. Set the allocated pointer to be known bad value to // ever deallocated. Set the allocated pointer to be known bad value to
// help debug if that ever happens. // help debug if that ever happens.
auto intPtrType = getIntPtrType(memSpace); auto intPtrType = getIntPtrType(memSpace);
Value deadBeefConst = Value deadBeefConst =
createIndexAttrConstant(rewriter, op->getLoc(), intPtrType, 0xdeadbeef); createIndexAttrConstant(rewriter, op->getLoc(), intPtrType, 0xdeadbeef);
Show All 14 Linesstruct LoadOpLowering : public LoadStoreOpLowering<memref::LoadOp> {
LogicalResult LogicalResult
matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor, matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override { ConversionPatternRewriter &rewriter) const override {
auto type = loadOp.getMemRefType(); auto type = loadOp.getMemRefType();
Value dataPtr = Value dataPtr =
getStridedElementPtr(loadOp.getLoc(), type, adaptor.getMemref(), getStridedElementPtr(loadOp.getLoc(), type, adaptor.getMemref(),
adaptor.getIndices(), rewriter); adaptor.getIndices(), rewriter);
rewriter.replaceOpWithNewOp<LLVM::LoadOp>(loadOp, dataPtr); rewriter.replaceOpWithNewOp<LLVM::LoadOp>(
loadOp, typeConverter->convertType(type.getElementType()), dataPtr);
return success(); return success();
} }
}; };
// Store operation is lowered to obtaining a pointer to the indexed element, // Store operation is lowered to obtaining a pointer to the indexed element,
// and storing the given value to it. // and storing the given value to it.
struct StoreOpLowering : public LoadStoreOpLowering<memref::StoreOp> { struct StoreOpLowering : public LoadStoreOpLowering<memref::StoreOp> {
using Base::Base; using Base::Base;
▲ Show 20 Lines • Show All 104 Lines • ▼ Show 20 Lines if (srcType.isa<MemRefType>() && dstType.isa<UnrankedMemRefType>()) {
// Set the rank in the destination from the memref type // Set the rank in the destination from the memref type
// Allocate space on the stack and copy the src memref descriptor // Allocate space on the stack and copy the src memref descriptor
// Set the ptr in the destination to the stack space // Set the ptr in the destination to the stack space
auto srcMemRefType = srcType.cast<MemRefType>(); auto srcMemRefType = srcType.cast<MemRefType>();
int64_t rank = srcMemRefType.getRank(); int64_t rank = srcMemRefType.getRank();
// ptr = AllocaOp sizeof(MemRefDescriptor) // ptr = AllocaOp sizeof(MemRefDescriptor)
auto ptr = getTypeConverter()->promoteOneMemRefDescriptor( auto ptr = getTypeConverter()->promoteOneMemRefDescriptor(
loc, adaptor.getSource(), rewriter); loc, adaptor.getSource(), rewriter);
// voidptr = BitCastOp srcType* to void* // voidptr = BitCastOp srcType* to void*
auto voidPtr = Value voidPtr;
rewriter.create<LLVM::BitcastOp>(loc, getVoidPtrType(), ptr) if (getTypeConverter()->useOpaquePointers())
.getResult(); voidPtr = ptr;
else
voidPtr = rewriter.create<LLVM::BitcastOp>(loc, getVoidPtrType(), ptr);
// rank = ConstantOp srcRank // rank = ConstantOp srcRank
auto rankVal = rewriter.create<LLVM::ConstantOp>( auto rankVal = rewriter.create<LLVM::ConstantOp>(
loc, getIndexType(), rewriter.getIndexAttr(rank)); loc, getIndexType(), rewriter.getIndexAttr(rank));
// undef = UndefOp // undef = UndefOp
UnrankedMemRefDescriptor memRefDesc = UnrankedMemRefDescriptor memRefDesc =
UnrankedMemRefDescriptor::undef(rewriter, loc, targetStructType); UnrankedMemRefDescriptor::undef(rewriter, loc, targetStructType);
// d1 = InsertValueOp undef, rank, 0 // d1 = InsertValueOp undef, rank, 0
memRefDesc.setRank(rewriter, loc, rankVal); memRefDesc.setRank(rewriter, loc, rankVal);
// d2 = InsertValueOp d1, voidptr, 1 // d2 = InsertValueOp d1, voidptr, 1
memRefDesc.setMemRefDescPtr(rewriter, loc, voidPtr); memRefDesc.setMemRefDescPtr(rewriter, loc, voidPtr);
rewriter.replaceOp(memRefCastOp, (Value)memRefDesc); rewriter.replaceOp(memRefCastOp, (Value)memRefDesc);
} else if (srcType.isa<UnrankedMemRefType>() && dstType.isa<MemRefType>()) { } else if (srcType.isa<UnrankedMemRefType>() && dstType.isa<MemRefType>()) {
// Casting from unranked type to ranked. // Casting from unranked type to ranked.
// The operation is assumed to be doing a correct cast. If the destination // The operation is assumed to be doing a correct cast. If the destination
// type mismatches the unranked the type, it is undefined behavior. // type mismatches the unranked the type, it is undefined behavior.
UnrankedMemRefDescriptor memRefDesc(adaptor.getSource()); UnrankedMemRefDescriptor memRefDesc(adaptor.getSource());
// ptr = ExtractValueOp src, 1 // ptr = ExtractValueOp src, 1
auto ptr = memRefDesc.memRefDescPtr(rewriter, loc); auto ptr = memRefDesc.memRefDescPtr(rewriter, loc);
// castPtr = BitCastOp i8* to structTy* // castPtr = BitCastOp i8* to structTy*
auto castPtr = Value castPtr;
rewriter if (getTypeConverter()->useOpaquePointers())
.create<LLVM::BitcastOp>( castPtr = ptr;
loc, LLVM::LLVMPointerType::get(targetStructType), ptr) else
.getResult(); castPtr = rewriter.create<LLVM::BitcastOp>(
loc, LLVM::LLVMPointerType::get(targetStructType), ptr);
// struct = LoadOp castPtr // struct = LoadOp castPtr
auto loadOp = rewriter.create<LLVM::LoadOp>(loc, castPtr); auto loadOp =
rewriter.create<LLVM::LoadOp>(loc, targetStructType, castPtr);
rewriter.replaceOp(memRefCastOp, loadOp.getResult()); rewriter.replaceOp(memRefCastOp, loadOp.getResult());
} else { } else {
llvm_unreachable("Unsupported unranked memref to unranked memref cast"); llvm_unreachable("Unsupported unranked memref to unranked memref cast");
} }
} }
}; };
/// Pattern to lower a `memref.copy` to llvm. /// Pattern to lower a `memref.copy` to llvm.
Show All 21 Lines lowerToMemCopyIntrinsic(memref::CopyOp op, OpAdaptor adaptor,
} }
// Get element size. // Get element size.
auto sizeInBytes = getSizeInBytes(loc, srcType.getElementType(), rewriter); auto sizeInBytes = getSizeInBytes(loc, srcType.getElementType(), rewriter);
// Compute total. // Compute total.
Value totalSize = Value totalSize =
rewriter.create<LLVM::MulOp>(loc, numElements, sizeInBytes); rewriter.create<LLVM::MulOp>(loc, numElements, sizeInBytes);
Type elementType = typeConverter->convertType(srcType.getElementType());
Value srcBasePtr = srcDesc.alignedPtr(rewriter, loc); Value srcBasePtr = srcDesc.alignedPtr(rewriter, loc);
Value srcOffset = srcDesc.offset(rewriter, loc); Value srcOffset = srcDesc.offset(rewriter, loc);
Value srcPtr = rewriter.create<LLVM::GEPOp>(loc, srcBasePtr.getType(), Value srcPtr = rewriter.create<LLVM::GEPOp>(
srcBasePtr, srcOffset); loc, srcBasePtr.getType(), elementType, srcBasePtr, srcOffset);
MemRefDescriptor targetDesc(adaptor.getTarget()); MemRefDescriptor targetDesc(adaptor.getTarget());
Value targetBasePtr = targetDesc.alignedPtr(rewriter, loc); Value targetBasePtr = targetDesc.alignedPtr(rewriter, loc);
Value targetOffset = targetDesc.offset(rewriter, loc); Value targetOffset = targetDesc.offset(rewriter, loc);
Value targetPtr = rewriter.create<LLVM::GEPOp>(loc, targetBasePtr.getType(), Value targetPtr = rewriter.create<LLVM::GEPOp>(
targetBasePtr, targetOffset); loc, targetBasePtr.getType(), elementType, targetBasePtr, targetOffset);
Value isVolatile = Value isVolatile =
rewriter.create<LLVM::ConstantOp>(loc, rewriter.getBoolAttr(false)); rewriter.create<LLVM::ConstantOp>(loc, rewriter.getBoolAttr(false));
rewriter.create<LLVM::MemcpyOp>(loc, targetPtr, srcPtr, totalSize, rewriter.create<LLVM::MemcpyOp>(loc, targetPtr, srcPtr, totalSize,
isVolatile); isVolatile);
rewriter.eraseOp(op); rewriter.eraseOp(op);
return success(); return success();
} }
LogicalResult LogicalResult
lowerToMemCopyFunctionCall(memref::CopyOp op, OpAdaptor adaptor, lowerToMemCopyFunctionCall(memref::CopyOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const { ConversionPatternRewriter &rewriter) const {
auto loc = op.getLoc(); auto loc = op.getLoc();
auto srcType = op.getSource().getType().cast<BaseMemRefType>(); auto srcType = op.getSource().getType().cast<BaseMemRefType>();
auto targetType = op.getTarget().getType().cast<BaseMemRefType>(); auto targetType = op.getTarget().getType().cast<BaseMemRefType>();
// First make sure we have an unranked memref descriptor representation. // First make sure we have an unranked memref descriptor representation.
auto makeUnranked = [&, this](Value ranked, BaseMemRefType type) { auto makeUnranked = [&, this](Value ranked, BaseMemRefType type) {
auto rank = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(), auto rank = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
type.getRank()); type.getRank());
auto *typeConverter = getTypeConverter(); auto *typeConverter = getTypeConverter();
auto ptr = auto ptr =
typeConverter->promoteOneMemRefDescriptor(loc, ranked, rewriter); typeConverter->promoteOneMemRefDescriptor(loc, ranked, rewriter);
auto voidPtr =
rewriter.create<LLVM::BitcastOp>(loc, getVoidPtrType(), ptr) Value voidPtr;
.getResult(); if (getTypeConverter()->useOpaquePointers())
voidPtr = ptr;
else
voidPtr = rewriter.create<LLVM::BitcastOp>(loc, getVoidPtrType(), ptr);
auto unrankedType = auto unrankedType =
UnrankedMemRefType::get(type.getElementType(), type.getMemorySpace()); UnrankedMemRefType::get(type.getElementType(), type.getMemorySpace());
return UnrankedMemRefDescriptor::pack(rewriter, loc, *typeConverter, return UnrankedMemRefDescriptor::pack(rewriter, loc, *typeConverter,
unrankedType, unrankedType,
ValueRange{rank, voidPtr}); ValueRange{rank, voidPtr});
}; };
// Save stack position before promoting descriptors // Save stack position before promoting descriptors
auto stackSaveOp = auto stackSaveOp =
rewriter.create<LLVM::StackSaveOp>(loc, getVoidPtrType()); rewriter.create<LLVM::StackSaveOp>(loc, getVoidPtrType());
Value unrankedSource = srcType.hasRank() Value unrankedSource = srcType.hasRank()
? makeUnranked(adaptor.getSource(), srcType) ? makeUnranked(adaptor.getSource(), srcType)
: adaptor.getSource(); : adaptor.getSource();
Value unrankedTarget = targetType.hasRank() Value unrankedTarget = targetType.hasRank()
? makeUnranked(adaptor.getTarget(), targetType) ? makeUnranked(adaptor.getTarget(), targetType)
: adaptor.getTarget(); : adaptor.getTarget();
// Now promote the unranked descriptors to the stack. // Now promote the unranked descriptors to the stack.
auto one = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(), auto one = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
rewriter.getIndexAttr(1)); rewriter.getIndexAttr(1));
auto promote = [&](Value desc) { auto promote = [&](Value desc) {
auto ptrType = LLVM::LLVMPointerType::get(desc.getType()); Type ptrType = getTypeConverter()->getPointer(desc.getType());
auto allocated = auto allocated =
rewriter.create<LLVM::AllocaOp>(loc, ptrType, ValueRange{one}); rewriter.create<LLVM::AllocaOp>(loc, ptrType, desc.getType(), one);
rewriter.create<LLVM::StoreOp>(loc, desc, allocated); rewriter.create<LLVM::StoreOp>(loc, desc, allocated);
return allocated; return allocated;
}; };
auto sourcePtr = promote(unrankedSource); auto sourcePtr = promote(unrankedSource);
auto targetPtr = promote(unrankedTarget); auto targetPtr = promote(unrankedTarget);
unsigned typeSize = unsigned typeSize =
▲ Show 20 Lines • Show All 74 Lines • ▼ Show 20 Lines if (offset != nullptr)
*offset = desc.offset(rewriter, loc); *offset = desc.offset(rewriter, loc);
return; return;
} }
unsigned memorySpace = unsigned memorySpace =
operandType.cast<UnrankedMemRefType>().getMemorySpaceAsInt(); operandType.cast<UnrankedMemRefType>().getMemorySpaceAsInt();
Type elementType = operandType.cast<UnrankedMemRefType>().getElementType(); Type elementType = operandType.cast<UnrankedMemRefType>().getElementType();
Type llvmElementType = typeConverter.convertType(elementType); Type llvmElementType = typeConverter.convertType(elementType);
Type elementPtrPtrType = LLVM::LLVMPointerType::get( LLVM::LLVMPointerType elementPtrType =
LLVM::LLVMPointerType::get(llvmElementType, memorySpace)); typeConverter.getPointer(llvmElementType, memorySpace);
// Extract pointer to the underlying ranked memref descriptor and cast it to // Extract pointer to the underlying ranked memref descriptor and cast it to
// ElemType**. // ElemType**.
UnrankedMemRefDescriptor unrankedDesc(convertedOperand); UnrankedMemRefDescriptor unrankedDesc(convertedOperand);
Value underlyingDescPtr = unrankedDesc.memRefDescPtr(rewriter, loc); Value underlyingDescPtr = unrankedDesc.memRefDescPtr(rewriter, loc);
*allocatedPtr = UnrankedMemRefDescriptor::allocatedPtr( *allocatedPtr = UnrankedMemRefDescriptor::allocatedPtr(
rewriter, loc, underlyingDescPtr, elementPtrPtrType); rewriter, loc, underlyingDescPtr, elementPtrType);
*alignedPtr = UnrankedMemRefDescriptor::alignedPtr( *alignedPtr = UnrankedMemRefDescriptor::alignedPtr(
rewriter, loc, typeConverter, underlyingDescPtr, elementPtrPtrType); rewriter, loc, typeConverter, underlyingDescPtr, elementPtrType);
if (offset != nullptr) { if (offset != nullptr) {
*offset = UnrankedMemRefDescriptor::offset( *offset = UnrankedMemRefDescriptor::offset(
rewriter, loc, typeConverter, underlyingDescPtr, elementPtrPtrType); rewriter, loc, typeConverter, underlyingDescPtr, elementPtrType);
} }
} }
struct MemRefReinterpretCastOpLowering struct MemRefReinterpretCastOpLowering
: public ConvertOpToLLVMPattern<memref::ReinterpretCastOp> { : public ConvertOpToLLVMPattern<memref::ReinterpretCastOp> {
using ConvertOpToLLVMPattern< using ConvertOpToLLVMPattern<
memref::ReinterpretCastOp>::ConvertOpToLLVMPattern; memref::ReinterpretCastOp>::ConvertOpToLLVMPattern;
▲ Show 20 Lines • Show All 178 Lines • ▼ Show 20 Lines convertSourceMemRefToDescriptor(ConversionPatternRewriter &rewriter,
// inner descriptor is allocated on stack. // inner descriptor is allocated on stack.
auto targetDesc = UnrankedMemRefDescriptor::undef( auto targetDesc = UnrankedMemRefDescriptor::undef(
rewriter, loc, typeConverter->convertType(targetType)); rewriter, loc, typeConverter->convertType(targetType));
targetDesc.setRank(rewriter, loc, resultRank); targetDesc.setRank(rewriter, loc, resultRank);
SmallVector<Value, 4> sizes; SmallVector<Value, 4> sizes;
UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(), UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),
targetDesc, sizes); targetDesc, sizes);
Value underlyingDescPtr = rewriter.create<LLVM::AllocaOp>( Value underlyingDescPtr = rewriter.create<LLVM::AllocaOp>(
loc, getVoidPtrType(), sizes.front(), std::nullopt); loc, getVoidPtrType(), IntegerType::get(getContext(), 8),
sizes.front());
targetDesc.setMemRefDescPtr(rewriter, loc, underlyingDescPtr); targetDesc.setMemRefDescPtr(rewriter, loc, underlyingDescPtr);
// Extract pointers and offset from the source memref. // Extract pointers and offset from the source memref.
Value allocatedPtr, alignedPtr, offset; Value allocatedPtr, alignedPtr, offset;
extractPointersAndOffset(loc, rewriter, *getTypeConverter(), extractPointersAndOffset(loc, rewriter, *getTypeConverter(),
reshapeOp.getSource(), adaptor.getSource(), reshapeOp.getSource(), adaptor.getSource(),
&allocatedPtr, &alignedPtr, &offset); &allocatedPtr, &alignedPtr, &offset);
// Set pointers and offset. // Set pointers and offset.
Type llvmElementType = typeConverter->convertType(elementType); Type llvmElementType = typeConverter->convertType(elementType);
auto elementPtrPtrType = LLVM::LLVMPointerType::get( LLVM::LLVMPointerType elementPtrType =
LLVM::LLVMPointerType::get(llvmElementType, addressSpace)); getTypeConverter()->getPointer(llvmElementType, addressSpace);
UnrankedMemRefDescriptor::setAllocatedPtr(rewriter, loc, underlyingDescPtr, UnrankedMemRefDescriptor::setAllocatedPtr(rewriter, loc, underlyingDescPtr,
elementPtrPtrType, allocatedPtr); elementPtrType, allocatedPtr);
UnrankedMemRefDescriptor::setAlignedPtr(rewriter, loc, *getTypeConverter(), UnrankedMemRefDescriptor::setAlignedPtr(rewriter, loc, *getTypeConverter(),
underlyingDescPtr, underlyingDescPtr, elementPtrType,
elementPtrPtrType, alignedPtr); alignedPtr);
UnrankedMemRefDescriptor::setOffset(rewriter, loc, *getTypeConverter(), UnrankedMemRefDescriptor::setOffset(rewriter, loc, *getTypeConverter(),
underlyingDescPtr, elementPtrPtrType, underlyingDescPtr, elementPtrType,
offset); offset);
// Use the offset pointer as base for further addressing. Copy over the new // Use the offset pointer as base for further addressing. Copy over the new
// shape and compute strides. For this, we create a loop from rank-1 to 0. // shape and compute strides. For this, we create a loop from rank-1 to 0.
Value targetSizesBase = UnrankedMemRefDescriptor::sizeBasePtr( Value targetSizesBase = UnrankedMemRefDescriptor::sizeBasePtr(
rewriter, loc, *getTypeConverter(), underlyingDescPtr, rewriter, loc, *getTypeConverter(), underlyingDescPtr, elementPtrType);
elementPtrPtrType);
Value targetStridesBase = UnrankedMemRefDescriptor::strideBasePtr( Value targetStridesBase = UnrankedMemRefDescriptor::strideBasePtr(
rewriter, loc, *getTypeConverter(), targetSizesBase, resultRank); rewriter, loc, *getTypeConverter(), targetSizesBase, resultRank);
Value shapeOperandPtr = shapeDesc.alignedPtr(rewriter, loc); Value shapeOperandPtr = shapeDesc.alignedPtr(rewriter, loc);
Value oneIndex = createIndexConstant(rewriter, loc, 1); Value oneIndex = createIndexConstant(rewriter, loc, 1);
Value resultRankMinusOne = Value resultRankMinusOne =
rewriter.create<LLVM::SubOp>(loc, resultRank, oneIndex); rewriter.create<LLVM::SubOp>(loc, resultRank, oneIndex);
Block *initBlock = rewriter.getInsertionBlock(); Block *initBlock = rewriter.getInsertionBlock();
Show All 19 Lines Value pred = rewriter.create<LLVM::ICmpOp>(
loc, IntegerType::get(rewriter.getContext(), 1), loc, IntegerType::get(rewriter.getContext(), 1),
LLVM::ICmpPredicate::sge, indexArg, zeroIndex); LLVM::ICmpPredicate::sge, indexArg, zeroIndex);
Block *bodyBlock = Block *bodyBlock =
rewriter.splitBlock(condBlock, rewriter.getInsertionPoint()); rewriter.splitBlock(condBlock, rewriter.getInsertionPoint());
rewriter.setInsertionPointToStart(bodyBlock); rewriter.setInsertionPointToStart(bodyBlock);
// Copy size from shape to descriptor. // Copy size from shape to descriptor.
Type llvmIndexPtrType = LLVM::LLVMPointerType::get(indexType); Type llvmIndexPtrType = getTypeConverter()->getPointer(indexType);
Value sizeLoadGep = rewriter.create<LLVM::GEPOp>(loc, llvmIndexPtrType, Value sizeLoadGep = rewriter.create<LLVM::GEPOp>(
loc, llvmIndexPtrType,
typeConverter->convertType(shapeMemRefType.getElementType()),
shapeOperandPtr, indexArg); shapeOperandPtr, indexArg);
Value size = rewriter.create<LLVM::LoadOp>(loc, sizeLoadGep); Value size = rewriter.create<LLVM::LoadOp>(loc, indexType, sizeLoadGep);
UnrankedMemRefDescriptor::setSize(rewriter, loc, *getTypeConverter(), UnrankedMemRefDescriptor::setSize(rewriter, loc, *getTypeConverter(),
targetSizesBase, indexArg, size); targetSizesBase, indexArg, size);
// Write stride value and compute next one. // Write stride value and compute next one.
UnrankedMemRefDescriptor::setStride(rewriter, loc, *getTypeConverter(), UnrankedMemRefDescriptor::setStride(rewriter, loc, *getTypeConverter(),
targetStridesBase, indexArg, strideArg); targetStridesBase, indexArg, strideArg);
Value nextStride = rewriter.create<LLVM::MulOp>(loc, strideArg, size); Value nextStride = rewriter.create<LLVM::MulOp>(loc, strideArg, size);
▲ Show 20 Lines • Show All 169 Lines • ▼ Show 20 Lines matchAndRewrite(memref::ViewOp viewOp, OpAdaptor adaptor,
// Create the descriptor. // Create the descriptor.
MemRefDescriptor sourceMemRef(adaptor.getSource()); MemRefDescriptor sourceMemRef(adaptor.getSource());
auto targetMemRef = MemRefDescriptor::undef(rewriter, loc, targetDescTy); auto targetMemRef = MemRefDescriptor::undef(rewriter, loc, targetDescTy);
// Field 1: Copy the allocated pointer, used for malloc/free. // Field 1: Copy the allocated pointer, used for malloc/free.
Value allocatedPtr = sourceMemRef.allocatedPtr(rewriter, loc); Value allocatedPtr = sourceMemRef.allocatedPtr(rewriter, loc);
auto srcMemRefType = viewOp.getSource().getType().cast<MemRefType>(); auto srcMemRefType = viewOp.getSource().getType().cast<MemRefType>();
Value bitcastPtr = rewriter.create<LLVM::BitcastOp>( Value bitcastPtr;
if (getTypeConverter()->useOpaquePointers())
bitcastPtr = allocatedPtr;
else
bitcastPtr = rewriter.create<LLVM::BitcastOp>(
loc, loc,
LLVM::LLVMPointerType::get(targetElementTy, LLVM::LLVMPointerType::get(targetElementTy,
srcMemRefType.getMemorySpaceAsInt()), srcMemRefType.getMemorySpaceAsInt()),
allocatedPtr); allocatedPtr);
targetMemRef.setAllocatedPtr(rewriter, loc, bitcastPtr); targetMemRef.setAllocatedPtr(rewriter, loc, bitcastPtr);
// Field 2: Copy the actual aligned pointer to payload. // Field 2: Copy the actual aligned pointer to payload.
Value alignedPtr = sourceMemRef.alignedPtr(rewriter, loc); Value alignedPtr = sourceMemRef.alignedPtr(rewriter, loc);
alignedPtr = rewriter.create<LLVM::GEPOp>( alignedPtr = rewriter.create<LLVM::GEPOp>(
loc, alignedPtr.getType(), alignedPtr, adaptor.getByteShift()); loc, alignedPtr.getType(),
typeConverter->convertType(srcMemRefType.getElementType()), alignedPtr,
adaptor.getByteShift());
if (getTypeConverter()->useOpaquePointers()) {
bitcastPtr = alignedPtr;
} else {
bitcastPtr = rewriter.create<LLVM::BitcastOp>( bitcastPtr = rewriter.create<LLVM::BitcastOp>(
loc, loc,
LLVM::LLVMPointerType::get(targetElementTy, LLVM::LLVMPointerType::get(targetElementTy,
srcMemRefType.getMemorySpaceAsInt()), srcMemRefType.getMemorySpaceAsInt()),
alignedPtr); alignedPtr);
}
targetMemRef.setAlignedPtr(rewriter, loc, bitcastPtr); targetMemRef.setAlignedPtr(rewriter, loc, bitcastPtr);
// Field 3: The offset in the resulting type must be 0. This is because of // Field 3: The offset in the resulting type must be 0. This is because of
// the type change: an offset on srcType* may not be expressible as an // the type change: an offset on srcType* may not be expressible as an
// offset on dstType*. // offset on dstType*.
targetMemRef.setOffset(rewriter, loc, targetMemRef.setOffset(rewriter, loc,
createIndexConstant(rewriter, loc, offset)); createIndexConstant(rewriter, loc, offset));
▲ Show 20 Lines • Show All 195 Lines • ▼ Show 20 Lines void runOnOperation() override {
const auto &dataLayoutAnalysis = getAnalysis<DataLayoutAnalysis>(); const auto &dataLayoutAnalysis = getAnalysis<DataLayoutAnalysis>();
LowerToLLVMOptions options(&getContext(), LowerToLLVMOptions options(&getContext(),
dataLayoutAnalysis.getAtOrAbove(op)); dataLayoutAnalysis.getAtOrAbove(op));
options.allocLowering = options.allocLowering =
(useAlignedAlloc ? LowerToLLVMOptions::AllocLowering::AlignedAlloc (useAlignedAlloc ? LowerToLLVMOptions::AllocLowering::AlignedAlloc
: LowerToLLVMOptions::AllocLowering::Malloc); : LowerToLLVMOptions::AllocLowering::Malloc);
options.useGenericFunctions = useGenericFunctions; options.useGenericFunctions = useGenericFunctions;
options.useOpaquePointers = useOpaquePointers;
if (indexBitwidth != kDeriveIndexBitwidthFromDataLayout) if (indexBitwidth != kDeriveIndexBitwidthFromDataLayout)
options.overrideIndexBitwidth(indexBitwidth); options.overrideIndexBitwidth(indexBitwidth);
LLVMTypeConverter typeConverter(&getContext(), options, LLVMTypeConverter typeConverter(&getContext(), options,
&dataLayoutAnalysis); &dataLayoutAnalysis);
RewritePatternSet patterns(&getContext()); RewritePatternSet patterns(&getContext());
populateFinalizeMemRefToLLVMConversionPatterns(typeConverter, patterns); populateFinalizeMemRefToLLVMConversionPatterns(typeConverter, patterns);
LLVMConversionTarget target(getContext()); LLVMConversionTarget target(getContext());
target.addLegalOp<func::FuncOp>(); target.addLegalOp<func::FuncOp>();
if (failed(applyPartialConversion(op, target, std::move(patterns)))) if (failed(applyPartialConversion(op, target, std::move(patterns))))
signalPassFailure(); signalPassFailure();
} }
}; };
} // namespace } // namespace