diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
--- a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
+++ b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
@@ -1670,11 +1670,12 @@
     Example:
 
     ```mlir
-    %dmat, %token = gpu.create_dn_mat async [%dep] %mem, %size : memref<?xf64>
+    %dmat, %token = gpu.create_dn_mat async [%dep] %handle, %rows, %cols, %mem : memref<?xf64>
     ```
   }];
 
   let arguments = (ins Variadic<GPU_AsyncToken>:$asyncDependencies,
+                       GPU_SparseEnvHandle:$env,
                        Index:$rows,
                        Index:$cols,
                        AnyMemRef:$memref);
@@ -1682,7 +1683,7 @@
 
   let assemblyFormat = [{
     custom<AsyncDependencies>(type($asyncToken), $asyncDependencies)
-    $rows `,` $cols `,` $memref attr-dict `:` type($memref)
+    $env `,` $rows `,` $cols `,` $memref attr-dict `:` type($memref)
   }];
 }
 
@@ -1789,6 +1790,41 @@
   }];
 }
 
+
+def GPU_Create2To4SpMatOp : GPU_Op<"create_2to4_spmat", [GPU_AsyncOpInterface]> {
+  let summary = "Create sparse matrix with 2:4 sparsity operation";
+  let description = [{
+    The `gpu.create_2to4_spmat` operation initializes a sparse matrix in dense 
+    format with 2:4 sparsity.
+    The buffers must already be copied from the host to the device prior to
+    using this operation. The operation returns a handle to the sparse
+    matrix descriptor.
+
+    If the `async` keyword is present, the op is executed asynchronously (i.e.
+    it does not block until the execution has finished on the device). In
+    that case, it returns a !gpu.async.token in addition to the environment.
+
+    Example:
+
+    ```mlir
+    %spmat, %token = gpu.create_2to4_spmat async [%dep] %env, %rows, %cols, %mem : memref<?xf64>
+    ```
+  }];
+
+  let arguments = (ins Variadic<GPU_AsyncToken>:$asyncDependencies,
+                       GPU_SparseEnvHandle:$env,
+                       Index:$rows,
+                       Index:$cols,
+                       AnyMemRef:$memref);
+  let results = (outs Res<GPU_SparseSpMatHandle>:$spMat, 
+                      Optional<GPU_AsyncToken>:$asyncToken);
+
+  let assemblyFormat = [{
+    custom<AsyncDependencies>(type($asyncToken), $asyncDependencies)
+    $env `,` $rows `,` $cols `,` $memref attr-dict `:` type($memref)
+  }];
+}
+
 def GPU_DestroySpMatOp : GPU_Op<"destroy_sp_mat", [GPU_AsyncOpInterface]> {
   let summary = "Destroy sparse matrix operation";
   let description = [{
@@ -1946,7 +1982,8 @@
   }];
 }
 
-def GPU_SpMMBufferSizeOp : GPU_Op<"spmm_buffer_size", [GPU_AsyncOpInterface]> {
+def GPU_SpMMBufferSizeOp : GPU_Op<"spmm_buffer_size", [GPU_AsyncOpInterface, 
+                                                       AttrSizedResultSegments]> {
   let summary = "Precompute buffersize for SpMM operation";
   let description = [{
     The `gpu.spmm_buffer_size` operation returns the buffer size required
@@ -1965,7 +2002,7 @@
     Example:
 
     ```mlir
-    %buffersz, %token = gpu.spmm_buffer_size async [%dep] %env, %spmatA{TRANSPOSE}, %dnmatB{TRANSPOSE}, %dnmatC into f32
+    %buffersz, %token = gpu.spmm_buffer_size async [%dep] %env, %spmatA{TRANSPOSE}, %dnmatB{TRANSPOSE}, %dnmatC : i64 into f32
     ```
   }];
 
@@ -1977,11 +2014,11 @@
                        GPU_SparseDnMatHandle:$dnmatB,
                        GPU_SparseDnMatHandle:$dnmatC,
                        TypeAttr:$computeType);
-  let results = (outs Res<Index>:$bufferSz, 
+  let results = (outs Variadic<Index>:$bufferSzs, 
                       Optional<GPU_AsyncToken>:$asyncToken);
 
   let builders = [OpBuilder<(ins
-      "Type":$bufferSz,
+      "Type":$bufferSzs,
       "Type":$asyncToken,
       "ValueRange":$asyncDependencies,
       "Value":$env,
@@ -1991,17 +2028,17 @@
       "Type":$computeType), [{
     auto modeA = gpu::TransposeMode::NON_TRANSPOSE;
     auto modeB = gpu::TransposeMode::NON_TRANSPOSE;
-    return build($_builder, $_state, bufferSz, asyncToken, asyncDependencies, 
+    return build($_builder, $_state, bufferSzs, asyncToken, asyncDependencies, 
                  env, modeA, modeB, spmatA, dnmatB, dnmatC, computeType);}]>
   ];
 
   let assemblyFormat = [{
     custom<AsyncDependencies>(type($asyncToken), $asyncDependencies)
-    $env `,` $spmatA (`{` $modeA^ `}`)? `,` $dnmatB (`{` $modeB^ `}`)? `,` $dnmatC attr-dict `into` $computeType
+    $env `,` $spmatA (`{` $modeA^ `}`)? `,` $dnmatB (`{` $modeB^ `}`)? `,` $dnmatC attr-dict `:` type($bufferSzs) `into` $computeType
   }];
 }
 
-def GPU_SpMMOp : GPU_Op<"spmm", [GPU_AsyncOpInterface]> {
+def GPU_SpMMOp : GPU_Op<"spmm", [GPU_AsyncOpInterface, AttrSizedOperandSegments]> {
   let summary = "SpMM operation";
   let description = [{
     The `gpu.spmm` operation performs the SpMM operation on the given sparse and
@@ -2032,7 +2069,7 @@
                        GPU_SparseDnMatHandle:$dnmatB,
                        GPU_SparseDnMatHandle:$dnmatC,
                        TypeAttr:$computeType,
-                       AnyMemRef:$buffer);
+                       Variadic<AnyMemRef>:$buffers);
   let results = (outs Optional<GPU_AsyncToken>:$asyncToken);
 
   let builders = [OpBuilder<(ins
@@ -2043,16 +2080,16 @@
       "Value":$dnmatB,
       "Value":$dnmatC,
       "Type":$computeType,
-      "Value":$buffer), [{
+      "ValueRange":$buffers), [{
     auto modeA = gpu::TransposeMode::NON_TRANSPOSE;
     auto modeB = gpu::TransposeMode::NON_TRANSPOSE;
     return build($_builder, $_state, asyncToken, asyncDependencies, env, modeA, 
-                 modeB, spmatA, dnmatB, dnmatC, computeType, buffer);}]>
+                 modeB, spmatA, dnmatB, dnmatC, computeType, buffers);}]>
   ];
 
   let assemblyFormat = [{
     custom<AsyncDependencies>(type($asyncToken), $asyncDependencies)
-    $env `,` $spmatA (`{` $modeA^ `}`)? `,` $dnmatB (`{` $modeB^ `}`)? `,` $dnmatC `,` $buffer attr-dict `:` type($buffer) `into` $computeType
+    $env `,` $spmatA (`{` $modeA^ `}`)? `,` $dnmatB (`{` $modeB^ `}`)? `,` $dnmatC `,` $buffers attr-dict `:` type($buffers) `into` $computeType
   }];
 }
 
diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
--- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
@@ -230,6 +230,42 @@
       {llvmIntPtrType, llvmIntPtrType, llvmIntPtrType, llvmPointerType,
        llvmPointerType, llvmPointerType, llvmInt32Type, llvmInt32Type,
        llvmInt32Type, llvmPointerType /* void *stream */}};
+  FunctionCallBuilder createSparseLtEnvCallBuilder = {
+      "mgpuCreateSparseLtEnv",
+      llvmPointerType,
+      {llvmPointerType /* void *stream */}};
+  FunctionCallBuilder destroySparseLtEnvCallBuilder = {
+      "mgpuDestroySparseLtEnv",
+      llvmVoidType,
+      {llvmPointerType, llvmPointerType /* void *stream */}};
+  FunctionCallBuilder createLtDnMatCallBuilder = {
+      "mgpuCreateLtDnMat",
+      llvmPointerType,
+      {llvmPointerType, llvmIntPtrType, llvmIntPtrType, llvmPointerType,
+       llvmInt32Type, llvmPointerType /* void *stream */}};
+  FunctionCallBuilder destroyCuSparseLtSpMatBuilder = {
+      "mgpuDestroyCuSparseLtSpMat",
+      llvmVoidType,
+      {llvmPointerType, llvmPointerType /* void *stream */}};
+  FunctionCallBuilder destroyCuSparseLtDnMatBuilder = {
+      "mgpuDestroyCuSparseLtDnMat",
+      llvmVoidType,
+      {llvmPointerType, llvmPointerType /* void *stream */}};
+  FunctionCallBuilder create2To4SpMatCallBuilder = {
+      "mgpuCusparseLtCreate2To4SpMat",
+      llvmPointerType,
+      {llvmPointerType, llvmIntPtrType, llvmIntPtrType, llvmPointerType,
+       llvmInt32Type, llvmPointerType /* void *stream */}};
+  FunctionCallBuilder cuSparseLtSpmmBufferSizeBuilder = {
+      "mgpuCuSparseLtSpmmBufferSize",
+      llvmPointerType,
+      {llvmPointerType, llvmPointerType, llvmPointerType /*void *stream*/}};
+  FunctionCallBuilder cuSparseLtSpmmBuilder = {
+      "mgpuCuSparseLtSpmm",
+      llvmVoidType,
+      {llvmPointerType, llvmPointerType, llvmPointerType, llvmPointerType,
+       llvmInt32Type, llvmPointerType, llvmPointerType, llvmPointerType,
+       llvmPointerType /*void *stream*/}};
   FunctionCallBuilder destroySpMatCallBuilder = {
       "mgpuDestroySpMat",
       llvmVoidType,
@@ -559,6 +595,20 @@
                   ConversionPatternRewriter &rewriter) const override;
 };
 
+class ConvertCreate2To4SpMatOpToGpuRuntimeCallPattern
+    : public ConvertOpToGpuRuntimeCallPattern<gpu::Create2To4SpMatOp> {
+public:
+  ConvertCreate2To4SpMatOpToGpuRuntimeCallPattern(
+      LLVMTypeConverter &typeConverter)
+      : ConvertOpToGpuRuntimeCallPattern<gpu::Create2To4SpMatOp>(
+            typeConverter) {}
+
+private:
+  LogicalResult
+  matchAndRewrite(gpu::Create2To4SpMatOp op, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override;
+};
+
 class ConvertDestroySpMatOpToGpuRuntimeCallPattern
     : public ConvertOpToGpuRuntimeCallPattern<gpu::DestroySpMatOp> {
 public:
@@ -697,6 +747,23 @@
   // TODO: add support to CUSPARSE_INDEX_16U: 1
 }
 
+static int32_t getCuSparseLtDataTypeFrom(Type type) {
+  if (type.isF16())
+    return 0; // CUSPARSE_COMPUTE_16F
+  if (type.isF32())
+    return 0; // CUSPARSE_COMPUTE_16F
+  if (type.isF64())
+    return 0; // CUSPARSE_COMPUTE_16F
+  if (type.isInteger(8))
+    return 1; // CUSPARSE_COMPUTE_32I
+  if (type.isInteger(16))
+    return 1; // CUSPARSE_COMPUTE_32I
+  if (type.isInteger(32))
+    return 1; // CUSPARSE_COMPUTE_32I
+
+  llvm_unreachable("unsupported element type");
+}
+
 // Corresponding to cudaDataType_t defined in CUDA library_types.h.
 static int32_t getCuSparseDataTypeFrom(Type type) {
   if (llvm::isa<ComplexType>(type)) {
@@ -735,6 +802,37 @@
   llvm_unreachable("unsupported element type");
 }
 
+// TODO:  We may want a run-time (of the mlir compiler) disablement/warning:
+// cusparseLt currently won't work for cuda architecture <8.0 and will trigger
+// a runtime (of the CUDA program) error , but it might be great if we could
+// at least output a warning when we found the target architecture is <8.0 and
+// the user still wants to use cusparseLt. to make sure when lowering gpu
+// sparse dialect to llvm calls, the cusparselt calls are disabled for cuda
+// architecture <8.0
+static bool is2To4Sparsity(Value spMat) {
+  if (auto op = spMat.getDefiningOp<gpu::Create2To4SpMatOp>())
+    return true;
+  if (auto op = spMat.getDefiningOp<gpu::CreateCooOp>())
+    return false;
+  if (auto op = spMat.getDefiningOp<gpu::CreateCsrOp>())
+    return false;
+  // print the spMat defining op
+  spMat.getDefiningOp()->print(llvm::errs());
+  llvm_unreachable("cannot find spmat def");
+}
+
+static std::string inferSpMMType(Value op) {
+  for (Operation *user : op.getUsers()) {
+    auto spmmOp = dyn_cast<gpu::SpMMOp>(user);
+    // if the other operator is 50% sparsity then we should use cusparseLt
+    if (!spmmOp)
+      continue;
+    if (is2To4Sparsity(spmmOp.getSpmatA()))
+      return "cusparseLt";
+  }
+  return "cusparse";
+}
+
 // Returns whether all operands are of LLVM type.
 static LogicalResult areAllLLVMTypes(Operation *op, ValueRange operands,
                                      ConversionPatternRewriter &rewriter) {
@@ -912,10 +1010,10 @@
   return false;
 }
 
-// Converts `gpu.wait` to runtime calls. The converted op synchronizes the host
-// with the stream/event operands. The operands are destroyed. That is, it
-// assumes that it is not used afterwards or elsewhere. Otherwise we will get a
-// runtime error. Eventually, we should guarantee this property.
+// Converts `gpu.wait` to runtime calls. The converted op synchronizes the
+// host with the stream/event operands. The operands are destroyed. That is,
+// it assumes that it is not used afterwards or elsewhere. Otherwise we will
+// get a runtime error. Eventually, we should guarantee this property.
 LogicalResult ConvertWaitOpToGpuRuntimeCallPattern::matchAndRewrite(
     gpu::WaitOp waitOp, OpAdaptor adaptor,
     ConversionPatternRewriter &rewriter) const {
@@ -961,7 +1059,8 @@
     auto operand = std::get<1>(pair);
     if (isDefinedByCallTo(operand, streamCreateCallBuilder.functionName)) {
       // The converted operand's definition created a stream. Insert an event
-      // into the stream just after the last use of the original token operand.
+      // into the stream just after the last use of the original token
+      // operand.
       auto *defOp = std::get<0>(pair).getDefiningOp();
       rewriter.setInsertionPointAfter(defOp);
       auto event = eventCreateCallBuilder.create(loc, rewriter, {}).getResult();
@@ -1032,8 +1131,9 @@
       /*alignment=*/0);
   auto arraySize =
       builder.create<LLVM::ConstantOp>(loc, llvmInt32Type, numArguments);
-  auto arrayPtr = builder.create<LLVM::AllocaOp>(
-      loc, llvmPointerPointerType, llvmPointerType, arraySize, /*alignment=*/0);
+  auto arrayPtr = builder.create<LLVM::AllocaOp>(loc, llvmPointerPointerType,
+                                                 llvmPointerType, arraySize,
+                                                 /*alignment=*/0);
   for (const auto &en : llvm::enumerate(arguments)) {
     Value fieldPtr = builder.create<LLVM::GEPOp>(
         loc, getTypeConverter()->getPointerType(argumentTypes[en.index()]),
@@ -1102,8 +1202,8 @@
         launchOp, "Cannot convert with more than one async dependency.");
 
   // Fail when the synchronous version of the op has async dependencies. The
-  // lowering destroys the stream, and we do not want to check that there is no
-  // use of the stream after this op.
+  // lowering destroys the stream, and we do not want to check that there is
+  // no use of the stream after this op.
   if (!launchOp.getAsyncToken() && !launchOp.getAsyncDependencies().empty())
     return rewriter.notifyMatchFailure(
         launchOp, "Cannot convert non-async op with async dependencies.");
@@ -1160,9 +1260,9 @@
     // Async launch: make dependent ops use the same stream.
     rewriter.replaceOp(launchOp, {stream});
   } else {
-    // Synchronize with host and destroy stream. This must be the stream created
-    // above (with no other uses) because we check that the synchronous version
-    // does not have any async dependencies.
+    // Synchronize with host and destroy stream. This must be the stream
+    // created above (with no other uses) because we check that the
+    // synchronous version does not have any async dependencies.
     streamSynchronizeCallBuilder.create(loc, rewriter, stream);
     streamDestroyCallBuilder.create(loc, rewriter, stream);
     rewriter.eraseOp(launchOp);
@@ -1288,6 +1388,13 @@
   return computeTypeConst;
 }
 
+static Value genConstInt32ForLtFromComputeMode(OpBuilder &builder, Location loc,
+                                          Type computeType) {
+  auto computeTypeInt = getCuSparseLtDataTypeFrom(computeType);
+  auto computeTypeConst = genConstInt32From(builder, loc, computeTypeInt);
+  return computeTypeConst;
+}
+
 LogicalResult ConvertCreateSparseEnvOpToGpuRuntimeCallPattern::matchAndRewrite(
     gpu::CreateSparseEnvOp op, OpAdaptor adaptor,
     ConversionPatternRewriter &rewriter) const {
@@ -1296,8 +1403,16 @@
     return failure();
   Location loc = op.getLoc();
   auto stream = adaptor.getAsyncDependencies().front();
-  auto handle =
-      createSparseEnvCallBuilder.create(loc, rewriter, {stream}).getResult();
+  // use the cusparseLt create call if the dnmat is used with spmat with
+  // 2:4 sparsity
+  Value handle;
+  if (inferSpMMType(op.getEnv()) == "cusparseLt") {
+    handle = createSparseLtEnvCallBuilder.create(loc, rewriter, {stream})
+                 .getResult();
+  } else {
+    handle =
+        createSparseEnvCallBuilder.create(loc, rewriter, {stream}).getResult();
+  }
   rewriter.replaceOp(op, {handle, stream});
   return success();
 }
@@ -1310,7 +1425,15 @@
     return failure();
   Location loc = op.getLoc();
   auto stream = adaptor.getAsyncDependencies().front();
-  destroySparseEnvCallBuilder.create(loc, rewriter, {adaptor.getEnv(), stream});
+  // use the cusparseLt destroy call if the dnmat is used with spmat with
+  // 2:4 sparsity
+  if (inferSpMMType(op.getEnv()) == "cusparseLt") {
+    destroySparseLtEnvCallBuilder.create(loc, rewriter,
+                                         {adaptor.getEnv(), stream});
+  } else {
+    destroySparseEnvCallBuilder.create(loc, rewriter,
+                                       {adaptor.getEnv(), stream});
+  }
   rewriter.replaceOp(op, {stream});
   return success();
 }
@@ -1362,13 +1485,29 @@
       MemRefDescriptor(adaptor.getMemref()).allocatedPtr(rewriter, loc);
   if (!getTypeConverter()->useOpaquePointers())
     pMat = rewriter.create<LLVM::BitcastOp>(loc, llvmPointerType, pMat);
+  // TODO: For now, we track the use of the handle and lower it to cusparse /
+  // cusparseLt accordingly. If in a block, both cusparse and cusparseLt are
+  // used, we require two separate Creation ops to be the correct logic. In
+  // future, we may add support to using one handle in sparse tensor / GPU
+  // dialect in both cusparse and cusparseLt. use the cusparseLt create call
+  // if the dnmat is used with spmat with 2:4 sparsity
+  Value handle;
   Type dType = op.getMemref().getType().getElementType();
   auto dtp = genConstInt32From(rewriter, loc, getCuSparseDataTypeFrom(dType));
-  auto handle =
-      createDnMatCallBuilder
-          .create(loc, rewriter,
-                  {adaptor.getRows(), adaptor.getCols(), pMat, dtp, stream})
-          .getResult();
+  if (inferSpMMType(op.getDmat()) == "cusparseLt") {
+    auto envHandle = adaptor.getEnv();
+    handle = createLtDnMatCallBuilder
+                 .create(loc, rewriter,
+                         {envHandle, adaptor.getRows(), adaptor.getCols(), pMat,
+                          dtp, stream})
+                 .getResult();
+  } else {
+    handle =
+        createDnMatCallBuilder
+            .create(loc, rewriter,
+                    {adaptor.getRows(), adaptor.getCols(), pMat, dtp, stream})
+            .getResult();
+  }
   rewriter.replaceOp(op, {handle, stream});
   return success();
 }
@@ -1381,7 +1520,14 @@
     return failure();
   Location loc = op.getLoc();
   auto stream = adaptor.getAsyncDependencies().front();
-  destroyDnMatCallBuilder.create(loc, rewriter, {adaptor.getDmat(), stream});
+  // use the cusparseLt destroy call if the dnmat is used with spmat with
+  // 2:4 sparsity
+  if (inferSpMMType(op.getDmat()) == "cusparseLt") {
+    destroyCuSparseLtDnMatBuilder.create(loc, rewriter,
+                                         {adaptor.getDmat(), stream});
+  } else {
+    destroyDnMatCallBuilder.create(loc, rewriter, {adaptor.getDmat(), stream});
+  }
   rewriter.replaceOp(op, {stream});
   return success();
 }
@@ -1459,6 +1605,31 @@
   return success();
 }
 
+LogicalResult ConvertCreate2To4SpMatOpToGpuRuntimeCallPattern::matchAndRewrite(
+    gpu::Create2To4SpMatOp op, OpAdaptor adaptor,
+    ConversionPatternRewriter &rewriter) const {
+  if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
+      failed(isAsyncWithOneDependency(rewriter, op)))
+    return failure();
+  Location loc = op.getLoc();
+  auto stream = adaptor.getAsyncDependencies().front();
+  Value pMat =
+      MemRefDescriptor(adaptor.getMemref()).allocatedPtr(rewriter, loc);
+  if (!getTypeConverter()->useOpaquePointers())
+    pMat = rewriter.create<LLVM::BitcastOp>(loc, llvmPointerType, pMat);
+  Type dType =
+      llvm::cast<MemRefType>(op.getMemref().getType()).getElementType();
+  auto dtp = genConstInt32From(rewriter, loc, getCuSparseDataTypeFrom(dType));
+  auto envHandle = adaptor.getEnv();
+  auto handle = create2To4SpMatCallBuilder
+                    .create(loc, rewriter,
+                            {envHandle, adaptor.getRows(), adaptor.getCols(),
+                             pMat, dtp, stream})
+                    .getResult();
+  rewriter.replaceOp(op, {handle, stream});
+  return success();
+}
+
 LogicalResult ConvertDestroySpMatOpToGpuRuntimeCallPattern::matchAndRewrite(
     gpu::DestroySpMatOp op, OpAdaptor adaptor,
     ConversionPatternRewriter &rewriter) const {
@@ -1467,7 +1638,14 @@
     return failure();
   Location loc = op.getLoc();
   auto stream = adaptor.getAsyncDependencies().front();
-  destroySpMatCallBuilder.create(loc, rewriter, {adaptor.getSpmat(), stream});
+  // use the cusparseLt destroy call if the spmat is 2:4 sparsity
+  if (is2To4Sparsity(op.getSpmat())) {
+    destroyCuSparseLtSpMatBuilder.create(loc, rewriter,
+                                         {adaptor.getSpmat(), stream});
+
+  } else {
+    destroySpMatCallBuilder.create(loc, rewriter, {adaptor.getSpmat(), stream});
+  }
   rewriter.replaceOp(op, {stream});
   return success();
 }
@@ -1526,15 +1704,26 @@
   auto modeA = genConstInt32From(rewriter, loc, adaptor.getModeA());
   auto modeB = genConstInt32From(rewriter, loc, adaptor.getModeB());
   auto stream = adaptor.getAsyncDependencies().front();
-  auto computeType =
-      genConstInt32FromComputeMode(rewriter, loc, adaptor.getComputeType());
 
-  auto bufferSize = spMMBufferSizeCallBuilder
-                        .create(loc, rewriter,
-                                {adaptor.getEnv(), modeA, modeB,
-                                 adaptor.getSpmatA(), adaptor.getDnmatB(),
-                                 adaptor.getDnmatC(), computeType, stream})
-                        .getResult();
+  Value bufferSize;
+  if (is2To4Sparsity(op.getSpmatA())) {
+
+    auto computeType =
+      genConstInt32ForLtFromComputeMode(rewriter, loc, adaptor.getComputeType());
+    bufferSize = cuSparseLtSpmmBufferSizeBuilder
+                     .create(loc, rewriter, 
+                             {adaptor.getEnv(), modeA, modeB, adaptor.getSpmatA(), computeType, stream})
+                     .getResult();
+  } else {
+   auto computeType =
+      genConstInt32FromComputeMode(rewriter, loc, adaptor.getComputeType());
+    bufferSize = spMMBufferSizeCallBuilder
+                     .create(loc, rewriter,
+                             {adaptor.getEnv(), modeA, modeB,
+                              adaptor.getSpmatA(), adaptor.getDnmatB(),
+                              adaptor.getDnmatC(), computeType, stream})
+                     .getResult();
+  }
   rewriter.replaceOp(op, {bufferSize, stream});
   return success();
 }
@@ -1574,14 +1763,31 @@
       genConstInt32FromComputeMode(rewriter, loc, adaptor.getComputeType());
 
   auto stream = adaptor.getAsyncDependencies().front();
-  Value pBuf =
-      MemRefDescriptor(adaptor.getBuffer()).allocatedPtr(rewriter, loc);
-  if (!getTypeConverter()->useOpaquePointers())
-    pBuf = rewriter.create<LLVM::BitcastOp>(loc, llvmPointerType, pBuf);
-  spMMCallBuilder.create(loc, rewriter,
-                         {adaptor.getEnv(), modeA, modeB, adaptor.getSpmatA(),
-                          adaptor.getDnmatB(), adaptor.getDnmatC(), computeType,
-                          pBuf, stream});
+
+  // lower to cusparseLt if applicable
+  if (is2To4Sparsity(op.getSpmatA())) {
+    SmallVector<Value> pBufs;
+    for (Value buffer : adaptor.getBuffers()) {
+      Value pBuf = MemRefDescriptor(buffer).allocatedPtr(rewriter, loc);
+      if (!getTypeConverter()->useOpaquePointers())
+        pBuf = rewriter.create<LLVM::BitcastOp>(loc, llvmPointerType, pBuf);
+      pBufs.push_back(pBuf);
+    }
+    cuSparseLtSpmmBuilder.create(loc, rewriter,
+                                 {adaptor.getEnv(), adaptor.getSpmatA(),
+                                  adaptor.getDnmatB(), adaptor.getDnmatC(),
+                                  computeType, pBufs[0], pBufs[1], pBufs[2],
+                                  stream});
+  } else {
+    Value pBuf = MemRefDescriptor(adaptor.getBuffers().front())
+                     .allocatedPtr(rewriter, loc);
+    if (!getTypeConverter()->useOpaquePointers())
+      pBuf = rewriter.create<LLVM::BitcastOp>(loc, llvmPointerType, pBuf);
+    spMMCallBuilder.create(loc, rewriter,
+                           {adaptor.getEnv(), modeA, modeB, adaptor.getSpmatA(),
+                            adaptor.getDnmatB(), adaptor.getDnmatC(),
+                            computeType, pBuf, stream});
+  }
   rewriter.replaceOp(op, {stream});
   return success();
 }
@@ -1646,6 +1852,7 @@
                ConvertDestroyDnMatOpToGpuRuntimeCallPattern,
                ConvertCreateCooOpToGpuRuntimeCallPattern,
                ConvertCreateCsrOpToGpuRuntimeCallPattern,
+               ConvertCreate2To4SpMatOpToGpuRuntimeCallPattern,
                ConvertDestroySpMatOpToGpuRuntimeCallPattern,
                ConvertSpMVBufferSizeOpToGpuRuntimeCallPattern,
                ConvertSpMVOpToGpuRuntimeCallPattern,
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
@@ -560,21 +560,23 @@
                rowA, colA, valA, isCOO, enableRT);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
-  auto dmatB = rewriter.create<gpu::CreateDnMatOp>(loc, dnMatHandleTp, tokenTp,
-                                                   token, szk, szn, matB);
+  auto dmatB = rewriter.create<gpu::CreateDnMatOp>(
+      loc, dnMatHandleTp, tokenTp, token, handle, szk, szn, matB);
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
-  auto dmatC = rewriter.create<gpu::CreateDnMatOp>(loc, dnMatHandleTp, tokenTp,
-                                                   token, szm, szn, matC);
+  auto dmatC = rewriter.create<gpu::CreateDnMatOp>(
+      loc, dnMatHandleTp, tokenTp, token, handle, szm, szn, matC);
   Value dnC = dmatC.getResult(0);
   token = dmatC.getAsyncToken();
 
+  auto modeA = gpu::TransposeMode::NON_TRANSPOSE;
+  auto modeB = gpu::TransposeMode::NON_TRANSPOSE;
   auto dmatCType = llvm::cast<ShapedType>(c.getType()).getElementType();
 
   // Precompute buffersize for SpMM.
   auto bufferComp = rewriter.create<gpu::SpMMBufferSizeOp>(
-      loc, indexTp, tokenTp, token, handle, spMatA, dnB, dnC,
-      /*computeType=*/dmatCType);
+      loc, indexTp, tokenTp, token, handle, modeA, modeB, spMatA, dnB, dnC,
+      /*computeType*/ dmatCType);
   Value bufferSz = bufferComp.getResult(0);
   token = bufferComp.getAsyncToken();
   auto buf = genAllocBuffer(rewriter, loc, bufferSz, token);
diff --git a/mlir/lib/ExecutionEngine/CMakeLists.txt b/mlir/lib/ExecutionEngine/CMakeLists.txt
--- a/mlir/lib/ExecutionEngine/CMakeLists.txt
+++ b/mlir/lib/ExecutionEngine/CMakeLists.txt
@@ -191,7 +191,11 @@
     find_library(CUDA_RUNTIME_LIBRARY cuda HINTS ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES} REQUIRED)
 
     # We need the libcusparse.so library.
-    find_library(CUDA_CUSPARSE_LIBRARY cusparse HINTS ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES} REQUIRED)
+    find_library(CUDA_CUSPARSE_LIBRARY cusparse HINTS ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES})
+
+    # We need to libcusparseLt.so library to provide sm80+ tensor core support for 2:4 sparsity acceleration.
+    find_library(CUDA_CUSPARSELT_LIBRARY cusparseLt HINTS ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES} )
+    find_path(CUDA_CUSPARSELT_HEADER cusparseLt.h HINTS ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} )
 
     add_mlir_library(mlir_cuda_runtime
       SHARED
@@ -200,15 +204,34 @@
       EXCLUDE_FROM_LIBMLIR
     )
     set_property(TARGET mlir_cuda_runtime PROPERTY CXX_STANDARD 14)
+    
     target_include_directories(mlir_cuda_runtime
-      PRIVATE
-      ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}
+        PRIVATE
+        ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}
     )
     target_link_libraries(mlir_cuda_runtime
-      PRIVATE
-      ${CUDA_RUNTIME_LIBRARY}
-      ${CUDA_CUSPARSE_LIBRARY}
-    )
+        PRIVATE
+        ${CUDA_RUNTIME_LIBRARY}
+      )
+ 
+    if (CUDA_CUSPARSE_LIBRARY)
+      target_link_libraries(mlir_cuda_runtime
+        PRIVATE
+        ${CUDA_CUSPARSE_LIBRARY}
+      )
+    endif()
+    if (CUDA_CUSPARSELT_LIBRARY AND CUDA_CUSPARSELT_HEADER)
+      target_include_directories(mlir_cuda_runtime
+        PRIVATE
+        ${CUDA_CUSPARSELT_HEADER}
+      )
+      target_link_libraries(mlir_cuda_runtime
+        PRIVATE
+        ${CUDA_CUSPARSELT_LIBRARY}
+      )
+    endif()
+
+    add_definitions(-DMLIR_CUDA_SM80_SPARSE_ENABLED=(CUDA_CUSPARSELT_LIBRARY AND CUDA_CUSPARSELT_HEADER))
   endif()
 
   if(MLIR_ENABLE_ROCM_RUNNER)
diff --git a/mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp b/mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp
--- a/mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp
+++ b/mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp
@@ -21,6 +21,10 @@
 #include "cuda_fp16.h"
 #include "cusparse.h"
 
+#if MLIR_CUDA_SM80_SPARSE_ENABLED
+#include "cusparseLt.h"
+#endif // MLIR_CUDA_SM80_SPARSE_ENABLED
+
 #ifdef _WIN32
 #define MLIR_CUDA_WRAPPERS_EXPORT __declspec(dllexport)
 #else
@@ -432,3 +436,152 @@
                                          matB, betap, matC, cTp,
                                          CUSPARSE_SDDMM_ALG_DEFAULT, buf))
 }
+
+///
+/// Wrapper methods for the cuSparseLt library.
+///
+#if MLIR_CUDA_SM80_SPARSE_ENABLED
+struct cusparseLtSpMatHandleAndData {
+  cusparseLtMatDescriptor_t mat;
+  void *values{nullptr};
+  // TODO: the following is associated with the SpMM operator rather than the
+  // sparse matrix. Create workspace buffers and pass them to the SpMM
+  // execution.
+  cusparseLtMatmulAlgSelection_t alg_sel;
+  cusparseLtMatmulPlan_t plan;
+  cusparseLtMatmulDescriptor_t matmul;
+};
+struct cusparseLtDnMatHandleAndData {
+  cusparseLtMatDescriptor_t mat;
+  void *values{nullptr};
+};
+struct cusparseLtWorkspaceSizes {
+  size_t workspace_size;
+  size_t compressed_size;
+  size_t compressed_buffer_size;
+};
+
+extern "C" MLIR_CUDA_WRAPPERS_EXPORT void *
+mgpuCreateSparseLtEnv(CUstream /*stream*/) {
+  cusparseLtHandle_t handle = nullptr;
+  // note that cuSparseLt still uses cusparseStatus_t
+  CUSPARSE_REPORT_IF_ERROR(cusparseLtInit(&handle))
+  return reinterpret_cast<void *>(handle);
+}
+
+extern "C" MLIR_CUDA_WRAPPERS_EXPORT void
+mgpuDestroySparseLtEnv(void *h, CUstream /*stream*/) {
+  cusparseLtHandle_t handle = reinterpret_cast<cusparseLtHandle_t>(h);
+  CUSPARSE_REPORT_IF_ERROR(cusparseLtDestroy(handle))
+}
+
+extern "C" MLIR_CUDA_WRAPPERS_EXPORT void *
+mgpuCreateCuSparseLtDnMat(void *h, intptr_t rows, intptr_t cols, void *values,
+                          int32_t dw, CUstream /*stream*/) {
+  cusparseLtMatDescriptor_t mat;
+  auto handle = reinterpret_cast<cusparseLtHandle_t *>(h);
+  cudaDataType_t dtp = dataTp(dw);
+  // assuming row-major when deciding lda
+  CUSPARSE_REPORT_IF_ERROR(
+      cusparseLtDenseDescriptorInit(handle, &mat, rows, cols, /*lda=*/cols,
+                                    /*alignment=*/16, dtp, CUSPARSE_ORDER_ROW))
+  cusparseLtDnMatHandleAndData matWithData{
+      .mat = mat,
+      .values = values,
+  };
+  return reinterpret_cast<void *>(matWithData);
+}
+
+// This can be used to destroy both dense matrices and sparse matrices in
+// cusparseLt
+extern "C" MLIR_CUDA_WRAPPERS_EXPORT void
+mgpuDestroyCuSparseLtSpMat(void *m, CUstream /*stream*/) {
+  auto matAndData = reinterpret_cast<cusparseLtSpMatHandleAndData>(m);
+  CUSPARSE_REPORT_IF_ERROR(cusparseLtMatDescriptorDestroy(&(mat->mat)))
+  // destroy the plan associated with the sparse matrix
+  CUSPARSE_REPORT_IF_ERROR(cusparseLtMatmulPlanDestroy(&(mat->plan)))
+}
+
+extern "C" MLIR_CUDA_WRAPPERS_EXPORT void
+mgpuDestroyCuSparseLtDnMat(void *m, CUstream /*stream*/) {
+  auto matAndData = reinterpret_cast<cusparseLtDnMatHandleAndData>(m);
+  CUSPARSE_REPORT_IF_ERROR(cusparseLtMatDescriptorDestroy(&(mat->mat)))
+}
+
+extern "C" MLIR_CUDA_WRAPPERS_EXPORT void *
+mgpuCusparseLtCreate2To4SpMat(void *h, intptr_t rows, intptr_t cols,
+                              void *values, int32_t dw, CUstream /*stream*/) {
+  cusparseLtSpMatHandleAndData matWithData;
+  matWithData.values = values;
+  auto handle = reinterpret_cast<cusparseLtHandle_t *>(h);
+  cudaDataType_t dtp = dataTp_cusparseLt(dw);
+  // assuming row-major when deciding lda
+  CUSPARSE_REPORT_IF_ERROR(cusparseLtStructuredDescriptorInit(
+      handle, &(matWithData.mat), rows, cols, /*ld=*/cols, /*alignment=*/16,
+      dtp, CUSPARSE_ORDER_ROW, CUSPARSELT_SPARSITY_50_PERCENT))
+
+  return reinterpret_cast<void *>(matWithData);
+}
+
+// Several things are being done in this stage, algorithm selection, planning,
+// and returning workspace and compressed matrices data buffer sizes.
+extern "C" MLIR_CUDA_WRAPPERS_EXPORT void *
+mgpuCuSparseLtSpMMBufferSize(void *h, int32_t ma, int32_t mb, void *a, int32_t ltctp, CUstream /*stream*/) {
+  // TODO: support more advanced settings, e.g., the input right operand is a
+  // sparse matrix assuming matA is the sparse matrix
+  auto handle = reinterpret_cast<cusparseLtHandle_t *>(h);
+  auto matA = reinterpret_cast<cusparseLtSpMatHandleAndData *>(a);
+
+  cusparseOperation_t modeA = static_cast<cusparseOperation_t>(ma);
+  cusparseOperation_t modeB = static_cast<cusparseOperation_t>(mb);
+  // cusparseComputeType is defined in cusparseLt.h
+  auto computeTypeLt = static_cast<cusparseComputeType>(ltctp);
+
+  CHECK_CUSPARSE(cusparseLtMatmulAlgSelectionInit(
+      handle, &(matWithData.alg_sel), &matmul, CUSPARSELT_MATMUL_ALG_DEFAULT))
+  int alg = 0;
+  CHECK_CUSPARSE(cusparseLtMatmulAlgSetAttribute(
+      handle, &(matWithData.alg_sel), CUSPARSELT_MATMUL_ALG_CONFIG_ID, &alg,
+      sizeof(alg)))
+  CHECK_CUSPARSE(cusparseLtMatmulDescriptorInit(
+      handle, &(matA.matmul), modeA, modeB, &(matA->mat),
+      &matB, &matC, &matC, computeTypeLt))
+  CHECK_CUSPARSE(cusparseLtMatmulPlanInit(handle, &(matWithData.plan), &matmul,
+                                          &(matWithData.alg_sel)))
+
+  CHECK_CUSPARSE(cusparseLtMatmulGetWorkspace(handle, &(matA.plan),
+                                              &(sizes.workspace_size)))
+  CHECK_CUSPARSE(cusparseLtSpMMACompressedSize(handle, &(matA.plan),
+                                               &(sizes.compressed_size),
+                                               &(sizes.compressed_buffer_size)))
+  // avoid zero-alloc
+  sizes.workspace_size = (sizes.workspace_size == 0 ? 1 : sizes.workspace_size);
+  sizes.compressed_size =
+      (sizes.compressed_size == 0 ? 1 : sizes.compressed_size);
+  sizes.compressed_buffer_size =
+      (sizes.compressed_buffer_size == 0 ? 1 : sizes.compressed_buffer_size);
+  return reinterpret_cast<void *>(sizes);
+}
+
+extern "C" MLIR_CUDA_WRAPPERS_EXPORT void
+mgpuCuSparseLtSpMM(void *h, void *a, void *b, void *c, int32_t dw, void *buf,
+                   void *dA_compressed, void *dA_compressedBuffer,
+                   CUstream stream) {
+  auto handle = reinterpret_cast<cusparseLtHandle_t *>(h);
+  auto matA = reinterpret_cast<cusparseLtSpMatHandleAndData *>(a);
+  auto matB = reinterpret_cast<cusparseLtDnMatHandleAndData *>(b);
+  auto matC = reinterpret_cast<cusparseLtDnMatHandleAndData *>(c);
+  ALPHABETA(dw, alpha, beta)
+
+  CHECK_CUSPARSE(cusparseLtSpMMACompress(handle, &(matA->plan), &(matA->values),
+                                         dA_compressed, dA_compressedBuffer,
+                                         stream))
+
+  // TODO: add support to multi-stream execution
+  // Perform the matrix multiplication. D = A*B+C using C==D for now
+  CHECK_CUSPARSE(cusparseLtMatmul(handle, &(matA->plan), &alpha, dA_compressed,
+                                  dB, &beta, matC->values, /*dD*/ matC->values,
+                                  d_workspace, &stream, 1))
+}
+
+#endif // MLIR_CUDA_SM80_SPARSE_ENABLED
\ No newline at end of file
diff --git a/mlir/test/Conversion/GPUCommon/2To4Sparsity/lit.local.cfg b/mlir/test/Conversion/GPUCommon/2To4Sparsity/lit.local.cfg
new file mode 100644
--- /dev/null
+++ b/mlir/test/Conversion/GPUCommon/2To4Sparsity/lit.local.cfg
@@ -0,0 +1,2 @@
+if not config.enable_cuda_runner or not config.mlir_run_cuda_sm80_tests:
+    config.unsupported = True
diff --git a/mlir/test/Conversion/GPUCommon/2To4Sparsity/lower-2to4-sparse-to-gpu-runtime-calls.mlir b/mlir/test/Conversion/GPUCommon/2To4Sparsity/lower-2to4-sparse-to-gpu-runtime-calls.mlir
new file mode 100644
--- /dev/null
+++ b/mlir/test/Conversion/GPUCommon/2To4Sparsity/lower-2to4-sparse-to-gpu-runtime-calls.mlir
@@ -0,0 +1,35 @@
+// RUN: mlir-opt %s --gpu-to-llvm='use-opaque-pointers=1' | FileCheck %s
+
+module attributes {gpu.container_module} {
+
+  // CHECK-LABEL: func @matmul
+  // CHECK: llvm.call @mgpuStreamCreate
+  // CHECK: llvm.call @mgpuMemAlloc
+  // CHECK: llvm.call @mgpuMemAlloc
+  // CHECK: llvm.call @mgpuCreateSparseLtEnv
+  // CHECK: llvm.call @mgpuDestroyCuSparseLtSpMat
+  // CHECK: llvm.call @mgpuCreateCuSparseLtDnMat
+  // CHECK: llvm.call @mgpuCuSparseLtSpMMBufferSize
+  // CHECK: llvm.call @mgpuCuSparseLtSpMM
+  // CHECK: llvm.call @mgpuDestroyCuSparseLtSpMat
+  // CHECK: llvm.call @mgpuDestroyCuSparseLtDnMat
+  // CHECK: llvm.call @mgpuDestroySparseLtEnv
+  // CHECK: llvm.call @mgpuStreamSynchronize
+  // CHECK: llvm.call @mgpuStreamDestroy
+  func.func @matmul(%arg0: index) {
+    %token0 = gpu.wait async
+    %mem1, %token1 = gpu.alloc async [%token0] (%arg0) : memref<?xf64>
+    %mem2, %token2 = gpu.alloc async [%token1] (%arg0) : memref<?xf64>
+    %env, %token3 = gpu.create_sparse_env async [%token2]
+    %spmat, %token4 = gpu.create_2to4_spmat async [%token3] %env, %arg0, %arg0, %mem1:  memref<?xf64>
+    %dnmat, %token5 = gpu.create_dn_mat async [%token4] %env, %arg0, %arg0, %mem2 : memref<?xf64>
+    %bufferSzs, %token6 = gpu.spmm_buffer_size async [%token5] %env, %spmat, %dnmat, %dnmat : index,index,index
+    %token7 = gpu.spmm async [%token6] %env, %spmat, %dnmat, %dnmat, %mem2, %mem2, %mem2 : memref<?xf64>,memref<?xf64>,memref<?xf64>
+    %token8 = gpu.destroy_sp_mat async [%token7] %spmat
+    %token9 = gpu.destroy_dn_mat async [%token8] %dnmat
+    %token10 = gpu.destroy_sparse_env async [%token9] %env
+    gpu.wait [%token10]
+    return
+  }
+
+}
diff --git a/mlir/test/Conversion/GPUCommon/lower-sparse-to-gpu-runtime-calls.mlir b/mlir/test/Conversion/GPUCommon/lower-sparse-to-gpu-runtime-calls.mlir
--- a/mlir/test/Conversion/GPUCommon/lower-sparse-to-gpu-runtime-calls.mlir
+++ b/mlir/test/Conversion/GPUCommon/lower-sparse-to-gpu-runtime-calls.mlir
@@ -52,8 +52,8 @@
     %mem2, %token2 = gpu.alloc async [%token1] (%arg0) : memref<?xf64>
     %env, %token3 = gpu.create_sparse_env async [%token2]
     %spmat, %token4 = gpu.create_csr async [%token3] %arg0, %arg0, %arg0, %mem1, %mem1, %mem2 : memref<?xindex>, memref<?xindex>, memref<?xf64>
-    %dnmat, %token5 = gpu.create_dn_mat async [%token4] %arg0, %arg0, %mem2 : memref<?xf64>
-    %bufferSz, %token6 = gpu.spmm_buffer_size async [%token5] %env, %spmat, %dnmat, %dnmat into f64
+    %dnmat, %token5 = gpu.create_dn_mat async [%token4] %env, %arg0, %arg0, %mem2 : memref<?xf64>
+    %bufferSz, %token6 = gpu.spmm_buffer_size async [%token5] %env, %spmat, %dnmat, %dnmat : index into f64
     %token7 = gpu.spmm async [%token6] %env, %spmat, %dnmat, %dnmat, %mem2 : memref<?xf64> into f64
     %token8 = gpu.destroy_sp_mat async [%token7] %spmat
     %token9 = gpu.destroy_dn_mat async [%token8] %dnmat
@@ -82,7 +82,7 @@
     %mem2, %token2 = gpu.alloc async [%token1] (%arg0) : memref<?xf64>
     %env, %token3 = gpu.create_sparse_env async [%token2]
     %spmat, %token4 = gpu.create_csr async [%token3] %arg0, %arg0, %arg0, %mem1, %mem1, %mem2 : memref<?xindex>, memref<?xindex>, memref<?xf64>
-    %dnmat, %token5 = gpu.create_dn_mat async [%token4] %arg0, %arg0, %mem2 : memref<?xf64>
+    %dnmat, %token5 = gpu.create_dn_mat async [%token4] %env, %arg0, %arg0, %mem2 : memref<?xf64>
     %bufferSz, %token6 = gpu.sddmm_buffer_size async [%token5] %env, %dnmat, %dnmat, %spmat into f64
     %token7 = gpu.sddmm async [%token6] %env, %dnmat, %dnmat, %spmat, %mem2 : memref<?xf64> into f64
     %token8 = gpu.destroy_sp_mat async [%token7] %spmat
diff --git a/mlir/test/Dialect/GPU/ops.mlir b/mlir/test/Dialect/GPU/ops.mlir
--- a/mlir/test/Dialect/GPU/ops.mlir
+++ b/mlir/test/Dialect/GPU/ops.mlir
@@ -339,9 +339,9 @@
     // CHECK: gpu.spmv async
     %token8 = gpu.spmv async [%token7] %env, %spmat, %dnvec, %dnvec, %mem2 : memref<?xf64>  into f64
     // CHECK: gpu.create_dn_mat async
-    %dnmat, %token9 = gpu.create_dn_mat async [%token8] %arg0, %arg0, %mem2 : memref<?xf64>
+    %dnmat, %token9 = gpu.create_dn_mat async [%token8] %env, %arg0, %arg0, %mem2 : memref<?xf64>
     // CHECK: gpu.spmm_buffer_size async
-    %bufferSz2, %token10 = gpu.spmm_buffer_size async [%token9] %env, %spmat, %dnmat, %dnmat  into f64
+    %bufferSz2, %token10 = gpu.spmm_buffer_size async [%token9] %env, %spmat, %dnmat, %dnmat : index into f64
     // CHECK: gpu.spmm async
     %token11 = gpu.spmm async [%token10] %env, %spmat, %dnmat, %dnmat, %mem2 : memref<?xf64>  into f64
     // CHECK: gpu.sddmm_buffer_size async
diff --git a/mlir/test/Dialect/GPU/sparse-roundtrip.mlir b/mlir/test/Dialect/GPU/sparse-roundtrip.mlir
--- a/mlir/test/Dialect/GPU/sparse-roundtrip.mlir
+++ b/mlir/test/Dialect/GPU/sparse-roundtrip.mlir
@@ -52,8 +52,8 @@
     %mem2, %token2 = gpu.alloc async [%token1] (%arg0) : memref<?xf64>
     %env, %token3 = gpu.create_sparse_env async [%token2]
     %spmat, %token4 = gpu.create_csr async [%token3] %arg0, %arg0, %arg0, %mem1, %mem1, %mem2 : memref<?xindex>, memref<?xindex>, memref<?xf64>
-    %dnmat, %token5 = gpu.create_dn_mat async [%token4] %arg0, %arg0, %mem2 : memref<?xf64>
-    %bufferSz, %token6 = gpu.spmm_buffer_size async [%token5] %env, %spmat, %dnmat, %dnmat into f64
+    %dnmat, %token5 = gpu.create_dn_mat async [%token4] %env, %arg0, %arg0, %mem2 : memref<?xf64>
+    %bufferSz, %token6 = gpu.spmm_buffer_size async [%token5] %env, %spmat, %dnmat, %dnmat : index into f64
     %token7 = gpu.spmm async [%token6] %env, %spmat, %dnmat, %dnmat, %mem2 : memref<?xf64> into f64
     %token8 = gpu.destroy_sp_mat async [%token7] %spmat
     %token9 = gpu.destroy_dn_mat async [%token8] %dnmat
@@ -82,7 +82,7 @@
     %mem2, %token2 = gpu.alloc async [%token1] (%arg0) : memref<?xf64>
     %env, %token3 = gpu.create_sparse_env async [%token2]
     %spmat, %token4 = gpu.create_csr async [%token3] %arg0, %arg0, %arg0, %mem1, %mem1, %mem2 : memref<?xindex>, memref<?xindex>, memref<?xf64>
-    %dnmat, %token5 = gpu.create_dn_mat async [%token4] %arg0, %arg0, %mem2 : memref<?xf64>
+    %dnmat, %token5 = gpu.create_dn_mat async [%token4] %env, %arg0, %arg0, %mem2 : memref<?xf64>
     %bufferSz, %token6 = gpu.sddmm_buffer_size async [%token5] %env, %dnmat, %dnmat, %spmat into f64
     %token7 = gpu.sddmm async [%token6] %env, %dnmat, %dnmat, %spmat, %mem2 : memref<?xf64> into f64
     %token8 = gpu.destroy_sp_mat async [%token7] %spmat
diff --git a/mlir/test/Dialect/SparseTensor/GPU/gpu_matmul_lib.mlir b/mlir/test/Dialect/SparseTensor/GPU/gpu_matmul_lib.mlir
--- a/mlir/test/Dialect/SparseTensor/GPU/gpu_matmul_lib.mlir
+++ b/mlir/test/Dialect/SparseTensor/GPU/gpu_matmul_lib.mlir
@@ -47,9 +47,9 @@
 // CHECK:           %[[VAL_41:.*]] = gpu.wait async
 // CHECK:           %[[VAL_42:.*]], %[[VAL_43:.*]] = gpu.create_sparse_env async {{\[}}%[[VAL_41]]]
 // CHECK:           %[[VAL_44:.*]], %[[VAL_45:.*]] = gpu.create_csr async {{\[}}%[[VAL_43]]] %[[VAL_6]], %[[VAL_7]], %[[VAL_5]], %[[VAL_14]], %[[VAL_19]], %[[VAL_24]] : memref<?xindex>, memref<?xindex>, memref<?xf64>
-// CHECK:           %[[VAL_46:.*]], %[[VAL_47:.*]] = gpu.create_dn_mat async {{\[}}%[[VAL_45]]] %[[VAL_7]], %[[VAL_8]], %[[VAL_31]] : memref<?x?xf64>
-// CHECK:           %[[VAL_48:.*]], %[[VAL_49:.*]] = gpu.create_dn_mat async {{\[}}%[[VAL_47]]] %[[VAL_6]], %[[VAL_8]], %[[VAL_38]] : memref<?x?xf64>
-// CHECK:           %[[VAL_50:.*]], %[[VAL_51:.*]] = gpu.spmm_buffer_size async {{\[}}%[[VAL_49]]] %[[VAL_42]], %[[VAL_44]], %[[VAL_46]], %[[VAL_48]]
+// CHECK:           %[[VAL_46:.*]], %[[VAL_47:.*]] = gpu.create_dn_mat async {{\[}}%[[VAL_45]]] %[[VAL_42]], %[[VAL_7]], %[[VAL_8]], %[[VAL_31]] : memref<?x?xf64>
+// CHECK:           %[[VAL_48:.*]], %[[VAL_49:.*]] = gpu.create_dn_mat async {{\[}}%[[VAL_47]]] %[[VAL_42]], %[[VAL_6]], %[[VAL_8]], %[[VAL_38]] : memref<?x?xf64>
+// CHECK:           %[[VAL_50:.*]], %[[VAL_51:.*]] = gpu.spmm_buffer_size async {{\[}}%[[VAL_49]]] %[[VAL_42]], %[[VAL_44]], %[[VAL_46]], %[[VAL_48]] : index
 // CHECK:           %[[VAL_52:.*]], %[[VAL_53:.*]] = gpu.alloc async {{\[}}%[[VAL_51]]] (%[[VAL_50]]) : memref<?xi8>
 // CHECK:           %[[VAL_54:.*]] = gpu.spmm async {{\[}}%[[VAL_53]]] %[[VAL_42]], %[[VAL_44]], %[[VAL_46]], %[[VAL_48]], %[[VAL_52]] : memref<?xi8>
 // CHECK:           %[[VAL_55:.*]] = gpu.destroy_sp_mat async {{\[}}%[[VAL_54]]] %[[VAL_44]]