[mlir][arith] Add expansion pattern for ext/trunc of bf16

[rsuderman]

Summary

bf16 has a trivial truncation/extension behavior with F32 that
can be described in elementary arith operations. Include some
expansions to efficiently convert.

[jpienaar]

Nice (sorry forgot to hit submit)

mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp

207

Could we do this before cloning?

305

rm ?

[rsuderman]

Updated for jpienaar@ comments

[jpienaar]

LG to me, and optionally done via populate, I'm not sure if always good idea vs letting lower level codegen handle it (seems direct). But given not supported on all backends, SGTM.

mlir/include/mlir/Dialect/Arith/Transforms/Passes.h

41

to lower level bitcasts and shifts ?

(something useful, is not too descriptive).

[rsuderman]

In D147091#4232242, @jpienaar wrote:

LG to me, and optionally done via populate, I'm not sure if always good idea vs letting lower level codegen handle it (seems direct). But given not supported on all backends, SGTM.

I agree. It does not appear that anything depends on the pass specifically and we should be able to integrate via the populate command.

[rsuderman]

Updated comment.

[bkramer]

mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp

223

Sorry for being late, but this expansion is simply incorrect. converting from f32 to bf16 needs a rounding step.

Can this pattern be removed or at least pulled out into an opt-in pass. Having it on by default just gives us incorrect results.

mlir/include/mlir/Dialect/Arith/Transforms/Passes.h

 /// types into supported ones. This is done by splitting original power-of-two
 /// i2N integer types into two iN halves.
 void populateArithWideIntEmulationPatterns(
     WideIntEmulationConverter &typeConverter, RewritePatternSet &patterns);
 
 /// Add patterns to expand Arith ceil/floor division ops.
 void populateCeilFloorDivExpandOpsPatterns(RewritePatternSet &patterns);
 
+/// Add patterns to expand Arith bf16 patterns to lower level bitcasts/shifts.
+void populateExpandBFloat16Patterns(RewritePatternSet &patterns);
+
 /// Add patterns to expand Arith ops.
 void populateArithExpandOpsPatterns(RewritePatternSet &patterns);
 
 /// Create a pass to legalize Arith ops.
 std::unique_ptr<Pass> createArithExpandOpsPass();
 
 /// Create a pass to replace signed ops with unsigned ones where they are proven
 /// equivalent.

mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp

 //===- ExpandOps.cpp - Pass to legalize Arith ops for LLVM lowering --===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Arith/Transforms/Passes.h"
 
 #include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 namespace mlir {
 namespace arith {
 #define GEN_PASS_DEF_ARITHEXPANDOPS
 #include "mlir/Dialect/Arith/Transforms/Passes.h.inc"
 } // namespace arith
 } // namespace mlir
 
 using namespace mlir;
 
 /// Create an integer or index constant.
 static Value createConst(Location loc, Type type, int value,
                          PatternRewriter &rewriter) {
+  auto attr = rewriter.getIntegerAttr(getElementTypeOrSelf(type), value);
+  if (auto shapedTy = dyn_cast<ShapedType>(type)) {
     return rewriter.create<arith::ConstantOp>(
-      loc, rewriter.getIntegerAttr(type, value));
+        loc, DenseElementsAttr::get(shapedTy, attr));
+  }
+
+  auto constant = rewriter.create<arith::ConstantOp>(loc, attr);
+  return constant;
 }
 
 namespace {
 
 /// Expands CeilDivUIOp (n, m) into
 ///  n == 0 ? 0 : ((n-1) / m) + 1
 struct CeilDivUIOpConverter : public OpRewritePattern<arith::CeilDivUIOp> {
   using OpRewritePattern::OpRewritePattern;
LogicalResult matchAndRewrite(OpTy op,
     // Handle the case where rhs is NaN: 'isNaN(rhs) ? rhs : select'.
     Value isNaN = rewriter.create<arith::CmpFOp>(loc, arith::CmpFPredicate::UNO,
                                                  rhs, rhs);
     rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, rhs, select);
     return success();
   }
 };
 
+struct BFloat16ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
+  using OpRewritePattern::OpRewritePattern;
+  LogicalResult matchAndRewrite(arith::ExtFOp op,
+                                PatternRewriter &rewriter) const final {
+    ImplicitLocOpBuilder b(op.getLoc(), rewriter);
+    auto operand = op.getOperand();
+    Type operandTy = operand.getType();
+    Type resultTy = op.getType();
+    Type operandETy = getElementTypeOrSelf(operandTy);
+    Type resultETy = getElementTypeOrSelf(resultTy);
+
+    if (!operandETy.isBF16() || !resultETy.isF32()) {
+      return rewriter.notifyMatchFailure(op, "not a ext of bf16 to f32.");
+    }
+
+    Type i16Ty = b.getI16Type();
+    Type i32Ty = b.getI32Type();
+    if (auto shapedTy = dyn_cast<ShapedType>(operandTy)) {
+      i16Ty = shapedTy.clone(i16Ty);
+      i32Ty = shapedTy.clone(i32Ty);
+    }
+
+    Value bitcast = b.create<arith::BitcastOp>(i16Ty, operand);
+    Value exti = b.create<arith::ExtUIOp>(i32Ty, bitcast);
+
+    Value c16 = createConst(op.getLoc(), i32Ty, 16, rewriter);
+    Value shl = b.create<arith::ShLIOp>(exti, c16);
+    Value result = b.create<arith::BitcastOp>(resultTy, shl);
+
+    rewriter.replaceOp(op, result);
+    return success();
+  }
+};
+
+struct BFloat16TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
+  using OpRewritePattern::OpRewritePattern;
+  LogicalResult matchAndRewrite(arith::TruncFOp op,
+                                PatternRewriter &rewriter) const final {
+    ImplicitLocOpBuilder b(op.getLoc(), rewriter);
+    auto operand = op.getOperand();
+    Type operandTy = operand.getType();
+    Type resultTy = op.getType();
+    Type operandETy = getElementTypeOrSelf(operandTy);
+    Type resultETy = getElementTypeOrSelf(resultTy);
+
+    if (!operandETy.isF32() || !resultETy.isBF16()) {
+      return rewriter.notifyMatchFailure(op, "not a trunc of f32 to bf16.");
+    }
+
+    Type i16Ty = b.getI16Type();
+    Type i32Ty = b.getI32Type();
+    if (auto shapedTy = dyn_cast<ShapedType>(operandTy)) {
+      i16Ty = shapedTy.clone(i16Ty);
+      i32Ty = shapedTy.clone(i32Ty);
+    }
+
+    Value bitcast = b.create<arith::BitcastOp>(i32Ty, operand);
+    Value c16 = createConst(op.getLoc(), i32Ty, 16, rewriter);
+    Value shl = b.create<arith::ShRUIOp>(bitcast, c16);
+    Value trunc = b.create<arith::TruncIOp>(i16Ty, shl);
+    Value result = b.create<arith::BitcastOp>(resultTy, trunc);
+
+    rewriter.replaceOp(op, result);
+    return success();
+  }
+};
+
 struct ArithExpandOpsPass
     : public arith::impl::ArithExpandOpsBase<ArithExpandOpsPass> {
   void runOnOperation() override {
     RewritePatternSet patterns(&getContext());
     ConversionTarget target(getContext());
 
     arith::populateArithExpandOpsPatterns(patterns);
 
     target.addLegalDialect<arith::ArithDialect>();
     // clang-format off
     target.addIllegalOp<
       arith::CeilDivSIOp,
       arith::CeilDivUIOp,
       arith::FloorDivSIOp,
       arith::MaxFOp,
       arith::MinFOp
     >();
+
+    target.addDynamicallyLegalOp<arith::ExtFOp>(
+      [](arith::ExtFOp op) {
+        Type inETy = getElementTypeOrSelf(op.getOperand().getType());
+        Type outETy = getElementTypeOrSelf(op.getType());
+        return !(inETy.isBF16() && outETy.isF32());
+      });
+
+    target.addDynamicallyLegalOp<arith::TruncFOp>(
+      [](arith::TruncFOp op)  {
+        Type inETy = getElementTypeOrSelf(op.getOperand().getType());
+        Type outETy = getElementTypeOrSelf(op.getType());
+        return !(inETy.isF32() && outETy.isBF16());
+      });
+
     // clang-format on
     if (failed(applyPartialConversion(getOperation(), target,
                                       std::move(patterns))))
       signalPassFailure();
   }
 };
 
 } // namespace
 
 void mlir::arith::populateCeilFloorDivExpandOpsPatterns(
     RewritePatternSet &patterns) {
   patterns
       .add<CeilDivSIOpConverter, CeilDivUIOpConverter, FloorDivSIOpConverter>(
           patterns.getContext());
 }
 
+void mlir::arith::populateExpandBFloat16Patterns(RewritePatternSet &patterns) {
+  patterns.add<BFloat16ExtFOpConverter, BFloat16TruncFOpConverter>(
+      patterns.getContext());
+}
+
 void mlir::arith::populateArithExpandOpsPatterns(RewritePatternSet &patterns) {
   populateCeilFloorDivExpandOpsPatterns(patterns);
   // clang-format off
   patterns.add<
     MaxMinFOpConverter<MaxFOp, arith::CmpFPredicate::UGT>,
-    MaxMinFOpConverter<MinFOp, arith::CmpFPredicate::ULT>
+    MaxMinFOpConverter<MinFOp, arith::CmpFPredicate::ULT>,
+    BFloat16ExtFOpConverter,
+    BFloat16TruncFOpConverter
    >(patterns.getContext());
   // clang-format on
 }
 
 std::unique_ptr<Pass> mlir::arith::createArithExpandOpsPass() {
   return std::make_unique<ArithExpandOpsPass>();
 }

mlir/test/Dialect/Arith/expand-ops.mlir

func.func @minf(%a: f32, %b: f32) -> f32 {
   return %result : f32
 }
 // CHECK-SAME: %[[LHS:.*]]: f32, %[[RHS:.*]]: f32)
 // CHECK-NEXT: %[[CMP:.*]] = arith.cmpf ult, %[[LHS]], %[[RHS]] : f32
 // CHECK-NEXT: %[[SELECT:.*]] = arith.select %[[CMP]], %[[LHS]], %[[RHS]] : f32
 // CHECK-NEXT: %[[IS_NAN:.*]] = arith.cmpf uno, %[[RHS]], %[[RHS]] : f32
 // CHECK-NEXT: %[[RESULT:.*]] = arith.select %[[IS_NAN]], %[[RHS]], %[[SELECT]] : f32
 // CHECK-NEXT: return %[[RESULT]] : f32
+
+// -----
+
+func.func @extf_bf16(%arg0 : bf16) -> f32 {
+    %0 = arith.extf %arg0 : bf16 to f32
+    return %0 : f32
+}
+
+// CHECK-LABEL: @extf_bf16
+// CHECK-SAME: %[[ARG0:.+]]: bf16
+// CHECK-DAG: %[[BITCAST:.+]] = arith.bitcast %[[ARG0]] : bf16 to i16
+// CHECK-DAG: %[[EXT:.+]] = arith.extui %[[BITCAST]] : i16 to i32
+// CHECK-DAG: %[[C16:.+]] = arith.constant 16
+// CHECK-DAG: %[[SHLI:.+]] = arith.shli %[[EXT]], %[[C16]]
+// CHECK-DAG: %[[BITCAST:.+]] = arith.bitcast %[[SHLI]] : i32 to f32
+// CHECK: return %[[BITCAST]]
+
+// -----
+
+func.func @extf_vector_bf16(%arg0 : vector<4xbf16>) -> vector<4xf32> {
+    %0 = arith.extf %arg0 : vector<4xbf16> to vector<4xf32>
+    return %0 : vector<4xf32>
+}
+
+// CHECK-LABEL: @extf_vector_bf16
+// CHECK-SAME: %[[ARG0:.+]]: vector<4xbf16>
+// CHECK-DAG: %[[BITCAST:.+]] = arith.bitcast %[[ARG0]] : vector<4xbf16> to vector<4xi16>
+// CHECK-DAG: %[[EXT:.+]] = arith.extui %[[BITCAST]] : vector<4xi16> to vector<4xi32>
+// CHECK-DAG: %[[C16:.+]] = arith.constant dense<16>
+// CHECK-DAG: %[[SHLI:.+]] = arith.shli %[[EXT]], %[[C16]]
+// CHECK-DAG: %[[BITCAST:.+]] = arith.bitcast %[[SHLI]] : vector<4xi32> to vector<4xf32>
+// CHECK: return %[[BITCAST]]
+
+// -----
+
+func.func @truncf_f32(%arg0 : f32) -> bf16 {
+    %0 = arith.truncf %arg0 : f32 to bf16
+    return %0 : bf16
+}
+
+// CHECK-LABEL: @truncf_f32
+// CHECK-SAME: %[[ARG0:.+]]: f32
+// CHECK-DAG: %[[C16:.+]] = arith.constant 16
+// CHECK-DAG: %[[BITCAST:.+]] = arith.bitcast %[[ARG0]] : f32 to i32
+// CHECK-DAG: %[[SHR:.+]] = arith.shrui %[[BITCAST]], %[[C16]]
+// CHECK-DAG: %[[TRUNC:.+]] = arith.trunci %[[SHR]] : i32 to i16
+// CHECK-DAG: %[[BITCAST:.+]] = arith.bitcast %[[TRUNC]] : i16 to bf16
+// CHECK: return %[[BITCAST]] : bf16
+
+// -----
+
+func.func @truncf_vector_f32(%arg0 : vector<4xf32>) -> vector<4xbf16> {
+    %0 = arith.truncf %arg0 : vector<4xf32> to vector<4xbf16>
+    return %0 : vector<4xbf16>
+}
+
+// CHECK-LABEL: @truncf_vector_f32
+// CHECK-SAME: %[[ARG0:.+]]: vector<4xf32>
+// CHECK-DAG: %[[C16:.+]] = arith.constant dense<16>
+// CHECK-DAG: %[[BITCAST:.+]] = arith.bitcast %[[ARG0]] : vector<4xf32> to vector<4xi32>
+// CHECK-DAG: %[[SHR:.+]] = arith.shrui %[[BITCAST]], %[[C16]]
+// CHECK-DAG: %[[TRUNC:.+]] = arith.trunci %[[SHR]] : vector<4xi32> to vector<4xi16>
+// CHECK-DAG: %[[BITCAST:.+]] = arith.bitcast %[[TRUNC]] : vector<4xi16> to vector<4xbf16>
+// CHECK: return %[[BITCAST]] : vector<4xbf16>