[msan] Fix bugs when instrument x86.avx512*_cvt* intrinsics.

[pengfei]

Summary

Scalar intrinsics x86.avx512*_cvt* have an extra rounding mode operand.
We can directly ignore it to reuse the SSE/AVX math.
This fix the bug https://bugs.llvm.org/show_bug.cgi?id=48298.

[craig.topper]

llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp

2738

HasRoundingMode would be more clear.

llvm/utils/FileCheck/FileCheck.cpp

81 ↗

(On Diff #307960)

Unrelated change?

[pengfei]

Revert unrelated change.

[pengfei]

Adress Craig's comments.

[pengfei]

Fix a missed change.

[pengfei]

llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp

2738

Yes, it's much better. Thanks.

[craig.topper]

LGTM

llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp

#endif
   // number \p Out elements, and (if has 2 arguments) copies the rest of the
   // elements from \p CopyOp.
   // In most cases conversion involves floating-point value which may trigger a
   // hardware exception when not fully initialized. For this reason we require
   // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
   // We copy the shadow of \p CopyOp[NumUsedElements:] to \p
   // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
   // return a fully initialized value.
-  void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements) {
+  void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements,
+                                    bool HasRoundingMode = false) {
     IRBuilder<> IRB(&I);
     Value *CopyOp, *ConvertOp;
 
-    switch (I.getNumArgOperands()) {
-    case 3:
-      assert(isa<ConstantInt>(I.getArgOperand(2)) && "Invalid rounding mode");
-      LLVM_FALLTHROUGH;
+    assert((!HasRoundingMode ||
+            isa<ConstantInt>(I.getArgOperand(I.getNumArgOperands() - 1))) &&
+           "Invalid rounding mode");
+
+    switch (I.getNumArgOperands() - HasRoundingMode) {
     case 2:
       CopyOp = I.getArgOperand(0);
       ConvertOp = I.getArgOperand(1);
       break;
     case 1:
       ConvertOp = I.getArgOperand(0);
       CopyOp = nullptr;
       break;
void visitIntrinsicInst(IntrinsicInst &I) {
     case Intrinsic::x86_avx512_vcvtss2usi32:
     case Intrinsic::x86_avx512_cvttss2usi64:
     case Intrinsic::x86_avx512_cvttss2usi:
     case Intrinsic::x86_avx512_cvttsd2usi64:
     case Intrinsic::x86_avx512_cvttsd2usi:
     case Intrinsic::x86_avx512_cvtusi2ss:
     case Intrinsic::x86_avx512_cvtusi642sd:
     case Intrinsic::x86_avx512_cvtusi642ss:
+      handleVectorConvertIntrinsic(I, 1, true);
+      break;
     case Intrinsic::x86_sse2_cvtsd2si64:
     case Intrinsic::x86_sse2_cvtsd2si:
     case Intrinsic::x86_sse2_cvtsd2ss:
     case Intrinsic::x86_sse2_cvttsd2si64:
     case Intrinsic::x86_sse2_cvttsd2si:
     case Intrinsic::x86_sse_cvtss2si64:
     case Intrinsic::x86_sse_cvtss2si:
     case Intrinsic::x86_sse_cvttss2si64:

llvm/test/Instrumentation/MemorySanitizer/vector_cvt.ll

 ; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
 ; RUN: %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
 ; REQUIRES: x86-registered-target
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 declare i32 @llvm.x86.sse2.cvtsd2si(<2 x double>) nounwind readnone
 declare <2 x double> @llvm.x86.sse2.cvtsi2sd(<2 x double>, i32) nounwind readnone
 declare x86_mmx @llvm.x86.sse.cvtps2pi(<4 x float>) nounwind readnone
+declare i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float>, i32) nounwind readnone
 
 ; Single argument vector conversion.
 
 define i32 @test_cvtsd2si(<2 x double> %value) sanitize_memory {
 entry:
   %0 = tail call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> %value)
   ret i32 %0
 }

 ; CHECK: extractelement <4 x i32> {{.*}}, i32 1
 ; CHECK: [[S:%[_01-9a-z]+]] = or i32
 ; CHECK: icmp ne {{.*}}[[S]], 0
 ; CHECK: br
 ; CHECK: call void @__msan_warning_with_origin_noreturn
 ; CHECK: call x86_mmx @llvm.x86.sse.cvtps2pi
 ; CHECK: store i64 0, {{.*}} @__msan_retval_tls
 ; CHECK: ret x86_mmx
+
+; avx512 rounding conversion.
+
+define i32 @pr48298(<4 x float> %value) sanitize_memory {
+entry:
+  %0 = tail call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> %value, i32 11)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @pr48298
+; CHECK: extractelement <4 x i32> {{.*}}, i32 0
+; CHECK: icmp ne i32 {{.*}}, 0
+; CHECK: br
+; CHECK: call void @__msan_warning_with_origin_noreturn
+; CHECK: call i32 @llvm.x86.avx512.vcvtss2usi32
+; CHECK: store i32 0, {{.*}} @__msan_retval_tls
+; CHECK: ret i32