[compiler-rt] Avoid signed shift overflow in __muloXi4 and __mulvXi3

[Ka-Ka]

Summary

When compiling compiler-rt with -fsanitize=undefined and running testcases you
end up with the following warning:

UBSan: int_mulo_impl.inc:21:36: left shift of 1 by 63 places cannot be represented in type 'di_int' (aka 'long long')

This can be avoided by simply doing the shift in a matching unsigned variant of
the type.

The same kind of pattern seems to exist in int_mulv_impl.inc

This was found in an out of tree target.

[Ka-Ka]

compiler-rt/lib/builtins/int_mulo_impl.inc

20–22

An alternative implementation would be to change MIN and MAX into:

const fixint_t MAX = (fixint_t)((~(fixuint_t)0) / 2);
const fixint_t MIN = -MAX - 1;

as this code pattern is used in another part of compiler-rt in compiler-rt/lib/builtins/fp_fixint_impl.inc

compiler-rt/lib/builtins/int_mulo_impl.inc

 
 #include "int_lib.h"
 
 // Returns: a * b
 
 // Effects: sets *overflow to 1  if a * b overflows
 
 static __inline fixint_t __muloXi4(fixint_t a, fixint_t b, int *overflow) {
   const int N = (int)(sizeof(fixint_t) * CHAR_BIT);
-  const fixint_t MIN = (fixint_t)1 << (N - 1);
+  const fixint_t MIN = (fixint_t)((fixuint_t)1 << (N - 1));
   const fixint_t MAX = ~MIN;
   *overflow = 0;
   fixint_t result = a * b;
   if (a == MIN) {
     if (b != 0 && b != 1)
       *overflow = 1;
     return result;
   }
   if (b == MIN) {

compiler-rt/lib/builtins/int_mulv_impl.inc

 #include "int_lib.h"
 
 // Returns: a * b
 
 // Effects: aborts if a * b overflows
 
 static __inline fixint_t __mulvXi3(fixint_t a, fixint_t b) {
   const int N = (int)(sizeof(fixint_t) * CHAR_BIT);
-  const fixint_t MIN = (fixint_t)1 << (N - 1);
+  const fixint_t MIN = (fixint_t)((fixuint_t)1 << (N - 1));
   const fixint_t MAX = ~MIN;
   if (a == MIN) {
     if (b == 0 || b == 1)
       return a * b;
     compilerrt_abort();
   }
   if (b == MIN) {
     if (a == 0 || a == 1)

compiler-rt/lib/builtins/mulodi4.c

 //===-- mulodi4.c - Implement __mulodi4 -----------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements __mulodi4 for the compiler_rt library.
 //
 //===----------------------------------------------------------------------===//
 
 #define fixint_t di_int
+#define fixuint_t du_int
 #include "int_mulo_impl.inc"
 
 // Returns: a * b
 
 // Effects: sets *overflow to 1  if a * b overflows
 
 COMPILER_RT_ABI di_int __mulodi4(di_int a, di_int b, int *overflow) {
   return __muloXi4(a, b, overflow);
 }

compiler-rt/lib/builtins/mulosi4.c

 //===-- mulosi4.c - Implement __mulosi4 -----------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements __mulosi4 for the compiler_rt library.
 //
 //===----------------------------------------------------------------------===//
 
 #define fixint_t si_int
+#define fixuint_t su_int
 #include "int_mulo_impl.inc"
 
 // Returns: a * b
 
 // Effects: sets *overflow to 1  if a * b overflows
 
 COMPILER_RT_ABI si_int __mulosi4(si_int a, si_int b, int *overflow) {
   return __muloXi4(a, b, overflow);
 }

compiler-rt/lib/builtins/muloti4.c

 
 #ifdef CRT_HAS_128BIT
 
 // Returns: a * b
 
 // Effects: sets *overflow to 1  if a * b overflows
 
 #define fixint_t ti_int
+#define fixuint_t tu_int
 #include "int_mulo_impl.inc"
 
 COMPILER_RT_ABI ti_int __muloti4(ti_int a, ti_int b, int *overflow) {
   return __muloXi4(a, b, overflow);
 }
 
 #endif // CRT_HAS_128BIT

compiler-rt/lib/builtins/mulvdi3.c

 //===-- mulvdi3.c - Implement __mulvdi3 -----------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements __mulvdi3 for the compiler_rt library.
 //
 //===----------------------------------------------------------------------===//
 
 #define fixint_t di_int
+#define fixuint_t du_int
 #include "int_mulv_impl.inc"
 
 // Returns: a * b
 
 // Effects: aborts if a * b overflows
 
 COMPILER_RT_ABI di_int __mulvdi3(di_int a, di_int b) { return __mulvXi3(a, b); }

compiler-rt/lib/builtins/mulvsi3.c

 //===-- mulvsi3.c - Implement __mulvsi3 -----------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements __mulvsi3 for the compiler_rt library.
 //
 //===----------------------------------------------------------------------===//
 
 #define fixint_t si_int
+#define fixuint_t su_int
 #include "int_mulv_impl.inc"
 
 // Returns: a * b
 
 // Effects: aborts if a * b overflows
 
 COMPILER_RT_ABI si_int __mulvsi3(si_int a, si_int b) { return __mulvXi3(a, b); }

compiler-rt/lib/builtins/mulvti3.c

 
 #ifdef CRT_HAS_128BIT
 
 // Returns: a * b
 
 // Effects: aborts if a * b overflows
 
 #define fixint_t ti_int
+#define fixuint_t tu_int
 #include "int_mulv_impl.inc"
 
 COMPILER_RT_ABI ti_int __mulvti3(ti_int a, ti_int b) { return __mulvXi3(a, b); }
 
 #endif // CRT_HAS_128BIT