Make llvm::crc32() work also for input sizes larger than 32 bits.

[hans]

Summary

The problem was noticed by the Chrome OS toolchain folks
(crbug.com/1048445) because llvm-objcopy --add-gnu-debuglink would
insert the wrong checksum when processing a binary clarger than 4 GB.
That use case regressed in 1e1e3ba2526 when we started using
llvm::crc32() in more places.

[hans]

I was curious if it would be possible to enable the warning for this:

diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index cd48f93f80c..5ec735d9c63 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -663,6 +663,9 @@ if (LLVM_ENABLE_WARNINGS AND (LLVM_COMPILER_IS_GCC_COMPATIBLE OR CLANG_CL))
 
   # Enable -Wstring-conversion to catch misuse of string literals.
   add_flag_if_supported("-Wstring-conversion" STRING_CONVERSION_FLAG)
+
+  # Enable -Wshorten-64-to-32.
+  add_flag_if_supported("-Wshorten-64-to-32" SHORTEN_64_TO_32)
 endif (LLVM_ENABLE_WARNINGS AND (LLVM_COMPILER_IS_GCC_COMPATIBLE OR CLANG_CL))
 
 if (LLVM_COMPILER_IS_GCC_COMPATIBLE AND NOT LLVM_ENABLE_WARNINGS)

but sadly it seems LLVM relies on these conversions a lot.

[rnk]

lgtm

[manojgupta]

Thanks, should this be cherry picked to 11.0 release as well?

[hans]

In D74039#1860541, @manojgupta wrote:

Thanks, should this be cherry picked to 11.0 release as well?

Definitely! (I assume you mean 10.0) Cherry-picked as d0104a596199a41963dbba70338d9ff3c36b185a

llvm/lib/Support/CRC.cpp

uint32_t llvm::crc32(uint32_t CRC, ArrayRef<uint8_t> Data) {
   }
   return CRC ^ 0xFFFFFFFFU;
 }
 
 #else
 
 #include <zlib.h>
 uint32_t llvm::crc32(uint32_t CRC, ArrayRef<uint8_t> Data) {
-  return ::crc32(CRC, (const Bytef *)Data.data(), Data.size());
+  // Zlib's crc32() only takes a 32-bit length, so we have to iterate for larger
+  // sizes. One could use crc32_z() instead, but that's a recent (2017) addition
+  // and may not be available on all systems.
+  do {
+    ArrayRef<uint8_t> Slice = Data.take_front(UINT32_MAX);
+    CRC = ::crc32(CRC, (const Bytef *)Slice.data(), (uInt)Slice.size());
+    Data = Data.drop_front(Slice.size());
+  } while (Data.size() > 0);
+  return CRC;
 }
 
 #endif
 
 uint32_t llvm::crc32(ArrayRef<uint8_t> Data) { return crc32(0, Data); }
 
 void JamCRC::update(ArrayRef<uint8_t> Data) {
   CRC ^= 0xFFFFFFFFU; // Undo CRC-32 Init.
   CRC = crc32(CRC, Data);
   CRC ^= 0xFFFFFFFFU; // Undo CRC-32 XorOut.
 }

llvm/unittests/Support/CRCTest.cpp

 //===- llvm/unittest/Support/CRCTest.cpp - CRC tests ----------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements unit tests for CRC calculation functions.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/CRC.h"
 #include "llvm/ADT/StringExtras.h"
 #include "gtest/gtest.h"
+#include <stdlib.h>
 
 using namespace llvm;
 
 namespace {
 
 TEST(CRCTest, CRC32) {
   EXPECT_EQ(0x414FA339U, llvm::crc32(arrayRefFromStringRef(
                              "The quick brown fox jumps over the lazy dog")));
for (int j = 7; j >= 0; j--) {
       uint32_t mask = -(crc & 1);
       crc = (crc >> 1) ^ (0xEDB88320 & mask);
     }
 
     // CRCTable[i] is the CRC-32 of i without the initial and final bit flips.
     uint8_t byte = i;
     EXPECT_EQ(crc, ~llvm::crc32(0xFFFFFFFFU, byte));
   }
+
+  EXPECT_EQ(0x00000000U, llvm::crc32(arrayRefFromStringRef("")));
+}
+
+#if (SIZE_MAX > UINT32_MAX) && defined(EXPENSIVE_CHECKS)
+TEST(CRCTest, LargeCRC32) {
+  // Check that crc32 can handle inputs with sizes larger than 32 bits.
+  size_t TestSize = (size_t)UINT32_MAX + 42;
+  uint8_t *TestData = (uint8_t*)calloc(TestSize, 1);
+  if (!TestData)
+    return;
+
+  // Test expectation generated with:
+  // $ truncate --size=`echo 2^32-1+42 | bc` /tmp/foo
+  // $ crc32 /tmp/foo
+  EXPECT_EQ(0xE46F28FBU, llvm::crc32(makeArrayRef(TestData, TestSize)));
+
+  free(TestData);
 }
+#endif
 
 } // end anonymous namespace