Support outline instrumentation for wide types

[kcc]

The code looks fine, please just add a test to llvm/test/Instrumentation/AddressSanitizer/instrumentation-with-call-threshold.ll
I actually considered to always emit calls for non 1-, 2-, 4-, -8, and 16-byte accesses to simplify the code and testing.
This will slowdown this use case somewhat, but those accesses are extremely rare anyway.
What do others think?

[ygribov]

What about AVX codes?

[kcc]

AVX access are power of two, right?
As soon as we get a hardware where we can test AVX, we'll add 32- and 64-byte accesses. They should not be instrumented with __asan_loadN

[ygribov]

Added test.

[kcc]

LGTM
I will add a test for i80 and commit

[kcc]

r206883.
thanks again.

lib/Transforms/Instrumentation/AddressSanitizer.cpp

void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
                          Value *Addr, uint32_t TypeSize, bool IsWrite,
                          Value *SizeArgument, bool UseCalls);
   Value *createSlowPathCmp(IRBuilder<> &IRB, Value *AddrLong,
                            Value *ShadowValue, uint32_t TypeSize);
   Instruction *generateCrashCode(Instruction *InsertBefore, Value *Addr,
                                  bool IsWrite, size_t AccessSizeIndex,
                                  Value *SizeArgument);
   void instrumentMemIntrinsic(MemIntrinsic *MI);
-  void instrumentMemIntrinsicParam(Instruction *OrigIns, Value *Addr,
-                                   Value *Size, Instruction *InsertBefore,
-                                   bool IsWrite, bool UseCalls);
   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
   bool runOnFunction(Function &F) override;
   bool maybeInsertAsanInitAtFunctionEntry(Function &F);
   bool doInitialization(Module &M) override;
   static char ID;  // Pass identification, replacement for typeid
 
  private:
   void initializeCallbacks(Module &M);
if (Mapping.Offset == 0)
     return Shadow;
   // (Shadow >> scale) | offset
   if (Mapping.OrShadowOffset)
     return IRB.CreateOr(Shadow, ConstantInt::get(IntptrTy, Mapping.Offset));
   else
     return IRB.CreateAdd(Shadow, ConstantInt::get(IntptrTy, Mapping.Offset));
 }
 
-void AddressSanitizer::instrumentMemIntrinsicParam(Instruction *OrigIns,
-                                                   Value *Addr, Value *Size,
-                                                   Instruction *InsertBefore,
-                                                   bool IsWrite,
-                                                   bool UseCalls) {
-  IRBuilder<> IRB(InsertBefore);
-  if (Size->getType() != IntptrTy)
-    Size = IRB.CreateIntCast(Size, IntptrTy, false);
-  // Check the first byte.
-  instrumentAddress(OrigIns, InsertBefore, Addr, 8, IsWrite, Size, false);
-  // Check the last byte.
-  IRB.SetInsertPoint(InsertBefore);
-  Value *SizeMinusOne = IRB.CreateSub(Size, ConstantInt::get(IntptrTy, 1));
-  Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
-  Value *AddrLast = IRB.CreateAdd(AddrLong, SizeMinusOne);
-  instrumentAddress(OrigIns, InsertBefore, AddrLast, 8, IsWrite, Size, false);
-}
-
 // Instrument memset/memmove/memcpy
 void AddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
   IRBuilder<> IRB(MI);
   Instruction *Call = 0;
   if (isa<MemTransferInst>(MI)) {
     Call = IRB.CreateCall3(
         isa<MemMoveInst>(MI) ? AsanMemmove : AsanMemcpy,
         IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
void AddressSanitizer::instrumentMop(Instruction *I, bool UseCalls) {
   if (TypeSize == 8  || TypeSize == 16 ||
       TypeSize == 32 || TypeSize == 64 || TypeSize == 128)
     return instrumentAddress(I, I, Addr, TypeSize, IsWrite, 0, UseCalls);
   // Instrument unusual size (but still multiple of 8).
   // We can not do it with a single check, so we do 1-byte check for the first
   // and the last bytes. We call __asan_report_*_n(addr, real_size) to be able
   // to report the actual access size.
   IRBuilder<> IRB(I);
+  Value *Size = ConstantInt::get(IntptrTy, TypeSize / 8);
+  Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
+  if (UseCalls) {
+    CallInst *Check = IRB.CreateCall2(
+        AsanMemoryAccessCallbackSized[IsWrite], AddrLong, Size);
+    Check->setDebugLoc(I->getDebugLoc());
+  } else {
     Value *LastByte =  IRB.CreateIntToPtr(
-      IRB.CreateAdd(IRB.CreatePointerCast(Addr, IntptrTy),
+        IRB.CreateAdd(AddrLong,
                       ConstantInt::get(IntptrTy, TypeSize / 8 - 1)),
         OrigPtrTy);
-  Value *Size = ConstantInt::get(IntptrTy, TypeSize / 8);
     instrumentAddress(I, I, Addr, 8, IsWrite, Size, false);
     instrumentAddress(I, I, LastByte, 8, IsWrite, Size, false);
   }
+}
 
 // Validate the result of Module::getOrInsertFunction called for an interface
 // function of AddressSanitizer. If the instrumented module defines a function
 // with the same name, their prototypes must match, otherwise
 // getOrInsertFunction returns a bitcast.
 static Function *checkInterfaceFunction(Constant *FuncOrBitcast) {
   if (isa<Function>(FuncOrBitcast)) return cast<Function>(FuncOrBitcast);
   FuncOrBitcast->dump();

test/Instrumentation/AddressSanitizer/instrumentation-with-call-threshold.ll

 ; Test asan internal compiler flags:
 ;   -asan-instrumentation-with-call-threshold
 ;   -asan-memory-access-callback-prefix
 
 ; RUN: opt < %s -asan -asan-module -asan-instrumentation-with-call-threshold=1 -S | FileCheck %s --check-prefix=CHECK-CALL
 ; RUN: opt < %s -asan -asan-module -asan-instrumentation-with-call-threshold=0 -S | FileCheck %s --check-prefix=CHECK-CALL
 ; RUN: opt < %s -asan -asan-module -asan-instrumentation-with-call-threshold=0 -asan-memory-access-callback-prefix=__foo_ -S | FileCheck %s --check-prefix=CHECK-CUSTOM-PREFIX
 ; RUN: opt < %s -asan -asan-module -asan-instrumentation-with-call-threshold=5 -S | FileCheck %s --check-prefix=CHECK-INLINE
 ; RUN: opt < %s -asan -asan-module  -S | FileCheck %s --check-prefix=CHECK-INLINE
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
 
-define void @test_load(i32* %a, i64* %b) sanitize_address {
+define void @test_load(i32* %a, i64* %b, i512* %c) sanitize_address {
 entry:
 ; CHECK-CALL: call void @__asan_load4
 ; CHECK-CALL: call void @__asan_load8
+; CHECK-CALL: call void @__asan_loadN
 ; CHECK-CUSTOM-PREFIX: call void @__foo_load4
 ; CHECK-CUSTOM-PREFIX: call void @__foo_load8
+; CHECK-CUSTOM-PREFIX: call void @__foo_loadN
 ; CHECK-INLINE-NOT: call void @__asan_load
   %tmp1 = load i32* %a
   %tmp2 = load i64* %b
+  %tmp3 = load i512* %c
   ret void
 }