[asan] Fix small X86_64 ShadowOffset for non-default shadow scale

[waltl]

Summary

The requirement is that shadow memory must be aligned to page
boundaries (4k in this case). Use a closed form equation that always
satisfies this requirement.

[waltl]

Miscellaneous cleanups.

[waltl]

Simplify offset computation

[vitalybuka]

llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp

293

@eugenis why do we need 0 here and not just kDefaultShadowScale

524

UB of shifting into sign bit?
0xFFFFF000ULL?
Same above.

[kcc]

llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp

524

Please don't use constants in the code.

[eugenis]

llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp

293

No idea. Looks strange. Check code history?

[vitalybuka]

llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp

293

it's from the original commit which added entire file. so no explanation

@waltl nothing need to be changed here

524

kSmallX86_64ShadowOffset << (Mapping.Scale - kDefaultShadowScale) ?

[waltl]

Address CR comments

[waltl]

llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp

524

This isn't quite right: we'd still need a mask, and the shift value in theory may be negative. Please take a look at the updated code.

compiler-rt/lib/asan/asan_mapping.h

 #if defined(_ASAN_SHADOW_SCALE)
 static const u64 kDefaultShadowScale = _ASAN_SHADOW_SCALE;
 #else
 static const u64 kDefaultShadowScale = 3;
 #endif
 static const u64 kDefaultShadowSentinel = ~(uptr)0;
 static const u64 kDefaultShadowOffset32 = 1ULL << 29;  // 0x20000000
 static const u64 kDefaultShadowOffset64 = 1ULL << 44;
-static const u64 kDefaultShort64bitShadowOffset = 0x7FFF8000;  // < 2G.
+static const u64 kDefaultShort64bitShadowOffset =
+    0x7FFFFFFF & (0xFFFFF000 << kDefaultShadowScale);  // < 2G.
 static const u64 kIosShadowOffset32 = 1ULL << 30;  // 0x40000000
 static const u64 kIosShadowOffset64 = 0x120200000;
 static const u64 kIosSimShadowOffset32 = 1ULL << 30;
 static const u64 kIosSimShadowOffset64 = kDefaultShadowOffset64;
 static const u64 kAArch64_ShadowOffset64 = 1ULL << 36;
 static const u64 kMIPS32_ShadowOffset32 = 0x0aaa0000;
 static const u64 kMIPS64_ShadowOffset64 = 1ULL << 37;
 static const u64 kPPC64_ShadowOffset64 = 1ULL << 41;

llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp

 #endif
 static const uint64_t kDefaultShadowOffset32 = 1ULL << 29;
 static const uint64_t kDefaultShadowOffset64 = 1ULL << 44;
 static const uint64_t kDynamicShadowSentinel =
     std::numeric_limits<uint64_t>::max();
 static const uint64_t kIOSShadowOffset32 = 1ULL << 30;
 static const uint64_t kIOSSimShadowOffset32 = 1ULL << 30;
 static const uint64_t kIOSSimShadowOffset64 = kDefaultShadowOffset64;
-static const uint64_t kSmallX86_64ShadowOffset = 0x7FFF8000;  // < 2G.
 static const uint64_t kLinuxKasan_ShadowOffset64 = 0xdffffc0000000000;
 static const uint64_t kPPC64_ShadowOffset64 = 1ULL << 41;
 static const uint64_t kSystemZ_ShadowOffset64 = 1ULL << 52;
 static const uint64_t kMIPS32_ShadowOffset32 = 0x0aaa0000;
 static const uint64_t kMIPS64_ShadowOffset64 = 1ULL << 37;
 static const uint64_t kAArch64_ShadowOffset64 = 1ULL << 36;
 static const uint64_t kFreeBSD_ShadowOffset32 = 1ULL << 30;
 static const uint64_t kFreeBSD_ShadowOffset64 = 1ULL << 46;
static cl::opt<bool> ClSkipPromotableAllocas(
     cl::init(true));
 
 // These flags allow to change the shadow mapping.
 // The shadow mapping looks like
 //    Shadow = (Mem >> scale) + offset
 
 static cl::opt<int> ClMappingScale("asan-mapping-scale",
                                    cl::desc("scale of asan shadow mapping"),
                                    cl::Hidden, cl::init(0));
 
 static cl::opt<unsigned long long> ClMappingOffset(
     "asan-mapping-offset",
     cl::desc("offset of asan shadow mapping [EXPERIMENTAL]"), cl::Hidden,
     cl::init(0));
 
 // Optimization flags. Not user visible, used mostly for testing
 // and benchmarking the tool.
static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize,
   bool IsMIPS64 = TargetTriple.getArch() == Triple::mips64 ||
                   TargetTriple.getArch() == Triple::mips64el;
   bool IsAArch64 = TargetTriple.getArch() == Triple::aarch64;
   bool IsWindows = TargetTriple.isOSWindows();
   bool IsFuchsia = TargetTriple.isOSFuchsia();
 
   ShadowMapping Mapping;
 
+  Mapping.Scale = kDefaultShadowScale;
+  if (ClMappingScale.getNumOccurrences() > 0) {
+    Mapping.Scale = ClMappingScale;
+  }
+
   if (LongSize == 32) {
     // Android is always PIE, which means that the beginning of the address
     // space is always available.
     if (IsAndroid)
       Mapping.Offset = 0;
     else if (IsMIPS32)
       Mapping.Offset = kMIPS32_ShadowOffset32;
     else if (IsFreeBSD)
if (LongSize == 32) {
     else if (IsNetBSD)
       Mapping.Offset = kNetBSD_ShadowOffset64;
     else if (IsPS4CPU)
       Mapping.Offset = kPS4CPU_ShadowOffset64;
     else if (IsLinux && IsX86_64) {
       if (IsKasan)
         Mapping.Offset = kLinuxKasan_ShadowOffset64;
       else
-        Mapping.Offset = kSmallX86_64ShadowOffset;
+        Mapping.Offset = 0x7FFFFFFF & (0xFFFFF000 << Mapping.Scale);
     } else if (IsWindows && IsX86_64) {
       Mapping.Offset = kWindowsShadowOffset64;
     } else if (IsMIPS64)
       Mapping.Offset = kMIPS64_ShadowOffset64;
     else if (IsIOS)
       // If we're targeting iOS and x86, the binary is built for iOS simulator.
       // We are using dynamic shadow offset on the 64-bit devices.
       Mapping.Offset =
         IsX86_64 ? kIOSSimShadowOffset64 : kDynamicShadowSentinel;
     else if (IsAArch64)
       Mapping.Offset = kAArch64_ShadowOffset64;
     else
       Mapping.Offset = kDefaultShadowOffset64;
   }
 
   if (ClForceDynamicShadow) {
     Mapping.Offset = kDynamicShadowSentinel;
   }
 
-  Mapping.Scale = kDefaultShadowScale;
-  if (ClMappingScale.getNumOccurrences() > 0) {
-    Mapping.Scale = ClMappingScale;
-  }
-
   if (ClMappingOffset.getNumOccurrences() > 0) {
     Mapping.Offset = ClMappingOffset;
   }
 
   // OR-ing shadow offset if more efficient (at least on x86) if the offset
   // is a power of two, but on ppc64 we have to use add since the shadow
   // offset is not necessary 1/8-th of the address space.  On SystemZ,
   // we could OR the constant in a single instruction, but it's more