[asan] Add full redzone after every stack variable

[waltl]

Summary

We were not doing that for large shadow granularity.

[waltl]

Add full redzone after every stack variable

[vitalybuka]

llvm/unittests/Transforms/Utils/ASanStackFrameLayoutTest.cpp

74–76

maybe more cases with unusual granularity?

[waltl]

Add more stack frame layout tests for large shadow granularity

[waltl]

Trivial format fixes

llvm/lib/Transforms/Utils/ASanStackFrameLayout.cpp

static inline bool CompareVars(const ASanStackVariableDescription &a,
                                const ASanStackVariableDescription &b) {
   return a.Alignment > b.Alignment;
 }
 
 // We also force minimal alignment for all vars to kMinAlignment so that vars
 // with e.g. alignment 1 and alignment 16 do not get reordered by CompareVars.
 static const size_t kMinAlignment = 16;
 
+// We want to add a full redzone after every variable.
 // The larger the variable Size the larger is the redzone.
 // The resulting frame size is a multiple of Alignment.
-static size_t VarAndRedzoneSize(size_t Size, size_t Alignment) {
+static size_t VarAndRedzoneSize(size_t Size, size_t Granularity,
+                                size_t Alignment) {
   size_t Res = 0;
   if (Size <= 4)  Res = 16;
   else if (Size <= 16) Res = 32;
   else if (Size <= 128) Res = Size + 32;
   else if (Size <= 512) Res = Size + 64;
   else if (Size <= 4096) Res = Size + 128;
   else                   Res = Size + 256;
-  return alignTo(Res, Alignment);
+  return alignTo(std::max(Res, 2 * Granularity), Alignment);
 }
 
 ASanStackFrameLayout
 ComputeASanStackFrameLayout(SmallVectorImpl<ASanStackVariableDescription> &Vars,
                             size_t Granularity, size_t MinHeaderSize) {
   assert(Granularity >= 8 && Granularity <= 64 &&
          (Granularity & (Granularity - 1)) == 0);
   assert(MinHeaderSize >= 16 && (MinHeaderSize & (MinHeaderSize - 1)) == 0 &&
for (size_t i = 0; i < NumVars; i++) {
     (void)Alignment;  // Used only in asserts.
     size_t Size = Vars[i].Size;
     assert((Alignment & (Alignment - 1)) == 0);
     assert(Layout.FrameAlignment >= Alignment);
     assert((Offset % Alignment) == 0);
     assert(Size > 0);
     size_t NextAlignment = IsLast ? Granularity
                    : std::max(Granularity, Vars[i + 1].Alignment);
-    size_t SizeWithRedzone = VarAndRedzoneSize(Size, NextAlignment);
+    size_t SizeWithRedzone = VarAndRedzoneSize(Size, Granularity,
+                                               NextAlignment);
     Vars[i].Offset = Offset;
     Offset += SizeWithRedzone;
   }
   if (Offset % MinHeaderSize) {
     Offset += MinHeaderSize - (Offset % MinHeaderSize);
   }
   Layout.FrameSize = Offset;
   assert((Layout.FrameSize % MinHeaderSize) == 0);

llvm/unittests/Transforms/Utils/ASanStackFrameLayoutTest.cpp

#define VAR(name, size, lifetime, alignment, line)                             \
   VAR(a, 3, 0, 1, 0);
   VAR(a, 4, 0, 1, 0);
   VAR(a, 7, 0, 1, 0);
   VAR(a, 8, 8, 1, 0);
   VAR(a, 9, 0, 1, 0);
   VAR(a, 16, 16, 1, 0);
   VAR(a, 41, 9, 1, 7);
   VAR(a, 105, 103, 1, 0);
+  VAR(a, 200, 97, 1, 0);
 
   TEST_LAYOUT({a1_1}, 8, 16, "1 16 1 4 a1_1", "LL1R", "LL1R");
-  TEST_LAYOUT({a1_1}, 64, 64, "1 64 1 4 a1_1", "L1", "L1");
+  TEST_LAYOUT({a1_1}, 16, 16, "1 16 1 4 a1_1", "L1R", "L1R");
+  TEST_LAYOUT({a1_1}, 32, 32, "1 32 1 4 a1_1", "L1R", "L1R");
+  TEST_LAYOUT({a1_1}, 64, 64, "1 64 1 4 a1_1", "L1R", "L1R");
   TEST_LAYOUT({p1_32}, 8, 32, "1 32 1 8 p1_32:15", "LLLL1RRR", "LLLL1RRR");
   TEST_LAYOUT({p1_32}, 8, 64, "1 64 1 8 p1_32:15", "LLLLLLLL1RRRRRRR",
               "LLLLLLLL1RRRRRRR");
 
   TEST_LAYOUT({a1_1}, 8, 32, "1 32 1 4 a1_1", "LLLL1RRR", "LLLL1RRR");
   TEST_LAYOUT({a2_1}, 8, 32, "1 32 2 4 a2_1", "LLLL2RRR", "LLLL2RRR");
   TEST_LAYOUT({a3_1}, 8, 32, "1 32 3 4 a3_1", "LLLL3RRR", "LLLL3RRR");
   TEST_LAYOUT({a4_1}, 8, 32, "1 32 4 4 a4_1", "LLLL4RRR", "LLLL4RRR");
TEST_LAYOUT(t, 8, 32, "2 256 1 11 p1_256:2700 272 1 4 a1_1",
                 "LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL1M1R");
   }
 
   {
     SmallVector<ASanStackVariableDescription, 10> t = {a1_1, a16_1, a41_1};
     TEST_LAYOUT(t, 8, 32, "3 32 1 4 a1_1 48 16 5 a16_1 80 41 7 a41_1:7",
                 "LLLL1M00MM000001RRRR", "LLLL1MSSMMSS0001RRRR");
   }
+
+  TEST_LAYOUT({a2_1}, 32, 32, "1 32 2 4 a2_1", "L2R", "L2R");
+  TEST_LAYOUT({a9_1}, 32, 32, "1 32 9 4 a9_1", "L9R", "L9R");
+  TEST_LAYOUT({a16_1}, 32, 32, "1 32 16 5 a16_1", "L16R", "LSR");
+  TEST_LAYOUT({p1_256}, 32, 32, "1 256 1 11 p1_256:2700",
+              "LLLLLLLL1R",
+              "LLLLLLLL1R");
+  TEST_LAYOUT({a41_1}, 32, 32, "1 32 41 7 a41_1:7", "L09R",
+              "LS9R");
+  TEST_LAYOUT({a105_1}, 32, 32, "1 32 105 6 a105_1", "L0009R",
+              "LSSSSR");
+  TEST_LAYOUT({a200_1}, 32, 32, "1 32 200 6 a200_1", "L0000008RR",
+              "LSSSS008RR");
+
+  {
+    SmallVector<ASanStackVariableDescription, 10> t = {a1_1, p1_256};
+    TEST_LAYOUT(t, 32, 32, "2 256 1 11 p1_256:2700 320 1 4 a1_1",
+                "LLLLLLLL1M1R",
+                "LLLLLLLL1M1R");
+  }
+
+  {
+    SmallVector<ASanStackVariableDescription, 10> t = {a1_1, a16_1, a41_1};
+    TEST_LAYOUT(t, 32, 32, "3 32 1 4 a1_1 96 16 5 a16_1 160 41 7 a41_1:7",
+                "L1M16M09R", "L1MSMS9R");
+  }
 #undef VAR
 #undef TEST_LAYOUT
 }