[asan] rewrite asan's stack frame layout

Summary:
Rewrite asan's stack frame layout.
First, most of the stack layout logic is moved into a separte file
to make it more testable and (potentially) useful for other projects.
Second, make the frames more compact by using adaptive redzones
(smaller for small objects, larger for large objects).
Third, try to minimized gaps due to large alignments (this is hypothetical since
today we don't see many stack vars aligned by more than 32).

The frames indeed become more compact, but I'll still need to run more benchmarks
before committing, but I am sking for review now to get early feedback.

This change will be accompanied by a trivial change in compiler-rt tests
to match the new frame sizes.

Reviewers: samsonov, dvyukov

Reviewed By: samsonov

CC: llvm-commits

Differential Revision: http://llvm-reviews.chandlerc.com/D2324

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@196568 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 64 insertions, 0 deletions

diff --git a/include/llvm/Transforms/Utils/ASanStackFrameLayout.h b/include/llvm/Transforms/Utils/ASanStackFrameLayout.h
new file mode 100644
index 0000000000..d39088ccb4
--- /dev/null
+++ b/include/llvm/Transforms/Utils/ASanStackFrameLayout.h
@@ -0,0 +1,64 @@
+//===- ASanStackFrameLayout.h - ComputeASanStackFrameLayout -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This header defines ComputeASanStackFrameLayout and auxilary data structs.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_TRANSFORMS_UTILS_ASANSTACKFRAMELAYOUT_H
+#define LLVM_TRANSFORMS_UTILS_ASANSTACKFRAMELAYOUT_H
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+
+namespace llvm {
+
+class AllocaInst;
+
+// These magic constants should be the same as in
+// in asan_internal.h from ASan runtime in compiler-rt.
+static const int kAsanStackLeftRedzoneMagic = 0xf1;
+static const int kAsanStackMidRedzoneMagic = 0xf2;
+static const int kAsanStackRightRedzoneMagic = 0xf3;
+
+// Input/output data struct for ComputeASanStackFrameLayout.
+struct ASanStackVariableDescription {
+  const char *Name;  // Name of the variable that will be displayed by asan
+                     // if a stack-related bug is reported.
+  size_t Size;       // Size of the variable in bytes.
+  size_t Alignment;  // Alignment of the variable (power of 2).
+  AllocaInst *AI;    // The actual AllocaInst.
+  size_t Offset;     // Offset from the beginning of the frame;
+                     // set by ComputeASanStackFrameLayout.
+};
+
+// Output data struct for ComputeASanStackFrameLayout.
+struct ASanStackFrameLayout {
+  // Frame description, see DescribeAddressIfStack in ASan runtime.
+  SmallString<64> DescriptionString;
+  // The contents of the shadow memory for the stack frame that we need
+  // to set at function entry.
+  SmallVector<uint8_t, 64> ShadowBytes;
+  size_t FrameAlignment;  // Alignment for the entire frame.
+  size_t FrameSize;       // Size of the frame in bytes.
+};
+
+void ComputeASanStackFrameLayout(
+    // The array of stack variables. The elements may get reordered and changed.
+    SmallVectorImpl<ASanStackVariableDescription> &Vars,
+    // AddressSanitizer's shadow granularity. Usually 8, may also be 16, 32, 64.
+    size_t Granularity,
+    // The minimal size of the left-most redzone (header).
+    // At least 4 pointer sizes, power of 2, and >= Granularity.
+    // The resulting FrameSize should be multiple of MinHeaderSize.
+    size_t MinHeaderSize,
+    // The result is put here.
+    ASanStackFrameLayout *Layout);
+
+} // llvm namespace
+
+#endif  // LLVM_TRANSFORMS_UTILS_ASANSTACKFRAMELAYOUT_H