[asan] rewrite asan's stack frame layout

Summary:
Rewrite asan's stack frame layout.
First, most of the stack layout logic is moved into a separte file
to make it more testable and (potentially) useful for other projects.
Second, make the frames more compact by using adaptive redzones
(smaller for small objects, larger for large objects).
Third, try to minimized gaps due to large alignments (this is hypothetical since
today we don't see many stack vars aligned by more than 32).

The frames indeed become more compact, but I'll still need to run more benchmarks
before committing, but I am sking for review now to get early feedback.

This change will be accompanied by a trivial change in compiler-rt tests
to match the new frame sizes.

Reviewers: samsonov, dvyukov

Reviewed By: samsonov

CC: llvm-commits

Differential Revision: http://llvm-reviews.chandlerc.com/D2324

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@196568 91177308-0d34-0410-b5e6-96231b3b80d8

2 files changed, 49 insertions, 19 deletions

diff --git a/test/Instrumentation/AddressSanitizer/basic.ll b/test/Instrumentation/AddressSanitizer/basic.ll
index 6002b9e897..fb32e704af 100644
--- a/test/Instrumentation/AddressSanitizer/basic.ll
+++ b/test/Instrumentation/AddressSanitizer/basic.ll
@@ -89,25 +89,6 @@ entry:
 ; CHECK-NOT: = alloca
 ; CHECK: ret void
 
-; Check that asan does not touch allocas with alignment > 32.
-define void @alloca_alignment_test() sanitize_address {
-entry:
-  %x = alloca [10 x i8], align 64
-  %y = alloca [10 x i8], align 128
-  %z = alloca [10 x i8], align 256
-  call void @alloca_test_use([10 x i8]* %x)
-  call void @alloca_test_use([10 x i8]* %y)
-  call void @alloca_test_use([10 x i8]* %z)
-  ret void
-}
-
-; CHECK: define void @alloca_alignment_test()
-; CHECK: = alloca{{.*}} align 64
-; CHECK: = alloca{{.*}} align 128
-; CHECK: = alloca{{.*}} align 256
-; CHECK: ret void
-
-
 define void @LongDoubleTest(x86_fp80* nocapture %a) nounwind uwtable sanitize_address {
 entry:
     store x86_fp80 0xK3FFF8000000000000000, x86_fp80* %a, align 16
diff --git a/test/Instrumentation/AddressSanitizer/stack_layout.ll b/test/Instrumentation/AddressSanitizer/stack_layout.ll
new file mode 100644
index 0000000000..f1f4f3f703
--- /dev/null
+++ b/test/Instrumentation/AddressSanitizer/stack_layout.ll
@@ -0,0 +1,49 @@
+; Test the ASan's stack layout.
+; More tests in tests/Transforms/Utils/ASanStackFrameLayoutTest.cpp
+; RUN: opt < %s -asan -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @Use(i8*)
+
+; CHECK: internal unnamed_addr constant{{.*}}3 32 10 3 XXX 64 20 3 YYY 128 30 3 ZZZ
+; CHECK: internal unnamed_addr constant{{.*}}3 32 5 3 AAA 64 55 3 BBB 160 555 3 CCC
+; CHECK: internal unnamed_addr constant{{.*}}3 256 128 3 CCC 448 128 3 BBB 608 128 3 AAA
+
+define void @Func1() sanitize_address {
+entry:
+; CHECK-LABEL: Func1
+; CHECK: alloca [192 x i8]
+; CHECK-NOT: alloca
+; CHECK: ret void
+  %XXX = alloca [10 x i8], align 1
+  %YYY = alloca [20 x i8], align 1
+  %ZZZ = alloca [30 x i8], align 1
+  ret void
+}
+
+define void @Func2() sanitize_address {
+entry:
+; CHECK-LABEL: Func2
+; CHECK: alloca [864 x i8]
+; CHECK-NOT: alloca
+; CHECK: ret void
+  %AAA = alloca [5 x i8], align 1
+  %BBB = alloca [55 x i8], align 1
+  %CCC = alloca [555 x i8], align 1
+  ret void
+}
+
+; Check that we reorder vars according to alignment and handle large alignments.
+define void @Func3() sanitize_address {
+entry:
+; CHECK-LABEL: Func3
+; CHECK: alloca [768 x i8]
+; CHECK-NOT: alloca
+; CHECK: ret void
+  %AAA = alloca [128 x i8], align 16
+  %BBB = alloca [128 x i8], align 64
+  %CCC = alloca [128 x i8], align 256
+  ret void
+}