tsan: move shadow stack from thread descriptors to fixed addresses

This allows to increase max shadow stack size to 64K,
and reliably catch shadow stack overflows instead of silently
corrupting memory.



git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@192797 91177308-0d34-0410-b5e6-96231b3b80d8

12 files changed, 132 insertions, 33 deletions

diff --git a/lib/tsan/lit_tests/deep_stack0.cc b/lib/tsan/lit_tests/deep_stack0.cc
new file mode 100644
index 00000000..7c0a8161
--- /dev/null
+++ b/lib/tsan/lit_tests/deep_stack0.cc
@@ -0,0 +1,38 @@
+// RUN: %clangxx_tsan -O1 %s -o %t && not %t 2>&1 | FileCheck %s
+#include <pthread.h>
+#include <stdio.h>
+#include <unistd.h>
+
+volatile int X;
+volatile int N;
+void (*volatile F)();
+
+static void foo() {
+  if (--N == 0)
+    X = 42;
+  else
+    F();
+}
+
+void *Thread(void *p) {
+  sleep(1);
+  F();
+  return 0;
+}
+
+int main() {
+  N = 50000;
+  F = foo;
+  pthread_t t;
+  pthread_attr_t a;
+  pthread_attr_init(&a);
+  pthread_attr_setstacksize(&a, N * 256 + (1 << 20));
+  pthread_create(&t, &a, Thread, 0);
+  X = 43;
+  pthread_join(t, 0);
+}
+
+// CHECK: WARNING: ThreadSanitizer: data race
+// CHECK:    #100 foo
+// We must output suffucuently large stack (at least 100 frames)
+
diff --git a/lib/tsan/lit_tests/deep_stack1.cc b/lib/tsan/lit_tests/deep_stack1.cc
new file mode 100644
index 00000000..929f1e60
--- /dev/null
+++ b/lib/tsan/lit_tests/deep_stack1.cc
@@ -0,0 +1,38 @@
+// RUN: %clangxx_tsan -O1 %s -o %t && not %t 2>&1 | FileCheck %s
+#include <pthread.h>
+#include <stdio.h>
+#include <unistd.h>
+
+volatile int X;
+volatile int N;
+void (*volatile F)();
+
+static void foo() {
+  if (--N == 0)
+    X = 42;
+  else
+    F();
+}
+
+void *Thread(void *p) {
+  F();
+  return 0;
+}
+
+int main() {
+  N = 50000;
+  F = foo;
+  pthread_t t;
+  pthread_attr_t a;
+  pthread_attr_init(&a);
+  pthread_attr_setstacksize(&a, N * 256 + (1 << 20));
+  pthread_create(&t, &a, Thread, 0);
+  sleep(1);
+  X = 43;
+  pthread_join(t, 0);
+}
+
+// CHECK: WARNING: ThreadSanitizer: data race
+// CHECK:    #100 foo
+// We must output suffucuently large stack (at least 100 frames)
+
diff --git a/lib/tsan/lit_tests/global_race.cc b/lib/tsan/lit_tests/global_race.cc
index 3e26792d..cbe9f147 100644
--- a/lib/tsan/lit_tests/global_race.cc
+++ b/lib/tsan/lit_tests/global_race.cc
@@ -4,7 +4,7 @@
 #include <stddef.h>
 
 int GlobalData[10];
-int y;
+int qwerty;
 namespace XXX {
   struct YYY {
     static int ZZZ[10];
@@ -14,19 +14,19 @@ namespace XXX {
 
 void *Thread(void *a) {
   GlobalData[2] = 42;
-  y = 1;
+  qwerty = 1;
   XXX::YYY::ZZZ[0] = 1;
   return 0;
 }
 
 int main() {
   fprintf(stderr, "addr=%p\n", GlobalData);
-  fprintf(stderr, "addr2=%p\n", &y);
+  fprintf(stderr, "addr2=%p\n", &qwerty);
   fprintf(stderr, "addr3=%p\n", XXX::YYY::ZZZ);
   pthread_t t;
   pthread_create(&t, 0, Thread, 0);
   GlobalData[2] = 43;
-  y = 0;
+  qwerty = 0;
   XXX::YYY::ZZZ[0] = 0;
   pthread_join(t, 0);
 }
@@ -37,6 +37,6 @@ int main() {
 // CHECK: WARNING: ThreadSanitizer: data race
 // CHECK: Location is global 'GlobalData' of size 40 at [[ADDR]] ({{.*}}+0x{{[0-9,a-f]+}})
 // CHECK: WARNING: ThreadSanitizer: data race
-// CHECK: Location is global 'y' of size 4 at [[ADDR2]] ({{.*}}+0x{{[0-9,a-f]+}})
+// CHECK: Location is global 'qwerty' of size 4 at [[ADDR2]] ({{.*}}+0x{{[0-9,a-f]+}})
 // CHECK: WARNING: ThreadSanitizer: data race
 // CHECK: Location is global 'XXX::YYY::ZZZ' of size 40 at [[ADDR3]] ({{.*}}+0x{{[0-9,a-f]+}})
diff --git a/lib/tsan/rtl/tsan_defs.h b/lib/tsan/rtl/tsan_defs.h
index 27f1db38..1e53a8e0 100644
--- a/lib/tsan/rtl/tsan_defs.h
+++ b/lib/tsan/rtl/tsan_defs.h
@@ -41,10 +41,8 @@ const int kTidBits = 13;
 const unsigned kMaxTid = 1 << kTidBits;
 const unsigned kMaxTidInClock = kMaxTid * 2;  // This includes msb 'freed' bit.
 const int kClkBits = 42;
-#ifndef TSAN_GO
-const int kShadowStackSize = 4 * 1024;
-const int kTraceStackSize = 256;
-#endif
+const uptr kShadowStackSize = 64 * 1024;
+const uptr kTraceStackSize = 256;
 
 #ifdef TSAN_SHADOW_COUNT
 # if TSAN_SHADOW_COUNT == 2 \
diff --git a/lib/tsan/rtl/tsan_platform.h b/lib/tsan/rtl/tsan_platform.h
index 70e1183a..32e22ba6 100644
--- a/lib/tsan/rtl/tsan_platform.h
+++ b/lib/tsan/rtl/tsan_platform.h
@@ -138,14 +138,20 @@ uptr GetRSS();
 
 const char *InitializePlatform();
 void FinalizePlatform();
+
+// The additional page is to catch shadow stack overflow as paging fault.
+const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace) + 4096
+    + 4095) & ~4095;
+
 uptr ALWAYS_INLINE GetThreadTrace(int tid) {
-  uptr p = kTraceMemBegin + (uptr)(tid * 2) * kTraceSize * sizeof(Event);
+  uptr p = kTraceMemBegin + (uptr)tid * kTotalTraceSize;
   DCHECK_LT(p, kTraceMemBegin + kTraceMemSize);
   return p;
 }
 
 uptr ALWAYS_INLINE GetThreadTraceHeader(int tid) {
-  uptr p = kTraceMemBegin + (uptr)(tid * 2 + 1) * kTraceSize * sizeof(Event);
+  uptr p = kTraceMemBegin + (uptr)tid * kTotalTraceSize
+      + kTraceSize * sizeof(Event);
   DCHECK_LT(p, kTraceMemBegin + kTraceMemSize);
   return p;
 }
diff --git a/lib/tsan/rtl/tsan_rtl.cc b/lib/tsan/rtl/tsan_rtl.cc
index dcaf0714..22cf3cfc 100644
--- a/lib/tsan/rtl/tsan_rtl.cc
+++ b/lib/tsan/rtl/tsan_rtl.cc
@@ -90,7 +90,6 @@ ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch,
   // they may be accessed before the ctor.
   // , ignore_reads_and_writes()
   // , in_rtl()
-  , shadow_stack_pos(&shadow_stack[0])
 #ifndef TSAN_GO
   , jmp_bufs(MBlockJmpBuf)
 #endif
@@ -201,8 +200,10 @@ void MapThreadTrace(uptr addr, uptr size) {
   DPrintf("#0: Mapping trace at %p-%p(0x%zx)\n", addr, addr + size, size);
   CHECK_GE(addr, kTraceMemBegin);
   CHECK_LE(addr + size, kTraceMemBegin + kTraceMemSize);
-  if (addr != (uptr)MmapFixedNoReserve(addr, size)) {
-    Printf("FATAL: ThreadSanitizer can not mmap thread trace\n");
+  uptr addr1 = (uptr)MmapFixedNoReserve(addr, size);
+  if (addr1 != addr) {
+    Printf("FATAL: ThreadSanitizer can not mmap thread trace (%p/%p->%p)\n",
+        addr, size, addr1);
     Die();
   }
 }
@@ -660,9 +661,9 @@ void FuncEntry(ThreadState *thr, uptr pc) {
 
   // Shadow stack maintenance can be replaced with
   // stack unwinding during trace switch (which presumably must be faster).
-  DCHECK_GE(thr->shadow_stack_pos, &thr->shadow_stack[0]);
+  DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
 #ifndef TSAN_GO
-  DCHECK_LT(thr->shadow_stack_pos, &thr->shadow_stack[kShadowStackSize]);
+  DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
 #else
   if (thr->shadow_stack_pos == thr->shadow_stack_end) {
     const int sz = thr->shadow_stack_end - thr->shadow_stack;
@@ -688,9 +689,9 @@ void FuncExit(ThreadState *thr) {
   thr->fast_state.IncrementEpoch();
   TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0);
 
-  DCHECK_GT(thr->shadow_stack_pos, &thr->shadow_stack[0]);
+  DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
 #ifndef TSAN_GO
-  DCHECK_LT(thr->shadow_stack_pos, &thr->shadow_stack[kShadowStackSize]);
+  DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
 #endif
   thr->shadow_stack_pos--;
 }
diff --git a/lib/tsan/rtl/tsan_rtl.h b/lib/tsan/rtl/tsan_rtl.h
index fc12f5d7..14d97df4 100644
--- a/lib/tsan/rtl/tsan_rtl.h
+++ b/lib/tsan/rtl/tsan_rtl.h
@@ -413,17 +413,13 @@ struct ThreadState {
   // for better performance.
   int ignore_reads_and_writes;
   int ignore_sync;
+  // C/C++ uses fixed size shadow stack embed into Trace.
+  // Go uses malloc-allocated shadow stack with dynamic size.
+  uptr *shadow_stack;
+  uptr *shadow_stack_end;
   uptr *shadow_stack_pos;
   u64 *racy_shadow_addr;
   u64 racy_state[2];
-#ifndef TSAN_GO
-  // C/C++ uses embed shadow stack of fixed size.
-  uptr shadow_stack[kShadowStackSize];
-#else
-  // Go uses satellite shadow stack with dynamic size.
-  uptr *shadow_stack;
-  uptr *shadow_stack_end;
-#endif
   MutexSet mset;
   ThreadClock clock;
 #ifndef TSAN_GO
diff --git a/lib/tsan/rtl/tsan_rtl_report.cc b/lib/tsan/rtl/tsan_rtl_report.cc
index bc26b636..c353cea8 100644
--- a/lib/tsan/rtl/tsan_rtl_report.cc
+++ b/lib/tsan/rtl/tsan_rtl_report.cc
@@ -410,7 +410,7 @@ void RestoreStack(int tid, const u64 epoch, StackTrace *stk, MutexSet *mset) {
   const u64 ebegin = RoundDown(eend, kTracePartSize);
   DPrintf("#%d: RestoreStack epoch=%zu ebegin=%zu eend=%zu partidx=%d\n",
           tid, (uptr)epoch, (uptr)ebegin, (uptr)eend, partidx);
-  InternalScopedBuffer<uptr> stack(1024);  // FIXME: de-hardcode 1024
+  InternalScopedBuffer<uptr> stack(kShadowStackSize);
   for (uptr i = 0; i < hdr->stack0.Size(); i++) {
     stack[i] = hdr->stack0.Get(i);
     DPrintf2("  #%02lu: pc=%zx\n", i, stack[i]);
diff --git a/lib/tsan/rtl/tsan_rtl_thread.cc b/lib/tsan/rtl/tsan_rtl_thread.cc
index 3ed1457e..4e451b04 100644
--- a/lib/tsan/rtl/tsan_rtl_thread.cc
+++ b/lib/tsan/rtl/tsan_rtl_thread.cc
@@ -91,18 +91,21 @@ void ThreadContext::OnStarted(void *arg) {
   epoch1 = (u64)-1;
   new(thr) ThreadState(CTX(), tid, unique_id,
       epoch0, args->stk_addr, args->stk_size, args->tls_addr, args->tls_size);
-#ifdef TSAN_GO
+#ifndef TSAN_GO
+  thr->shadow_stack = &ThreadTrace(thr->tid)->shadow_stack[0];
+  thr->shadow_stack_pos = thr->shadow_stack;
+  thr->shadow_stack_end = thr->shadow_stack + kShadowStackSize;
+#else
   // Setup dynamic shadow stack.
   const int kInitStackSize = 8;
-  args->thr->shadow_stack = (uptr*)internal_alloc(MBlockShadowStack,
+  thr->shadow_stack = (uptr*)internal_alloc(MBlockShadowStack,
       kInitStackSize * sizeof(uptr));
-  args->thr->shadow_stack_pos = thr->shadow_stack;
-  args->thr->shadow_stack_end = thr->shadow_stack + kInitStackSize;
+  thr->shadow_stack_pos = thr->shadow_stack;
+  thr->shadow_stack_end = thr->shadow_stack + kInitStackSize;
 #endif
 #ifndef TSAN_GO
-  AllocatorThreadStart(args->thr);
+  AllocatorThreadStart(thr);
 #endif
-  thr = args->thr;
   thr->fast_synch_epoch = epoch0;
   AcquireImpl(thr, 0, &sync);
   thr->fast_state.SetHistorySize(flags()->history_size);
diff --git a/lib/tsan/rtl/tsan_sync.cc b/lib/tsan/rtl/tsan_sync.cc
index c6ddcdb3..f8f3c40f 100644
--- a/lib/tsan/rtl/tsan_sync.cc
+++ b/lib/tsan/rtl/tsan_sync.cc
@@ -265,6 +265,11 @@ void StackTrace::ObtainCurrent(ThreadState *thr, uptr toppc) {
       n_ = c_ - !!toppc;
     }
   } else {
+    // Cap potentially huge stacks.
+    if (n_ + !!toppc > kTraceStackSize) {
+      start = n_ - kTraceStackSize + !!toppc;
+      n_ = kTraceStackSize - !!toppc;
+    }
     s_ = (uptr*)internal_alloc(MBlockStackTrace,
                                (n_ + !!toppc) * sizeof(s_[0]));
   }
diff --git a/lib/tsan/rtl/tsan_trace.h b/lib/tsan/rtl/tsan_trace.h
index 7df71604..5ed0356e 100644
--- a/lib/tsan/rtl/tsan_trace.h
+++ b/lib/tsan/rtl/tsan_trace.h
@@ -62,6 +62,11 @@ struct TraceHeader {
 struct Trace {
   TraceHeader headers[kTraceParts];
   Mutex mtx;
+#ifndef TSAN_GO
+  // Must be last to catch overflow as paging fault.
+  // Go shadow stack is dynamically allocated.
+  uptr shadow_stack[kShadowStackSize];
+#endif
 
   Trace()
     : mtx(MutexTypeTrace, StatMtxTrace) {
diff --git a/lib/tsan/tests/unit/tsan_stack_test.cc b/lib/tsan/tests/unit/tsan_stack_test.cc
index d5392959..9aa29676 100644
--- a/lib/tsan/tests/unit/tsan_stack_test.cc
+++ b/lib/tsan/tests/unit/tsan_stack_test.cc
@@ -19,6 +19,10 @@ namespace __tsan {
 
 static void TestStackTrace(StackTrace *trace) {
   ThreadState thr(0, 0, 0, 0, 0, 0, 0, 0);
+  uptr stack[128];
+  thr.shadow_stack = &stack[0];
+  thr.shadow_stack_pos = &stack[0];
+  thr.shadow_stack_end = &stack[128];
 
   trace->ObtainCurrent(&thr, 0);
   EXPECT_EQ(trace->Size(), (uptr)0);
@@ -60,7 +64,12 @@ TEST(StackTrace, StaticTrim) {
   ScopedInRtl in_rtl;
   uptr buf[2];
   StackTrace trace(buf, 2);
+
   ThreadState thr(0, 0, 0, 0, 0, 0, 0, 0);
+  uptr stack[128];
+  thr.shadow_stack = &stack[0];
+  thr.shadow_stack_pos = &stack[0];
+  thr.shadow_stack_end = &stack[128];
 
   *thr.shadow_stack_pos++ = 100;
   *thr.shadow_stack_pos++ = 101;