[stackprotector] Update the StackProtector pass to perform datalayout analysis.

This modifies the pass to classify every SSP-triggering AllocaInst according to
an SSPLayoutKind (LargeArray, SmallArray, AddrOf).  This analysis is collected
by the pass and made available for use, but no other pass uses it yet.

The next patch will make use of this analysis in PEI and StackSlot
passes.  The end goal is to support ssp-strong stack layout rules.

WIP.

Differential Revision: http://llvm-reviews.chandlerc.com/D1789


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@193653 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 30 insertions, 1 deletions

diff --git a/include/llvm/CodeGen/StackProtector.h b/include/llvm/CodeGen/StackProtector.h
index d23a9d0c5b..63cf9429d2 100644
--- a/include/llvm/CodeGen/StackProtector.h
+++ b/include/llvm/CodeGen/StackProtector.h
@@ -19,6 +19,7 @@
 
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/Triple.h"
+#include "llvm/ADT/ValueMap.h"
 #include "llvm/Pass.h"
 #include "llvm/Target/TargetLowering.h"
 
@@ -29,6 +30,24 @@ class Module;
 class PHINode;
 
 class StackProtector : public FunctionPass {
+public:
+  /// SSPLayoutKind.  Stack Smashing Protection (SSP) rules require that 
+  /// vulnerable stack allocations are located close the stack protector.
+  enum SSPLayoutKind {
+    SSPLK_None,       //< Did not trigger a stack protector.  No effect on data
+                      //< layout.
+    SSPLK_LargeArray, //< Array or nested array >= SSP-buffer-size.  Closest
+                      //< to the stack protector.
+    SSPLK_SmallArray, //< Array or nested array < SSP-buffer-size. 2nd closest
+                      //< to the stack protector.
+    SSPLK_AddrOf      //< The address of this allocation is exposed and
+                      //< triggered protection.  3rd closest to the protector.
+  };
+
+  /// A mapping of AllocaInsts to their required SSP layout.
+  typedef ValueMap<const AllocaInst*, SSPLayoutKind> SSPLayoutMap;
+
+private:
   const TargetMachine *TM;
 
   /// TLI - Keep a pointer of a TargetLowering to consult for determining
@@ -41,6 +60,11 @@ class StackProtector : public FunctionPass {
 
   DominatorTree *DT;
 
+  /// Layout - Mapping of allocations to the required SSPLayoutKind.
+  /// StackProtector analysis will update this map when determining if an
+  /// AllocaInst triggers a stack protector.
+  SSPLayoutMap Layout;
+
   /// \brief The minimum size of buffers that will receive stack smashing
   /// protection when -fstack-protection is used.
   unsigned SSPBufferSize;
@@ -66,7 +90,10 @@ class StackProtector : public FunctionPass {
   /// ContainsProtectableArray - Check whether the type either is an array or
   /// contains an array of sufficient size so that we need stack protectors
   /// for it.
-  bool ContainsProtectableArray(Type *Ty, bool Strong = false,
+  /// \param [out] IsLarge is set to true if a protectable array is found and
+  /// it is "large" ( >= ssp-buffer-size).  In the case of a structure with
+  /// multiple arrays, this gets set if any of them is large.
+  bool ContainsProtectableArray(Type *Ty, bool &IsLarge, bool Strong = false,
                                 bool InStruct = false) const;
 
   /// \brief Check whether a stack allocation has its address taken.
@@ -90,6 +117,8 @@ public:
     AU.addPreserved<DominatorTree>();
   }
 
+  SSPLayoutKind getSSPLayout(const AllocaInst *AI) const;
+
   virtual bool runOnFunction(Function &Fn);
 };
 } // end namespace llvm