Fix the segfault reported in PR 11990.

The sefault occurs due to an infinite loop when the verifier tries to
determine the size of a type of the form "%rt = type { %rt }" while
checking an alloca of the type.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@196626 91177308-0d34-0410-b5e6-96231b3b80d8

2 files changed, 11 insertions, 7 deletions

diff --git a/lib/IR/Type.cpp b/lib/IR/Type.cpp
index 03b1122e9e..86f2d89dd3 100644
--- a/lib/IR/Type.cpp
+++ b/lib/IR/Type.cpp
@@ -155,14 +155,14 @@ int Type::getFPMantissaWidth() const {
 /// isSizedDerivedType - Derived types like structures and arrays are sized
 /// iff all of the members of the type are sized as well.  Since asking for
 /// their size is relatively uncommon, move this operation out of line.
-bool Type::isSizedDerivedType() const {
+bool Type::isSizedDerivedType(SmallPtrSet<const Type*, 4> *Visited) const {
   if (const ArrayType *ATy = dyn_cast<ArrayType>(this))
-    return ATy->getElementType()->isSized();
+    return ATy->getElementType()->isSized(Visited);
 
   if (const VectorType *VTy = dyn_cast<VectorType>(this))
-    return VTy->getElementType()->isSized();
+    return VTy->getElementType()->isSized(Visited);
 
-  return cast<StructType>(this)->isSized();
+  return cast<StructType>(this)->isSized(Visited);
 }
 
 //===----------------------------------------------------------------------===//
@@ -550,17 +550,20 @@ StructType *StructType::create(StringRef Name, Type *type, ...) {
   return llvm::StructType::create(Ctx, StructFields, Name);
 }
 
-bool StructType::isSized() const {
+bool StructType::isSized(SmallPtrSet<const Type*, 4> *Visited) const {
   if ((getSubclassData() & SCDB_IsSized) != 0)
     return true;
   if (isOpaque())
     return false;
 
+  if (Visited && !Visited->insert(this))
+    return false;
+
   // Okay, our struct is sized if all of the elements are, but if one of the
   // elements is opaque, the struct isn't sized *yet*, but may become sized in
   // the future, so just bail out without caching.
   for (element_iterator I = element_begin(), E = element_end(); I != E; ++I)
-    if (!(*I)->isSized())
+    if (!(*I)->isSized(Visited))
       return false;
 
   // Here we cheat a bit and cast away const-ness. The goal is to memoize when
diff --git a/lib/IR/Verifier.cpp b/lib/IR/Verifier.cpp
index da6b573a0c..febd29f072 100644
--- a/lib/IR/Verifier.cpp
+++ b/lib/IR/Verifier.cpp
@@ -1861,11 +1861,12 @@ void Verifier::visitStoreInst(StoreInst &SI) {
 }
 
 void Verifier::visitAllocaInst(AllocaInst &AI) {
+  SmallPtrSet<const Type*, 4> Visited;
   PointerType *PTy = AI.getType();
   Assert1(PTy->getAddressSpace() == 0,
           "Allocation instruction pointer not in the generic address space!",
           &AI);
-  Assert1(PTy->getElementType()->isSized(), "Cannot allocate unsized type",
+  Assert1(PTy->getElementType()->isSized(&Visited), "Cannot allocate unsized type",
           &AI);
   Assert1(AI.getArraySize()->getType()->isIntegerTy(),
           "Alloca array size must have integer type", &AI);