Bug 18228 - Fix accepting bitcasts between vectors of pointers with a

different number of elements.

Bitcasts were passing with vectors of pointers with different number of
elements since the number of elements was checking
SrcTy->getVectorNumElements() == SrcTy->getVectorNumElements() which
isn't helpful. The addrspacecast was also wrong, but that case at least
is caught by the verifier. Refactor bitcast and addrspacecast handling
in castIsValid to be more readable and fix this problem.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@199821 91177308-0d34-0410-b5e6-96231b3b80d8

3 files changed, 64 insertions, 16 deletions

diff --git a/lib/IR/Instructions.cpp b/lib/IR/Instructions.cpp
index 5a50b73650..c805b72d92 100644
--- a/lib/IR/Instructions.cpp
+++ b/lib/IR/Instructions.cpp
@@ -2818,30 +2818,55 @@ CastInst::castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) {
         return false;
     return SrcTy->getScalarType()->isIntegerTy() &&
            DstTy->getScalarType()->isPointerTy();
-  case Instruction::BitCast:
+  case Instruction::BitCast: {
+    PointerType *SrcPtrTy = dyn_cast<PointerType>(SrcTy->getScalarType());
+    PointerType *DstPtrTy = dyn_cast<PointerType>(DstTy->getScalarType());
+
     // BitCast implies a no-op cast of type only. No bits change.
     // However, you can't cast pointers to anything but pointers.
-    if (SrcTy->isPtrOrPtrVectorTy() != DstTy->isPtrOrPtrVectorTy())
+    if (!SrcPtrTy != !DstPtrTy)
       return false;
 
     // For non-pointer cases, the cast is okay if the source and destination bit
     // widths are identical.
-    if (!SrcTy->isPtrOrPtrVectorTy())
+    if (!SrcPtrTy)
       return SrcTy->getPrimitiveSizeInBits() == DstTy->getPrimitiveSizeInBits();
 
-    // If both are pointers then the address spaces must match and vector of
-    // pointers must have the same number of elements.
-    return SrcTy->getPointerAddressSpace() == DstTy->getPointerAddressSpace() &&
-           SrcTy->isVectorTy() == DstTy->isVectorTy() &&
-           (!SrcTy->isVectorTy() ||
-            SrcTy->getVectorNumElements() == SrcTy->getVectorNumElements());
-
-  case Instruction::AddrSpaceCast:
-    return SrcTy->isPtrOrPtrVectorTy() && DstTy->isPtrOrPtrVectorTy() &&
-           SrcTy->getPointerAddressSpace() != DstTy->getPointerAddressSpace() &&
-           SrcTy->isVectorTy() == DstTy->isVectorTy() &&
-           (!SrcTy->isVectorTy() ||
-            SrcTy->getVectorNumElements() == SrcTy->getVectorNumElements());
+    // If both are pointers then the address spaces must match.
+    if (SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace())
+      return false;
+
+    // A vector of pointers must have the same number of elements.
+    if (VectorType *SrcVecTy = dyn_cast<VectorType>(SrcTy)) {
+      if (VectorType *DstVecTy = dyn_cast<VectorType>(DstTy))
+        return (SrcVecTy->getNumElements() == DstVecTy->getNumElements());
+
+      return false;
+    }
+
+    return true;
+  }
+  case Instruction::AddrSpaceCast: {
+    PointerType *SrcPtrTy = dyn_cast<PointerType>(SrcTy->getScalarType());
+    if (!SrcPtrTy)
+      return false;
+
+    PointerType *DstPtrTy = dyn_cast<PointerType>(DstTy->getScalarType());
+    if (!DstPtrTy)
+      return false;
+
+    if (SrcPtrTy->getAddressSpace() == DstPtrTy->getAddressSpace())
+      return false;
+
+    if (VectorType *SrcVecTy = dyn_cast<VectorType>(SrcTy)) {
+      if (VectorType *DstVecTy = dyn_cast<VectorType>(DstTy))
+        return (SrcVecTy->getNumElements() == DstVecTy->getNumElements());
+
+      return false;
+    }
+
+    return true;
+  }
   }
 }
 
diff --git a/test/Assembler/invalid_cast3.ll b/test/Assembler/invalid_cast3.ll
new file mode 100644
index 0000000000..cc956cee59
--- /dev/null
+++ b/test/Assembler/invalid_cast3.ll
@@ -0,0 +1,7 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+
+; CHECK: invalid cast opcode for cast from '<4 x i32*>' to '<2 x i32*>'
+define <2 x i32*> @illegal_vector_pointer_bitcast_num_elements(<4 x i32*> %c) {
+  %bc = bitcast <4 x i32*> %c to <2 x i32*>
+  ret <2 x i32*> %bc
+}
diff --git a/unittests/IR/InstructionsTest.cpp b/unittests/IR/InstructionsTest.cpp
index 0ae236cb3c..cbdf18c646 100644
--- a/unittests/IR/InstructionsTest.cpp
+++ b/unittests/IR/InstructionsTest.cpp
@@ -145,6 +145,7 @@ TEST(InstructionsTest, CastInst) {
 
   Type *V2Int64PtrTy = VectorType::get(Int64PtrTy, 2);
   Type *V2Int32PtrTy = VectorType::get(Int32PtrTy, 2);
+  Type *V4Int32PtrTy = VectorType::get(Int32PtrTy, 4);
 
   const Constant* c8 = Constant::getNullValue(V8x8Ty);
   const Constant* c64 = Constant::getNullValue(V8x64Ty);
@@ -205,6 +206,21 @@ TEST(InstructionsTest, CastInst) {
   EXPECT_FALSE(CastInst::isBitCastable(V2Int64Ty, V2Int32Ty));
 
 
+  EXPECT_FALSE(CastInst::castIsValid(Instruction::BitCast,
+                                     Constant::getNullValue(V4Int32PtrTy),
+                                     V2Int32PtrTy));
+  EXPECT_FALSE(CastInst::castIsValid(Instruction::BitCast,
+                                     Constant::getNullValue(V2Int32PtrTy),
+                                     V4Int32PtrTy));
+
+  EXPECT_FALSE(CastInst::castIsValid(Instruction::AddrSpaceCast,
+                                     Constant::getNullValue(V4Int32PtrAS1Ty),
+                                     V2Int32PtrTy));
+  EXPECT_FALSE(CastInst::castIsValid(Instruction::AddrSpaceCast,
+                                     Constant::getNullValue(V2Int32PtrTy),
+                                     V4Int32PtrAS1Ty));
+
+
   // Check that assertion is not hit when creating a cast with a vector of
   // pointers
   // First form