[msan] Handle x86 vector pack intrinsics.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@210020 91177308-0d34-0410-b5e6-96231b3b80d8

2 files changed, 74 insertions, 0 deletions

diff --git a/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 2323a8c575..b05138ac98 100644
--- a/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -1944,6 +1944,28 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     setOriginForNaryOp(I);
   }
 
+  // \brief Instrument vector shift instrinsic.
+  //
+  // This function instruments intrinsics like x86_mmx_packsswb, that
+  // packs elements of 2 input vectors into half as much bits with saturation.
+  // Shadow is propagated with the same intrinsic applied to
+  // sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
+  void handleVectorPackIntrinsic(IntrinsicInst &I) {
+    assert(I.getNumArgOperands() == 2);
+    IRBuilder<> IRB(&I);
+    Value *S1 = getShadow(&I, 0);
+    Value *S2 = getShadow(&I, 1);
+    Type *T = S1->getType();
+    Value *S1_ext = IRB.CreateSExt(
+        IRB.CreateICmpNE(S1, llvm::Constant::getNullValue(T)), T);
+    Value *S2_ext = IRB.CreateSExt(
+        IRB.CreateICmpNE(S2, llvm::Constant::getNullValue(T)), T);
+    Value *S = IRB.CreateCall2(I.getCalledValue(), S1_ext, S2_ext,
+                               "_msprop_vector_pack");
+    setShadow(&I, S);
+    setOriginForNaryOp(I);
+  }
+
   void visitIntrinsicInst(IntrinsicInst &I) {
     switch (I.getIntrinsicID()) {
     case llvm::Intrinsic::bswap:
@@ -2060,6 +2082,20 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     // case llvm::Intrinsic::x86_sse2_psll_dq_bs:
     // case llvm::Intrinsic::x86_sse2_psrl_dq_bs:
 
+    case llvm::Intrinsic::x86_sse2_packsswb_128:
+    case llvm::Intrinsic::x86_sse2_packssdw_128:
+    case llvm::Intrinsic::x86_sse2_packuswb_128:
+    case llvm::Intrinsic::x86_sse41_packusdw:
+    case llvm::Intrinsic::x86_avx2_packsswb:
+    case llvm::Intrinsic::x86_avx2_packssdw:
+    case llvm::Intrinsic::x86_avx2_packuswb:
+    case llvm::Intrinsic::x86_avx2_packusdw:
+    case llvm::Intrinsic::x86_mmx_packsswb:
+    case llvm::Intrinsic::x86_mmx_packssdw:
+    case llvm::Intrinsic::x86_mmx_packuswb:
+      handleVectorPackIntrinsic(I);
+      break;
+
     default:
       if (!handleUnknownIntrinsic(I))
         visitInstruction(I);
diff --git a/test/Instrumentation/MemorySanitizer/vector_pack.ll b/test/Instrumentation/MemorySanitizer/vector_pack.ll
new file mode 100644
index 0000000000..9c168788a5
--- /dev/null
+++ b/test/Instrumentation/MemorySanitizer/vector_pack.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32>, <4 x i32>) nounwind readnone
+declare <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> %a, <16 x i16> %b) nounwind readnone
+
+define <8 x i16> @Test_packssdw_128(<4 x i32> %a, <4 x i32> %b) sanitize_memory {
+entry:
+  %c = tail call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> %a, <4 x i32> %b) nounwind
+  ret <8 x i16> %c
+}
+
+; CHECK-LABEL: @Test_packssdw_128(
+; CHECK-DAG: icmp ne <4 x i32> {{.*}}, zeroinitializer
+; CHECK-DAG: sext <4 x i1> {{.*}} to <4 x i32>
+; CHECK-DAG: icmp ne <4 x i32> {{.*}}, zeroinitializer
+; CHECK-DAG: sext <4 x i1> {{.*}} to <4 x i32>
+; CHECK-DAG: call <8 x i16> @llvm.x86.sse2.packssdw.128(
+; CHECK-DAG: call <8 x i16> @llvm.x86.sse2.packssdw.128(
+; CHECK: ret <8 x i16>
+
+
+define <32 x i8> @Test_avx_packuswb(<16 x i16> %a, <16 x i16> %b) sanitize_memory {
+entry:
+  %c = tail call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> %a, <16 x i16> %b) nounwind
+  ret <32 x i8> %c
+}
+
+; CHECK-LABEL: @Test_avx_packuswb(
+; CHECK-DAG: icmp ne <16 x i16> {{.*}}, zeroinitializer
+; CHECK-DAG: sext <16 x i1> {{.*}} to <16 x i16>
+; CHECK-DAG: icmp ne <16 x i16> {{.*}}, zeroinitializer
+; CHECK-DAG: sext <16 x i1> {{.*}} to <16 x i16>
+; CHECK-DAG: call <32 x i8> @llvm.x86.avx2.packuswb(
+; CHECK-DAG: call <32 x i8> @llvm.x86.avx2.packuswb(
+; CHECK: ret <32 x i8>