[Stackmap] Liveness Analysis Pass

This optional register liveness analysis pass can be enabled with either
-enable-stackmap-liveness, -enable-patchpoint-liveness, or both. The pass
traverses each basic block in a machine function. For each basic block the
instructions are processed in reversed order and if a patchpoint or stackmap
instruction is encountered the current live-out register set is encoded as a
register mask and attached to the instruction.

Later on during stackmap generation the live-out register mask is processed and
also emitted as part of the stackmap.

This information is optional and intended for optimization purposes only. This
will enable a client of the stackmap to reason about the registers it can use
and which registers need to be preserved.

Reviewed by Andy

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@197317 91177308-0d34-0410-b5e6-96231b3b80d8

16 files changed, 614 insertions, 52 deletions

diff --git a/include/llvm/CodeGen/MachineFrameInfo.h b/include/llvm/CodeGen/MachineFrameInfo.h
index 747938f3f9..d5a0c16641 100644
--- a/include/llvm/CodeGen/MachineFrameInfo.h
+++ b/include/llvm/CodeGen/MachineFrameInfo.h
@@ -145,6 +145,14 @@ class MachineFrameInfo {
   /// to builtin \@llvm.returnaddress.
   bool ReturnAddressTaken;
 
+  /// HasStackMap - This boolean keeps track of whether there is a call
+  /// to builtin \@llvm.experimental.stackmap.
+  bool HasStackMap;
+
+  /// HasPatchPoint - This boolean keeps track of whether there is a call
+  /// to builtin \@llvm.experimental.patchpoint.
+  bool HasPatchPoint;
+
   /// StackSize - The prolog/epilog code inserter calculates the final stack
   /// offsets for all of the fixed size objects, updating the Objects list
   /// above.  It then updates StackSize to contain the number of bytes that need
@@ -235,6 +243,8 @@ public:
     HasVarSizedObjects = false;
     FrameAddressTaken = false;
     ReturnAddressTaken = false;
+    HasStackMap = false;
+    HasPatchPoint = false;
     AdjustsStack = false;
     HasCalls = false;
     StackProtectorIdx = -1;
@@ -280,6 +290,18 @@ public:
   bool isReturnAddressTaken() const { return ReturnAddressTaken; }
   void setReturnAddressIsTaken(bool s) { ReturnAddressTaken = s; }
 
+  /// hasStackMap - This method may be called any time after instruction
+  /// selection is complete to determine if there is a call to builtin
+  /// \@llvm.experimental.stackmap.
+  bool hasStackMap() const { return HasStackMap; }
+  void setHasStackMap(bool s = true) { HasStackMap = s; }
+
+  /// hasPatchPoint - This method may be called any time after instruction
+  /// selection is complete to determine if there is a call to builtin
+  /// \@llvm.experimental.patchpoint.
+  bool hasPatchPoint() const { return HasPatchPoint; }
+  void setHasPatchPoint(bool s = true) { HasPatchPoint = s; }
+
   /// getObjectIndexBegin - Return the minimum frame object index.
   ///
   int getObjectIndexBegin() const { return -NumFixedObjects; }
diff --git a/include/llvm/CodeGen/MachineFunction.h b/include/llvm/CodeGen/MachineFunction.h
index 43b370cccf..09cc1e5dfb 100644
--- a/include/llvm/CodeGen/MachineFunction.h
+++ b/include/llvm/CodeGen/MachineFunction.h
@@ -426,6 +426,15 @@ public:
     OperandRecycler.deallocate(Cap, Array);
   }
 
+  /// \brief Allocate and initialize a register mask with @p NumRegister bits.
+  uint32_t *allocateRegisterMask(unsigned NumRegister) {
+    unsigned Size = (NumRegister + 31) / 32;
+    uint32_t *Mask = Allocator.Allocate<uint32_t>(Size);
+    for (unsigned i = 0; i != Size; ++i)
+      Mask[i] = 0;
+    return Mask;
+  }
+
   /// allocateMemRefsArray - Allocate an array to hold MachineMemOperand
   /// pointers.  This array is owned by the MachineFunction.
   MachineInstr::mmo_iterator allocateMemRefsArray(unsigned long Num);
diff --git a/include/llvm/CodeGen/MachineOperand.h b/include/llvm/CodeGen/MachineOperand.h
index 40f3580bfd..c2a0f65666 100644
--- a/include/llvm/CodeGen/MachineOperand.h
+++ b/include/llvm/CodeGen/MachineOperand.h
@@ -56,6 +56,7 @@ public:
     MO_GlobalAddress,          ///< Address of a global value
     MO_BlockAddress,           ///< Address of a basic block
     MO_RegisterMask,           ///< Mask of preserved registers.
+    MO_RegisterLiveOut,        ///< Mask of live-out registers.
     MO_Metadata,               ///< Metadata reference (for debug info)
     MO_MCSymbol                ///< MCSymbol reference (for debug/eh info)
   };
@@ -153,7 +154,7 @@ private:
     const ConstantFP *CFP;    // For MO_FPImmediate.
     const ConstantInt *CI;    // For MO_CImmediate. Integers > 64bit.
     int64_t ImmVal;           // For MO_Immediate.
-    const uint32_t *RegMask;  // For MO_RegisterMask.
+    const uint32_t *RegMask;  // For MO_RegisterMask and MO_RegisterLiveOut.
     const MDNode *MD;         // For MO_Metadata.
     MCSymbol *Sym;            // For MO_MCSymbol
 
@@ -246,6 +247,8 @@ public:
   bool isBlockAddress() const { return OpKind == MO_BlockAddress; }
   /// isRegMask - Tests if this is a MO_RegisterMask operand.
   bool isRegMask() const { return OpKind == MO_RegisterMask; }
+  /// isRegLiveOut - Tests if this is a MO_RegisterLiveOut operand.
+  bool isRegLiveOut() const { return OpKind == MO_RegisterLiveOut; }
   /// isMetadata - Tests if this is a MO_Metadata operand.
   bool isMetadata() const { return OpKind == MO_Metadata; }
   bool isMCSymbol() const { return OpKind == MO_MCSymbol; }
@@ -476,6 +479,12 @@ public:
     return Contents.RegMask;
   }
 
+  /// getRegLiveOut - Returns a bit mask of live-out registers.
+  const uint32_t *getRegLiveOut() const {
+    assert(isRegLiveOut() && "Wrong MachineOperand accessor");
+    return Contents.RegMask;
+  }
+
   const MDNode *getMetadata() const {
     assert(isMetadata() && "Wrong MachineOperand accessor");
     return Contents.MD;
@@ -659,6 +668,12 @@ public:
     Op.Contents.RegMask = Mask;
     return Op;
   }
+  static MachineOperand CreateRegLiveOut(const uint32_t *Mask) {
+    assert(Mask && "Missing live-out register mask");
+    MachineOperand Op(MachineOperand::MO_RegisterLiveOut);
+    Op.Contents.RegMask = Mask;
+    return Op;
+  }
   static MachineOperand CreateMetadata(const MDNode *Meta) {
     MachineOperand Op(MachineOperand::MO_Metadata);
     Op.Contents.MD = Meta;
diff --git a/include/llvm/CodeGen/Passes.h b/include/llvm/CodeGen/Passes.h
index ae4a2fa0bf..3fbc081ea3 100644
--- a/include/llvm/CodeGen/Passes.h
+++ b/include/llvm/CodeGen/Passes.h
@@ -568,6 +568,11 @@ namespace llvm {
   /// bundles (created earlier, e.g. during pre-RA scheduling).
   extern char &FinalizeMachineBundlesID;
 
+  /// StackMapLiveness - This pass analyses the register live-out set of
+  /// stackmap/patchpoint intrinsics and attaches the calculated information to
+  /// the intrinsic for later emission to the StackMap.
+  extern char &StackMapLivenessID;
+
 } // End llvm namespace
 
 #endif
diff --git a/include/llvm/CodeGen/StackMapLivenessAnalysis.h b/include/llvm/CodeGen/StackMapLivenessAnalysis.h
new file mode 100644
index 0000000000..86ff5ebebd
--- /dev/null
+++ b/include/llvm/CodeGen/StackMapLivenessAnalysis.h
@@ -0,0 +1,65 @@
+//===--- StackMapLivenessAnalysis - StackMap Liveness Analysis --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass calculates the liveness for each basic block in a function and
+// attaches the register live-out information to a stackmap or patchpoint
+// intrinsic if present.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_STACKMAP_LIVENESS_ANALYSIS_H
+#define LLVM_CODEGEN_STACKMAP_LIVENESS_ANALYSIS_H
+
+#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+
+
+namespace llvm {
+
+/// \brief This pass calculates the liveness information for each basic block in
+/// a function and attaches the register live-out information to a stackmap or
+/// patchpoint intrinsic if present.
+///
+/// This is an optional pass that has to be explicitly enabled via the
+/// -enable-stackmap-liveness and/or -enable-patchpoint-liveness flag. The pass
+/// skips functions that don't have any stackmap or patchpoint intrinsics. The
+/// information provided by this pass is optional and not required by the
+/// aformentioned intrinsics to function.
+class StackMapLiveness : public MachineFunctionPass {
+  MachineFunction *MF;
+  const TargetRegisterInfo *TRI;
+  LivePhysRegs LiveRegs;
+public:
+  static char ID;
+
+  /// \brief Default construct and initialize the pass.
+  StackMapLiveness();
+
+  /// \brief Tell the pass manager which passes we depend on and what
+  /// information we preserve.
+  virtual void getAnalysisUsage(AnalysisUsage &AU) const;
+
+  /// \brief Calculate the liveness information for the given machine function.
+  virtual bool runOnMachineFunction(MachineFunction &MF);
+
+private:
+  /// \brief Performs the actual liveness calculation for the function.
+  bool calculateLiveness();
+
+  /// \brief Add the current register live set to the instruction.
+  void addLiveOutSetToMI(MachineInstr &MI);
+
+  /// \brief Create a register mask and initialize it with the registers from
+  /// the register live set.
+  uint32_t *createRegisterMask() const;
+};
+
+} // llvm namespace
+
+#endif // LLVM_CODEGEN_STACKMAP_LIVENESS_ANALYSIS_H
diff --git a/include/llvm/CodeGen/StackMaps.h b/include/llvm/CodeGen/StackMaps.h
index c61ba59eeb..40de34fdc6 100644
--- a/include/llvm/CodeGen/StackMaps.h
+++ b/include/llvm/CodeGen/StackMaps.h
@@ -93,6 +93,22 @@ public:
       : LocType(LocType), Size(Size), Reg(Reg), Offset(Offset) {}
   };
 
+  struct LiveOutReg {
+    unsigned short Reg;
+    unsigned short RegNo;
+    unsigned short Size;
+
+    LiveOutReg() : Reg(0), RegNo(0), Size(0) {}
+    LiveOutReg(unsigned short Reg, unsigned short RegNo, unsigned short Size)
+      : Reg(Reg), RegNo(RegNo), Size(Size) {}
+
+    void MarkInvalid() { Reg = 0; }
+
+    // Only sort by the dwarf register number.
+    bool operator< (const LiveOutReg &LO) const { return RegNo < LO.RegNo; }
+    static bool IsInvalid(const LiveOutReg &LO) { return LO.Reg == 0; }
+  };
+
   // OpTypes are used to encode information about the following logical
   // operand (which may consist of several MachineOperands) for the
   // OpParser.
@@ -115,15 +131,18 @@ public:
 
 private:
   typedef SmallVector<Location, 8> LocationVec;
+  typedef SmallVector<LiveOutReg, 8> LiveOutVec;
 
   struct CallsiteInfo {
     const MCExpr *CSOffsetExpr;
     uint64_t ID;
     LocationVec Locations;
+    LiveOutVec LiveOuts;
     CallsiteInfo() : CSOffsetExpr(0), ID(0) {}
     CallsiteInfo(const MCExpr *CSOffsetExpr, uint64_t ID,
-                 LocationVec Locations)
-      : CSOffsetExpr(CSOffsetExpr), ID(ID), Locations(Locations) {}
+                 LocationVec &Locations, LiveOutVec &LiveOuts)
+      : CSOffsetExpr(CSOffsetExpr), ID(ID), Locations(Locations),
+        LiveOuts(LiveOuts) {}
   };
 
   typedef std::vector<CallsiteInfo> CallsiteInfoList;
@@ -154,8 +173,15 @@ private:
 
   std::pair<Location, MachineInstr::const_mop_iterator>
   parseOperand(MachineInstr::const_mop_iterator MOI,
-               MachineInstr::const_mop_iterator MOE);
+               MachineInstr::const_mop_iterator MOE) const;
+
+  /// \brief Create a live-out register record for the given register @p Reg.
+  LiveOutReg createLiveOutReg(unsigned Reg, const MCRegisterInfo &MCRI,
+                              const TargetRegisterInfo *TRI) const;
 
+  /// \brief Parse the register live-out mask and return a vector of live-out
+  /// registers that need to be recorded in the stackmap.
+  LiveOutVec parseRegisterLiveOutMask(const uint32_t *Mask) const;
 
   /// This should be called by the MC lowering code _immediately_ before
   /// lowering the MI to an MCInst. It records where the operands for the
diff --git a/include/llvm/InitializePasses.h b/include/llvm/InitializePasses.h
index 029ae30859..7be62ca47c 100644
--- a/include/llvm/InitializePasses.h
+++ b/include/llvm/InitializePasses.h
@@ -266,6 +266,7 @@ void initializeLoopVectorizePass(PassRegistry&);
 void initializeSLPVectorizerPass(PassRegistry&);
 void initializeBBVectorizePass(PassRegistry&);
 void initializeMachineFunctionPrinterPassPass(PassRegistry&);
+void initializeStackMapLivenessPass(PassRegistry&);
 }
 
 #endif
diff --git a/lib/CodeGen/CMakeLists.txt b/lib/CodeGen/CMakeLists.txt
index 0beca0ceed..123d86f950 100644
--- a/lib/CodeGen/CMakeLists.txt
+++ b/lib/CodeGen/CMakeLists.txt
@@ -97,6 +97,7 @@ add_llvm_library(LLVMCodeGen
   StackColoring.cpp
   StackProtector.cpp
   StackSlotColoring.cpp
+  StackMapLivenessAnalysis.cpp
   StackMaps.cpp
   TailDuplication.cpp
   TargetFrameLoweringImpl.cpp
diff --git a/lib/CodeGen/CodeGen.cpp b/lib/CodeGen/CodeGen.cpp
index 7430c53b8d..30b40a12da 100644
--- a/lib/CodeGen/CodeGen.cpp
+++ b/lib/CodeGen/CodeGen.cpp
@@ -69,6 +69,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeVirtRegRewriterPass(Registry);
   initializeLowerIntrinsicsPass(Registry);
   initializeMachineFunctionPrinterPassPass(Registry);
+  initializeStackMapLivenessPass(Registry);
 }
 
 void LLVMInitializeCodeGen(LLVMPassRegistryRef R) {
diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp
index 8d89276e73..79939d075d 100644
--- a/lib/CodeGen/MachineInstr.cpp
+++ b/lib/CodeGen/MachineInstr.cpp
@@ -199,7 +199,8 @@ bool MachineOperand::isIdenticalTo(const MachineOperand &Other) const {
   case MachineOperand::MO_BlockAddress:
     return getBlockAddress() == Other.getBlockAddress() &&
            getOffset() == Other.getOffset();
-  case MO_RegisterMask:
+  case MachineOperand::MO_RegisterMask:
+  case MachineOperand::MO_RegisterLiveOut:
     return getRegMask() == Other.getRegMask();
   case MachineOperand::MO_MCSymbol:
     return getMCSymbol() == Other.getMCSymbol();
@@ -241,6 +242,7 @@ hash_code llvm::hash_value(const MachineOperand &MO) {
     return hash_combine(MO.getType(), MO.getTargetFlags(),
                         MO.getBlockAddress(), MO.getOffset());
   case MachineOperand::MO_RegisterMask:
+  case MachineOperand::MO_RegisterLiveOut:
     return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getRegMask());
   case MachineOperand::MO_Metadata:
     return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getMetadata());
@@ -368,6 +370,9 @@ void MachineOperand::print(raw_ostream &OS, const TargetMachine *TM) const {
   case MachineOperand::MO_RegisterMask:
     OS << "<regmask>";
     break;
+  case MachineOperand::MO_RegisterLiveOut:
+    OS << "<regliveout>";
+    break;
   case MachineOperand::MO_Metadata:
     OS << '<';
     WriteAsOperand(OS, getMetadata(), /*PrintType=*/false);
diff --git a/lib/CodeGen/Passes.cpp b/lib/CodeGen/Passes.cpp
index db7021372b..c9a55f0788 100644
--- a/lib/CodeGen/Passes.cpp
+++ b/lib/CodeGen/Passes.cpp
@@ -30,6 +30,11 @@
 
 using namespace llvm;
 
+namespace llvm {
+extern cl::opt<bool> EnableStackMapLiveness;
+extern cl::opt<bool> EnablePatchPointLiveness;
+}
+
 static cl::opt<bool> DisablePostRA("disable-post-ra", cl::Hidden,
     cl::desc("Disable Post Regalloc"));
 static cl::opt<bool> DisableBranchFold("disable-branch-fold", cl::Hidden,
@@ -536,6 +541,9 @@ void TargetPassConfig::addMachinePasses() {
 
   if (addPreEmitPass())
     printAndVerify("After PreEmit passes");
+
+  if (EnableStackMapLiveness || EnablePatchPointLiveness)
+    addPass(&StackMapLivenessID);
 }
 
 /// Add passes that optimize machine instructions in SSA form.
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 1eb590b55f..e58856f024 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6886,6 +6886,9 @@ void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
   DAG.ReplaceAllUsesWith(Call, MN);
 
   DAG.DeleteNode(Call);
+
+  // Inform the Frame Information that we have a stackmap in this function.
+  FuncInfo.MF->getFrameInfo()->setHasStackMap();
 }
 
 /// \brief Lower llvm.experimental.patchpoint directly to its target opcode.
@@ -7025,6 +7028,9 @@ void SelectionDAGBuilder::visitPatchpoint(const CallInst &CI) {
   } else
     DAG.ReplaceAllUsesWith(Call, MN);
   DAG.DeleteNode(Call);
+
+  // Inform the Frame Information that we have a patchpoint in this function.
+  FuncInfo.MF->getFrameInfo()->setHasPatchPoint();
 }
 
 /// TargetLowering::LowerCallTo - This is the default LowerCallTo
diff --git a/lib/CodeGen/StackMapLivenessAnalysis.cpp b/lib/CodeGen/StackMapLivenessAnalysis.cpp
new file mode 100644
index 0000000000..a374417a90
--- /dev/null
+++ b/lib/CodeGen/StackMapLivenessAnalysis.cpp
@@ -0,0 +1,128 @@
+//===-- StackMapLivenessAnalysis.cpp - StackMap live Out Analysis ----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the StackMap Liveness analysis pass. The pass calculates
+// the liveness for each basic block in a function and attaches the register
+// live-out information to a stackmap or patchpoint intrinsic if present.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "stackmaps"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFunctionAnalysis.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/StackMapLivenessAnalysis.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+
+
+using namespace llvm;
+
+namespace llvm {
+cl::opt<bool> EnableStackMapLiveness("enable-stackmap-liveness",
+  cl::Hidden, cl::desc("Enable StackMap Liveness Analysis Pass"));
+cl::opt<bool> EnablePatchPointLiveness("enable-patchpoint-liveness",
+  cl::Hidden, cl::desc("Enable PatchPoint Liveness Analysis Pass"));
+}
+
+STATISTIC(NumStackMapFuncVisited, "Number of functions visited");
+STATISTIC(NumStackMapFuncSkipped, "Number of functions skipped");
+STATISTIC(NumBBsVisited,          "Number of basic blocks visited");
+STATISTIC(NumBBsHaveNoStackmap,   "Number of basic blocks with no stackmap");
+STATISTIC(NumStackMaps,           "Number of StackMaps visited");
+
+char StackMapLiveness::ID = 0;
+char &llvm::StackMapLivenessID = StackMapLiveness::ID;
+INITIALIZE_PASS(StackMapLiveness, "stackmap-liveness",
+                "StackMap Liveness Analysis", false, false)
+
+/// Default construct and initialize the pass.
+StackMapLiveness::StackMapLiveness() : MachineFunctionPass(ID) {
+  initializeStackMapLivenessPass(*PassRegistry::getPassRegistry());
+}
+
+/// Tell the pass manager which passes we depend on and what information we
+/// preserve.
+void StackMapLiveness::getAnalysisUsage(AnalysisUsage &AU) const {
+  // We preserve all information.
+  AU.setPreservesAll();
+  AU.setPreservesCFG();
+  // Default dependencie for all MachineFunction passes.
+  AU.addRequired<MachineFunctionAnalysis>();
+}
+
+/// Calculate the liveness information for the given machine function.
+bool StackMapLiveness::runOnMachineFunction(MachineFunction &_MF) {
+  DEBUG(dbgs() << "********** COMPUTING STACKMAP LIVENESS: "
+               << _MF.getName() << " **********\n");
+  MF = &_MF;
+  TRI = MF->getTarget().getRegisterInfo();
+  ++NumStackMapFuncVisited;
+
+  // Skip this function if there are no stackmaps or patchpoints to process.
+  if (!((MF->getFrameInfo()->hasStackMap() && EnableStackMapLiveness) ||
+        (MF->getFrameInfo()->hasPatchPoint() && EnablePatchPointLiveness))) {
+    ++NumStackMapFuncSkipped;
+    return false;
+  }
+  return calculateLiveness();
+}
+
+/// Performs the actual liveness calculation for the function.
+bool StackMapLiveness::calculateLiveness() {
+  bool HasChanged = false;
+  // For all basic blocks in the function.
+  for (MachineFunction::iterator MBBI = MF->begin(), MBBE = MF->end();
+       MBBI != MBBE; ++MBBI) {
+    DEBUG(dbgs() << "****** BB " << MBBI->getName() << " ******\n");
+    LiveRegs.init(TRI);
+    LiveRegs.addLiveOuts(MBBI);
+    bool HasStackMap = false;
+    // Reverse iterate over all instructions and add the current live register
+    // set to an instruction if we encounter a stackmap or patchpoint
+    // instruction.
+    for (MachineBasicBlock::reverse_iterator I = MBBI->rbegin(),
+         E = MBBI->rend(); I != E; ++I) {
+      int Opc = I->getOpcode();
+      if ((EnableStackMapLiveness && (Opc == TargetOpcode::STACKMAP)) ||
+          (EnablePatchPointLiveness && (Opc == TargetOpcode::PATCHPOINT))) {
+        addLiveOutSetToMI(*I);
+        HasChanged = true;
+        HasStackMap = true;
+        ++NumStackMaps;
+      }
+      DEBUG(dbgs() << "   " << *I << "   " << LiveRegs);
+      LiveRegs.stepBackward(*I);
+    }
+    ++NumBBsVisited;
+    if (!HasStackMap)
+      ++NumBBsHaveNoStackmap;
+  }
+  return HasChanged;
+}
+
+/// Add the current register live set to the instruction.
+void StackMapLiveness::addLiveOutSetToMI(MachineInstr &MI) {
+  uint32_t *Mask = createRegisterMask();
+  MachineOperand MO = MachineOperand::CreateRegLiveOut(Mask);
+  MI.addOperand(*MF, MO);
+}
+
+/// Create a register mask and initialize it with the registers from the
+/// register live set.
+uint32_t *StackMapLiveness::createRegisterMask() const {
+  // The mask is owned and cleaned up by the Machine Function.
+  uint32_t *Mask = MF->allocateRegisterMask(TRI->getNumRegs());
+  for (LivePhysRegs::const_iterator RI = LiveRegs.begin(), RE = LiveRegs.end();
+       RI != RE; ++RI)
+    Mask[*RI / 32] |= 1U << (*RI % 32);
+  return Mask;
+}
diff --git a/lib/CodeGen/StackMaps.cpp b/lib/CodeGen/StackMaps.cpp
index 29ff0475a4..8f6972d60f 100644
--- a/lib/CodeGen/StackMaps.cpp
+++ b/lib/CodeGen/StackMaps.cpp
@@ -68,10 +68,10 @@ unsigned PatchPointOpers::getNextScratchIdx(unsigned StartIdx) const {
 
 std::pair<StackMaps::Location, MachineInstr::const_mop_iterator>
 StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI,
-                        MachineInstr::const_mop_iterator MOE) {
+                        MachineInstr::const_mop_iterator MOE) const {
   const MachineOperand &MOP = *MOI;
-  assert(!MOP.isRegMask() && (!MOP.isReg() || !MOP.isImplicit()) &&
-         "Register mask and implicit operands should not be processed.");
+  assert((!MOP.isReg() || !MOP.isImplicit()) &&
+         "Implicit operands should not be processed.");
 
   if (MOP.isImm()) {
     // Verify anyregcc
@@ -106,6 +106,9 @@ StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI,
     }
   }
 
+  if (MOP.isRegMask() || MOP.isRegLiveOut())
+    return std::make_pair(Location(), ++MOI);
+
   // Otherwise this is a reg operand. The physical register number will
   // ultimately be encoded as a DWARF regno. The stack map also records the size
   // of a spill slot that can hold the register content. (The runtime can
@@ -120,6 +123,65 @@ StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI,
     Location(Location::Register, RC->getSize(), MOP.getReg(), 0), ++MOI);
 }
 
+/// Go up the super-register chain until we hit a valid dwarf register number.
+static unsigned short getDwarfRegNum(unsigned Reg, const MCRegisterInfo &MCRI,
+                                     const TargetRegisterInfo *TRI) {
+  int RegNo = MCRI.getDwarfRegNum(Reg, false);
+  for (MCSuperRegIterator SR(Reg, TRI);
+       SR.isValid() && RegNo < 0; ++SR)
+    RegNo = TRI->getDwarfRegNum(*SR, false);
+
+  assert(RegNo >= 0 && "Invalid Dwarf register number.");
+  return (unsigned short) RegNo;
+}
+
+/// Create a live-out register record for the given register Reg.
+StackMaps::LiveOutReg
+StackMaps::createLiveOutReg(unsigned Reg, const MCRegisterInfo &MCRI,
+                            const TargetRegisterInfo *TRI) const {
+  unsigned RegNo = getDwarfRegNum(Reg, MCRI, TRI);
+  unsigned Size = TRI->getMinimalPhysRegClass(Reg)->getSize();
+  return LiveOutReg(Reg, RegNo, Size);
+}
+
+/// Parse the register live-out mask and return a vector of live-out registers
+/// that need to be recorded in the stackmap.
+StackMaps::LiveOutVec
+StackMaps::parseRegisterLiveOutMask(const uint32_t *Mask) const {
+  assert(Mask && "No register mask specified");
+  const TargetRegisterInfo *TRI = AP.TM.getRegisterInfo();
+  MCContext &OutContext = AP.OutStreamer.getContext();
+  const MCRegisterInfo &MCRI = *OutContext.getRegisterInfo();
+  LiveOutVec LiveOuts;
+
+  // Create a LiveOutReg for each bit that is set in the register mask.
+  for (unsigned Reg = 0, NumRegs = TRI->getNumRegs(); Reg != NumRegs; ++Reg)
+    if ((Mask[Reg / 32] >> Reg % 32) & 1)
+      LiveOuts.push_back(createLiveOutReg(Reg, MCRI, TRI));
+
+  // We don't need to keep track of a register if its super-register is already
+  // in the list. Merge entries that refer to the same dwarf register and use
+  // the maximum size that needs to be spilled.
+  std::sort(LiveOuts.begin(), LiveOuts.end());
+  for (LiveOutVec::iterator I = LiveOuts.begin(), E = LiveOuts.end();
+       I != E; ++I) {
+    for (LiveOutVec::iterator II = next(I); II != E; ++II) {
+      if (I->RegNo != II->RegNo) {
+        // Skip all the now invalid entries.
+        I = --II;
+        break;
+      }
+      I->Size = std::max(I->Size, II->Size);
+      if (TRI->isSuperRegister(I->Reg, II->Reg))
+        I->Reg = II->Reg;
+      II->MarkInvalid();
+    }
+  }
+  LiveOuts.erase(std::remove_if(LiveOuts.begin(), LiveOuts.end(),
+                                LiveOutReg::IsInvalid), LiveOuts.end());
+  return LiveOuts;
+}
+
 void StackMaps::recordStackMapOpers(const MachineInstr &MI, uint64_t ID,
                                     MachineInstr::const_mop_iterator MOI,
                                     MachineInstr::const_mop_iterator MOE,
@@ -129,7 +191,8 @@ void StackMaps::recordStackMapOpers(const MachineInstr &MI, uint64_t ID,
   MCSymbol *MILabel = OutContext.CreateTempSymbol();
   AP.OutStreamer.EmitLabel(MILabel);
 
-  LocationVec CallsiteLocs;
+  LocationVec Locations;
+  LiveOutVec LiveOuts;
 
   if (recordResult) {
     std::pair<Location, MachineInstr::const_mop_iterator> ParseResult =
@@ -138,7 +201,7 @@ void StackMaps::recordStackMapOpers(const MachineInstr &MI, uint64_t ID,
     Location &Loc = ParseResult.first;
     assert(Loc.LocType == Location::Register &&
            "Stackmap return location must be a register.");
-    CallsiteLocs.push_back(Loc);
+    Locations.push_back(Loc);
   }
 
   while (MOI != MOE) {
@@ -151,7 +214,9 @@ void StackMaps::recordStackMapOpers(const MachineInstr &MI, uint64_t ID,
       Loc.Offset = ConstPool.getConstantIndex(Loc.Offset);
     }
 
-    CallsiteLocs.push_back(Loc);
+    // Skip the register mask and register live-out mask
+    if (Loc.LocType != Location::Unprocessed)
+      Locations.push_back(Loc);
   }
 
   const MCExpr *CSOffsetExpr = MCBinaryExpr::CreateSub(
@@ -159,21 +224,23 @@ void StackMaps::recordStackMapOpers(const MachineInstr &MI, uint64_t ID,
     MCSymbolRefExpr::Create(AP.CurrentFnSym, OutContext),
     OutContext);
 
-  CSInfos.push_back(CallsiteInfo(CSOffsetExpr, ID, CallsiteLocs));
+  if (MOI->isRegLiveOut())
+    LiveOuts = parseRegisterLiveOutMask(MOI->getRegLiveOut());
+
+  CSInfos.push_back(CallsiteInfo(CSOffsetExpr, ID, Locations, LiveOuts));
 }
 
 static MachineInstr::const_mop_iterator
 getStackMapEndMOP(MachineInstr::const_mop_iterator MOI,
                   MachineInstr::const_mop_iterator MOE) {
   for (; MOI != MOE; ++MOI)
-    if (MOI->isRegMask() || (MOI->isReg() && MOI->isImplicit()))
+    if (MOI->isRegLiveOut() || (MOI->isReg() && MOI->isImplicit()))
       break;
-
   return MOI;
 }
 
 void StackMaps::recordStackMap(const MachineInstr &MI) {
-  assert(MI.getOpcode() == TargetOpcode::STACKMAP && "exected stackmap");
+  assert(MI.getOpcode() == TargetOpcode::STACKMAP && "expected stackmap");
 
   int64_t ID = MI.getOperand(0).getImm();
   recordStackMapOpers(MI, ID, llvm::next(MI.operands_begin(), 2),
@@ -182,7 +249,7 @@ void StackMaps::recordStackMap(const MachineInstr &MI) {
 }
 
 void StackMaps::recordPatchPoint(const MachineInstr &MI) {
-  assert(MI.getOpcode() == TargetOpcode::PATCHPOINT && "exected stackmap");
+  assert(MI.getOpcode() == TargetOpcode::PATCHPOINT && "expected patchpoint");
 
   PatchPointOpers opers(&MI);
   int64_t ID = opers.getMetaOper(PatchPointOpers::IDPos).getImm();
@@ -221,6 +288,11 @@ void StackMaps::recordPatchPoint(const MachineInstr &MI) {
 ///     uint16 : Dwarf RegNum
 ///     int32  : Offset
 ///   }
+///   uint16 : NumLiveOuts
+///   LiveOuts[NumLiveOuts]
+///     uint16 : Dwarf RegNum
+///     uint8  : Reserved
+///     uint8  : Size in Bytes
 /// }
 ///
 /// Location Encoding, Type, Value:
@@ -273,6 +345,7 @@ void StackMaps::serializeToStackMapSection() {
 
     uint64_t CallsiteID = CSII->ID;
     const LocationVec &CSLocs = CSII->Locations;
+    const LiveOutVec &LiveOuts = CSII->LiveOuts;
 
     DEBUG(dbgs() << WSMP << "callsite " << CallsiteID << "\n");
 
@@ -280,11 +353,12 @@ void StackMaps::serializeToStackMapSection() {
     // runtime than crash in case of in-process compilation. Currently, we do
     // simple overflow checks, but we may eventually communicate other
     // compilation errors this way.
-    if (CSLocs.size() > UINT16_MAX) {
-      AP.OutStreamer.EmitIntValue(UINT32_MAX, 8); // Invalid ID.
+    if (CSLocs.size() > UINT16_MAX || LiveOuts.size() > UINT16_MAX) {
+      AP.OutStreamer.EmitIntValue(UINT64_MAX, 8); // Invalid ID.
       AP.OutStreamer.EmitValue(CSII->CSOffsetExpr, 4);
       AP.OutStreamer.EmitIntValue(0, 2); // Reserved.
       AP.OutStreamer.EmitIntValue(0, 2); // 0 locations.
+      AP.OutStreamer.EmitIntValue(0, 2); // 0 live-out registers.
       continue;
     }
 
@@ -361,6 +435,24 @@ void StackMaps::serializeToStackMapSection() {
       AP.OutStreamer.EmitIntValue(RegNo, 2);
       AP.OutStreamer.EmitIntValue(Offset, 4);
     }
+
+    DEBUG(dbgs() << WSMP << "  has " << LiveOuts.size()
+                 << " live-out registers\n");
+
+    AP.OutStreamer.EmitIntValue(LiveOuts.size(), 2);
+
+    operIdx = 0;
+    for (LiveOutVec::const_iterator LI = LiveOuts.begin(), LE = LiveOuts.end();
+         LI != LE; ++LI, ++operIdx) {
+      DEBUG(dbgs() << WSMP << "  LO " << operIdx << ": "
+                   << MCRI.getName(LI->Reg)
+                   << "     [encoding: .short " << LI->RegNo
+                   << ", .byte 0, .byte " << LI->Size << "]\n");
+
+      AP.OutStreamer.EmitIntValue(LI->RegNo, 2);
+      AP.OutStreamer.EmitIntValue(0, 1);
+      AP.OutStreamer.EmitIntValue(LI->Size, 1);
+    }
   }
 
   AP.OutStreamer.AddBlankLine();
diff --git a/test/CodeGen/X86/stackmap-liveness.ll b/test/CodeGen/X86/stackmap-liveness.ll
new file mode 100644
index 0000000000..b1a931dd7c
--- /dev/null
+++ b/test/CodeGen/X86/stackmap-liveness.ll
@@ -0,0 +1,178 @@
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7-avx -disable-fp-elim | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7-avx -disable-fp-elim -enable-stackmap-liveness| FileCheck -check-prefix=STACK %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7-avx -disable-fp-elim -enable-patchpoint-liveness| FileCheck -check-prefix=PATCH %s
+;
+; Note: Print verbose stackmaps using -debug-only=stackmaps.
+
+; CHECK-LABEL:  .section  __LLVM_STACKMAPS,__llvm_stackmaps
+; CHECK-NEXT:   __LLVM_StackMaps:
+; CHECK-NEXT:   .long   0
+; Num LargeConstants
+; CHECK-NEXT:   .long   0
+; Num Callsites
+; CHECK-NEXT:   .long   5
+define void @stackmap_liveness() {
+entry:
+  %a1 = call <2 x double> asm sideeffect "", "={xmm2}"() nounwind
+; StackMap 1 (no liveness information available)
+; CHECK-LABEL:  .long L{{.*}}-_stackmap_liveness
+; CHECK-NEXT:   .short  0
+; CHECK-NEXT:   .short  0
+; Num LiveOut Entries: 0
+; CHECK-NEXT:   .short  0
+
+; StackMap 1 (stackmap liveness information enabled)
+; STACK-LABEL:  .long L{{.*}}-_stackmap_liveness
+; STACK-NEXT:   .short  0
+; STACK-NEXT:   .short  0
+; Num LiveOut Entries: 2
+; STACK-NEXT:   .short  2
+; LiveOut Entry 1: %RSP (8 bytes)
+; STACK-NEXT:   .short  7
+; STACK-NEXT:   .byte 0
+; STACK-NEXT:   .byte 8
+; LiveOut Entry 2: %YMM2 (16 bytes) --> %XMM2
+; STACK-NEXT:   .short  19
+; STACK-NEXT:   .byte 0
+; STACK-NEXT:   .byte 16
+
+; StackMap 1 (patchpoint liveness information enabled)
+; PATCH-LABEL:  .long L{{.*}}-_stackmap_liveness
+; PATCH-NEXT:   .short  0
+; PATCH-NEXT:   .short  0
+; Num LiveOut Entries: 0
+; PATCH-NEXT:   .short  0
+  call void (i64, i32, ...)* @llvm.experimental.stackmap(i64 1, i32 5)
+  %a2 = call i64 asm sideeffect "", "={r8}"() nounwind
+  %a3 = call i8 asm sideeffect "", "={ah}"() nounwind
+  %a4 = call <4 x double> asm sideeffect "", "={ymm0}"() nounwind
+  %a5 = call <4 x double> asm sideeffect "", "={ymm1}"() nounwind
+
+; StackMap 2 (no liveness information available)
+; CHECK-LABEL:  .long L{{.*}}-_stackmap_liveness
+; CHECK-NEXT:   .short  0
+; CHECK-NEXT:   .short  0
+; Num LiveOut Entries: 0
+; CHECK-NEXT:   .short  0
+
+; StackMap 2 (stackmap liveness information enabled)
+; STACK-LABEL:  .long L{{.*}}-_stackmap_liveness
+; STACK-NEXT:   .short  0
+; STACK-NEXT:   .short  0
+; Num LiveOut Entries: 6
+; STACK-NEXT:   .short  6
+; LiveOut Entry 2: %RAX (1 bytes) --> %AL or %AH
+; STACK-NEXT:   .short  0
+; STACK-NEXT:   .byte 0
+; STACK-NEXT:   .byte 1
+; LiveOut Entry 2: %RSP (8 bytes)
+; STACK-NEXT:   .short  7
+; STACK-NEXT:   .byte 0
+; STACK-NEXT:   .byte 8
+; LiveOut Entry 2: %R8 (8 bytes)
+; STACK-NEXT:   .short  8
+; STACK-NEXT:   .byte 0
+; STACK-NEXT:   .byte 8
+; LiveOut Entry 2: %YMM0 (32 bytes)
+; STACK-NEXT:   .short  17
+; STACK-NEXT:   .byte 0
+; STACK-NEXT:   .byte 32
+; LiveOut Entry 2: %YMM1 (32 bytes)
+; STACK-NEXT:   .short  18
+; STACK-NEXT:   .byte 0
+; STACK-NEXT:   .byte 32
+; LiveOut Entry 2: %YMM2 (16 bytes) --> %XMM2
+; STACK-NEXT:   .short  19
+; STACK-NEXT:   .byte 0
+; STACK-NEXT:   .byte 16
+
+; StackMap 2 (patchpoint liveness information enabled)
+; PATCH-LABEL:  .long L{{.*}}-_stackmap_liveness
+; PATCH-NEXT:   .short  0
+; PATCH-NEXT:   .short  0
+; Num LiveOut Entries: 0
+; PATCH-NEXT:   .short  0
+  call void (i64, i32, ...)* @llvm.experimental.stackmap(i64 2, i32 5)
+  call void asm sideeffect "", "{r8},{ah},{ymm0},{ymm1}"(i64 %a2, i8 %a3, <4 x double> %a4, <4 x double> %a5) nounwind
+
+; StackMap 3 (no liveness information available)
+; CHECK-LABEL:  .long L{{.*}}-_stackmap_liveness
+; CHECK-NEXT:   .short  0
+; CHECK-NEXT:   .short  0
+; Num LiveOut Entries: 0
+; CHECK-NEXT:   .short  0
+
+; StackMap 3 (stackmap liveness information enabled)
+; STACK-LABEL:  .long L{{.*}}-_stackmap_liveness
+; STACK-NEXT:   .short  0
+; STACK-NEXT:   .short  0
+; Num LiveOut Entries: 2
+; STACK-NEXT:   .short  2
+; LiveOut Entry 2: %RSP (8 bytes)
+; STACK-NEXT:   .short  7
+; STACK-NEXT:   .byte 0
+; STACK-NEXT:   .byte 8
+; LiveOut Entry 2: %YMM2 (16 bytes) --> %XMM2
+; STACK-NEXT:   .short  19
+; STACK-NEXT:   .byte 0
+; STACK-NEXT:   .byte 16
+
+; StackMap 3 (patchpoint liveness information enabled)
+; PATCH-LABEL:  .long L{{.*}}-_stackmap_liveness
+; PATCH-NEXT:   .short  0
+; PATCH-NEXT:   .short  0
+; Num LiveOut Entries: 0
+; PATCH-NEXT:   .short  0
+  call void (i64, i32, ...)* @llvm.experimental.stackmap(i64 3, i32 5)
+  call void asm sideeffect "", "{xmm2}"(<2 x double> %a1) nounwind
+  ret void
+}
+
+define void @mixed_liveness() {
+entry:
+  %a1 = call <2 x double> asm sideeffect "", "={xmm2}"() nounwind
+; StackMap 4 (stackmap liveness information enabled)
+; STACK-LABEL:  .long L{{.*}}-_mixed_liveness
+; STACK-NEXT:   .short  0
+; STACK-NEXT:   .short  0
+; Num LiveOut Entries: 1
+; STACK-NEXT:   .short  1
+; LiveOut Entry 1: %YMM2 (16 bytes) --> %XMM2
+; STACK-NEXT:   .short  19
+; STACK-NEXT:   .byte 0
+; STACK-NEXT:   .byte 16
+; StackMap 5 (stackmap liveness information enabled)
+; STACK-LABEL:  .long L{{.*}}-_mixed_liveness
+; STACK-NEXT:   .short  0
+; STACK-NEXT:   .short  0
+; Num LiveOut Entries: 0
+; STACK-NEXT:   .short  0
+
+; StackMap 4 (patchpoint liveness information enabled)
+; PATCH-LABEL:  .long L{{.*}}-_mixed_liveness
+; PATCH-NEXT:   .short  0
+; PATCH-NEXT:   .short  0
+; Num LiveOut Entries: 0
+; PATCH-NEXT:   .short  0
+; StackMap 5 (patchpoint liveness information enabled)
+; PATCH-LABEL:  .long L{{.*}}-_mixed_liveness
+; PATCH-NEXT:   .short  0
+; PATCH-NEXT:   .short  0
+; Num LiveOut Entries: 2
+; PATCH-NEXT:   .short  2
+; LiveOut Entry 1: %RSP (8 bytes)
+; PATCH-NEXT:   .short  7
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 1: %YMM2 (16 bytes) --> %XMM2
+; PATCH-NEXT:   .short  19
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 16
+  call void (i64, i32, ...)* @llvm.experimental.stackmap(i64 4, i32 5)
+  call anyregcc void (i64, i32, i8*, i32, ...)* @llvm.experimental.patchpoint.void(i64 5, i32 0, i8* null, i32 0)
+  call void asm sideeffect "", "{xmm2}"(<2 x double> %a1) nounwind
+  ret void
+}
+
+declare void @llvm.experimental.stackmap(i64, i32, ...)
+declare void @llvm.experimental.patchpoint.void(i64, i32, i8*, i32, ...)
diff --git a/test/CodeGen/X86/stackmap.ll b/test/CodeGen/X86/stackmap.ll
index e0cb7589c4..e4194adfaa 100644
--- a/test/CodeGen/X86/stackmap.ll
+++ b/test/CodeGen/X86/stackmap.ll
@@ -183,15 +183,15 @@ entry:
 ;
 ; Verify 17 stack map entries.
 ;
-; CHECK-LABEL:.long L{{.*}}-_spilledValue
-; CHECK-NEXT: .short 0
-; CHECK-NEXT: .short 17
+; CHECK-LABEL:  .long L{{.*}}-_spilledValue
+; CHECK-NEXT:   .short 0
+; CHECK-NEXT:   .short 17
 ;
 ; Check that at least one is a spilled entry from RBP.
 ; Location: Indirect RBP + ...
-; CHECK:      .byte 3
-; CHECK-NEXT: .byte 8
-; CHECK-NEXT: .short 6
+; CHECK:        .byte 3
+; CHECK-NEXT:   .byte 8
+; CHECK-NEXT:   .short 6
 define void @spilledValue(i64 %arg0, i64 %arg1, i64 %arg2, i64 %arg3, i64 %arg4, i64 %l0, i64 %l1, i64 %l2, i64 %l3, i64 %l4, i64 %l5, i64 %l6, i64 %l7, i64 %l8, i64 %l9, i64 %l10, i64 %l11, i64 %l12, i64 %l13, i64 %l14, i64 %l15, i64 %l16) {
 entry:
   call void (i64, i32, i8*, i32, ...)* @llvm.experimental.patchpoint.void(i64 11, i32 15, i8* null, i32 5, i64 %arg0, i64 %arg1, i64 %arg2, i64 %arg3, i64 %arg4, i64 %l0, i64 %l1, i64 %l2, i64 %l3, i64 %l4, i64 %l5, i64 %l6, i64 %l7, i64 %l8, i64 %l9, i64 %l10, i64 %l11, i64 %l12, i64 %l13, i64 %l14, i64 %l15, i64 %l16)
@@ -202,15 +202,15 @@ entry:
 ;
 ; Verify 17 stack map entries.
 ;
-; CHECK-LABEL: .long L{{.*}}-_spilledStackMapValue
-; CHECK-NEXT:  .short 0
-; CHECK-NEXT:  .short 17
+; CHECK-LABEL:  .long L{{.*}}-_spilledStackMapValue
+; CHECK-NEXT:   .short 0
+; CHECK-NEXT:   .short 17
 ;
 ; Check that at least one is a spilled entry from RBP.
 ; Location: Indirect RBP + ...
-; CHECK:      .byte 3
-; CHECK-NEXT: .byte 8
-; CHECK-NEXT: .short 6
+; CHECK:        .byte 3
+; CHECK-NEXT:   .byte 8
+; CHECK-NEXT:   .short 6
 define webkit_jscc void @spilledStackMapValue(i64 %l0, i64 %l1, i64 %l2, i64 %l3, i64 %l4, i64 %l5, i64 %l6, i64 %l7, i64 %l8, i64 %l9, i64 %l10, i64 %l11, i64 %l12, i64 %l13, i64 %l14, i64 %l15, i64 %l16) {
 entry:
   call void (i64, i32, ...)* @llvm.experimental.stackmap(i64 12, i32 15, i64 %l0, i64 %l1, i64 %l2, i64 %l3, i64 %l4, i64 %l5, i64 %l6, i64 %l7, i64 %l8, i64 %l9, i64 %l10, i64 %l11, i64 %l12, i64 %l13, i64 %l14, i64 %l15, i64 %l16)
@@ -219,16 +219,16 @@ entry:
 
 ; Spill a subregister stackmap operand.
 ;
-; CHECK-LABEL: .long L{{.*}}-_spillSubReg
-; CHECK-NEXT:  .short 0
+; CHECK-LABEL:  .long L{{.*}}-_spillSubReg
+; CHECK-NEXT:   .short 0
 ; 4 locations
-; CHECK-NEXT:  .short 1
+; CHECK-NEXT:   .short 1
 ;
 ; Check that the subregister operand is a 4-byte spill.
 ; Location: Indirect, 4-byte, RBP + ...
-; CHECK:      .byte 3
-; CHECK-NEXT: .byte 4
-; CHECK-NEXT: .short 6
+; CHECK:        .byte 3
+; CHECK-NEXT:   .byte 4
+; CHECK-NEXT:   .short 6
 define void @spillSubReg(i64 %arg) #0 {
 bb:
   br i1 undef, label %bb1, label %bb2
@@ -259,23 +259,23 @@ bb61:
 ; Map a single byte subregister. There is no DWARF register number, so
 ; we expect the register to be encoded with the proper size and spill offset. We don't know which
 ;
-; CHECK-LABEL: .long L{{.*}}-_subRegOffset
-; CHECK-NEXT:  .short 0
+; CHECK-LABEL:  .long L{{.*}}-_subRegOffset
+; CHECK-NEXT:   .short 0
 ; 2 locations
-; CHECK-NEXT:  .short 2
+; CHECK-NEXT:   .short 2
 ;
 ; Check that the subregister operands are 1-byte spills.
 ; Location 0: Register, 4-byte, AL
-; CHECK-NEXT: .byte 1
-; CHECK-NEXT: .byte 1
-; CHECK-NEXT: .short 0
-; CHECK-NEXT: .long 0
+; CHECK-NEXT:   .byte 1
+; CHECK-NEXT:   .byte 1
+; CHECK-NEXT:   .short 0
+; CHECK-NEXT:   .long 0
 ;
 ; Location 1: Register, 4-byte, BL
-; CHECK-NEXT: .byte 1
-; CHECK-NEXT: .byte 1
-; CHECK-NEXT: .short 3
-; CHECK-NEXT: .long 0
+; CHECK-NEXT:   .byte 1
+; CHECK-NEXT:   .byte 1
+; CHECK-NEXT:   .short 3
+; CHECK-NEXT:   .long 0
 define void @subRegOffset(i16 %arg) {
   %v = mul i16 %arg, 5
   %a0 = trunc i16 %v to i8
@@ -289,10 +289,10 @@ define void @subRegOffset(i16 %arg) {
 
 ; Map a constant value.
 ;
-; CHECK-LABEL: .long L{{.*}}-_liveConstant
-; CHECK-NEXT:  .short 0
+; CHECK-LABEL:  .long L{{.*}}-_liveConstant
+; CHECK-NEXT:   .short 0
 ; 1 location
-; CHECK-NEXT:  .short 1
+; CHECK-NEXT:   .short 1
 ; Loc 0: SmallConstant
 ; CHECK-NEXT:   .byte   4
 ; CHECK-NEXT:   .byte   8
@@ -316,9 +316,9 @@ define void @liveConstant() {
 ; CHECK-NEXT:   .byte	8
 ; CHECK-NEXT:   .short	6
 ; CHECK-NEXT:   .long
-; CHECK-NEXT:   .quad
+
 ; Callsite 17
-; CHECK-NEXT:   .long	L{{.*}}-_directFrameIdx
+; CHECK-LABEL:  .long	L{{.*}}-_directFrameIdx
 ; CHECK-NEXT:   .short	0
 ; 2 locations
 ; CHECK-NEXT:   .short	2