[SystemZ] Be more careful about inverting CC masks

System z branches have a mask to select which of the 4 CC values should
cause the branch to be taken.  We can invert a branch by inverting the mask.
However, not all instructions can produce all 4 CC values, so inverting
the branch like this can lead to some oddities.  For example, integer
comparisons only produce a CC of 0 (equal), 1 (less) or 2 (greater).
If an integer EQ is reversed to NE before instruction selection,
the branch will test for 1 or 2.  If instead the branch is reversed
after instruction selection (by inverting the mask), it will test for
1, 2 or 3.  Both are correct, but the second isn't really canonical.
This patch therefore keeps track of which CC values are possible
and uses this when inverting a mask.

Although this is mostly cosmestic, it fixes undefined behavior
for the CIJNLH in branch-08.ll.  Another fix would have been
to mask out bit 0 when generating the fused compare and branch,
but the point of this patch is that we shouldn't need to do that
in the first place.

The patch also makes it easier to reuse CC results from other instructions.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@187495 91177308-0d34-0410-b5e6-96231b3b80d8

56 files changed, 286 insertions, 205 deletions

diff --git a/lib/Target/SystemZ/SystemZ.h b/lib/Target/SystemZ/SystemZ.h
index 24612bbce8..4c1e81a082 100644
--- a/lib/Target/SystemZ/SystemZ.h
+++ b/lib/Target/SystemZ/SystemZ.h
@@ -30,16 +30,28 @@ namespace llvm {
     const unsigned CCMASK_3 = 1 << 0;
     const unsigned CCMASK_ANY = CCMASK_0 | CCMASK_1 | CCMASK_2 | CCMASK_3;
 
-    // Condition-code mask assignments for floating-point comparisons.
+    // Condition-code mask assignments for integer and floating-point
+    // comparisons.
     const unsigned CCMASK_CMP_EQ = CCMASK_0;
     const unsigned CCMASK_CMP_LT = CCMASK_1;
     const unsigned CCMASK_CMP_GT = CCMASK_2;
-    const unsigned CCMASK_CMP_UO = CCMASK_3;
     const unsigned CCMASK_CMP_NE = CCMASK_CMP_LT | CCMASK_CMP_GT;
     const unsigned CCMASK_CMP_LE = CCMASK_CMP_EQ | CCMASK_CMP_LT;
     const unsigned CCMASK_CMP_GE = CCMASK_CMP_EQ | CCMASK_CMP_GT;
+
+    // Condition-code mask assignments for floating-point comparisons only.
+    const unsigned CCMASK_CMP_UO = CCMASK_3;
     const unsigned CCMASK_CMP_O  = CCMASK_ANY ^ CCMASK_CMP_UO;
 
+    // All condition-code values produced by comparisons.
+    const unsigned CCMASK_ICMP = CCMASK_0 | CCMASK_1 | CCMASK_2;
+    const unsigned CCMASK_FCMP = CCMASK_0 | CCMASK_1 | CCMASK_2 | CCMASK_3;
+
+    // Condition-code mask assignments for CS.
+    const unsigned CCMASK_CS_EQ = CCMASK_0;
+    const unsigned CCMASK_CS_NE = CCMASK_1;
+    const unsigned CCMASK_CS    = CCMASK_0 | CCMASK_1;
+
     // Return true if Val fits an LLILL operand.
     static inline bool isImmLL(uint64_t Val) {
       return (Val & ~0x000000000000ffffULL) == 0;
diff --git a/lib/Target/SystemZ/SystemZISelLowering.cpp b/lib/Target/SystemZ/SystemZISelLowering.cpp
index d7cd5256e0..34697fed80 100644
--- a/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -1015,15 +1015,21 @@ static bool preferUnsignedComparison(SelectionDAG &DAG, SDValue CmpOp0,
   return false;
 }
 
-// Return a target node that compares CmpOp0 and CmpOp1.  Set CCMask to the
-// 4-bit condition-code mask for CC.
+// Return a target node that compares CmpOp0 with CmpOp1 and stores a
+// 2-bit result in CC.  Set CCValid to the CCMASK_* of all possible
+// 2-bit results and CCMask to the subset of those results that are
+// associated with Cond.
 static SDValue emitCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
-                       ISD::CondCode CC, unsigned &CCMask) {
+                       ISD::CondCode Cond, unsigned &CCValid,
+                       unsigned &CCMask) {
   bool IsUnsigned = false;
-  CCMask = CCMaskForCondCode(CC);
-  if (!CmpOp0.getValueType().isFloatingPoint()) {
+  CCMask = CCMaskForCondCode(Cond);
+  if (CmpOp0.getValueType().isFloatingPoint())
+    CCValid = SystemZ::CCMASK_FCMP;
+  else {
     IsUnsigned = CCMask & SystemZ::CCMASK_CMP_UO;
-    CCMask &= ~SystemZ::CCMASK_CMP_UO;
+    CCValid = SystemZ::CCMASK_ICMP;
+    CCMask &= CCValid;
     adjustSubwordCmp(DAG, IsUnsigned, CmpOp0, CmpOp1, CCMask);
     if (preferUnsignedComparison(DAG, CmpOp0, CmpOp1, CCMask))
       IsUnsigned = true;
@@ -1065,10 +1071,11 @@ SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
   SDValue Dest     = Op.getOperand(4);
   SDLoc DL(Op);
 
-  unsigned CCMask;
-  SDValue Flags = emitCmp(DAG, CmpOp0, CmpOp1, CC, CCMask);
+  unsigned CCValid, CCMask;
+  SDValue Flags = emitCmp(DAG, CmpOp0, CmpOp1, CC, CCValid, CCMask);
   return DAG.getNode(SystemZISD::BR_CCMASK, DL, Op.getValueType(),
-                     Chain, DAG.getConstant(CCMask, MVT::i32), Dest, Flags);
+                     Chain, DAG.getConstant(CCValid, MVT::i32),
+                     DAG.getConstant(CCMask, MVT::i32), Dest, Flags);
 }
 
 SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
@@ -1080,12 +1087,13 @@ SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
   SDLoc DL(Op);
 
-  unsigned CCMask;
-  SDValue Flags = emitCmp(DAG, CmpOp0, CmpOp1, CC, CCMask);
+  unsigned CCValid, CCMask;
+  SDValue Flags = emitCmp(DAG, CmpOp0, CmpOp1, CC, CCValid, CCMask);
 
-  SmallVector<SDValue, 4> Ops;
+  SmallVector<SDValue, 5> Ops;
   Ops.push_back(TrueOp);
   Ops.push_back(FalseOp);
+  Ops.push_back(DAG.getConstant(CCValid, MVT::i32));
   Ops.push_back(DAG.getConstant(CCMask, MVT::i32));
   Ops.push_back(Flags);
 
@@ -1704,7 +1712,8 @@ SystemZTargetLowering::emitSelect(MachineInstr *MI,
   unsigned DestReg  = MI->getOperand(0).getReg();
   unsigned TrueReg  = MI->getOperand(1).getReg();
   unsigned FalseReg = MI->getOperand(2).getReg();
-  unsigned CCMask   = MI->getOperand(3).getImm();
+  unsigned CCValid  = MI->getOperand(3).getImm();
+  unsigned CCMask   = MI->getOperand(4).getImm();
   DebugLoc DL       = MI->getDebugLoc();
 
   MachineBasicBlock *StartMBB = MBB;
@@ -1715,7 +1724,8 @@ SystemZTargetLowering::emitSelect(MachineInstr *MI,
   //   BRC CCMask, JoinMBB
   //   # fallthrough to FalseMBB
   MBB = StartMBB;
-  BuildMI(MBB, DL, TII->get(SystemZ::BRC)).addImm(CCMask).addMBB(JoinMBB);
+  BuildMI(MBB, DL, TII->get(SystemZ::BRC))
+    .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB);
   MBB->addSuccessor(JoinMBB);
   MBB->addSuccessor(FalseMBB);
 
@@ -1751,7 +1761,8 @@ SystemZTargetLowering::emitCondStore(MachineInstr *MI,
   MachineOperand Base = MI->getOperand(1);
   int64_t Disp        = MI->getOperand(2).getImm();
   unsigned IndexReg   = MI->getOperand(3).getReg();
-  unsigned CCMask     = MI->getOperand(4).getImm();
+  unsigned CCValid    = MI->getOperand(4).getImm();
+  unsigned CCMask     = MI->getOperand(5).getImm();
   DebugLoc DL         = MI->getDebugLoc();
 
   StoreOpcode = TII->getOpcodeForOffset(StoreOpcode, Disp);
@@ -1761,7 +1772,7 @@ SystemZTargetLowering::emitCondStore(MachineInstr *MI,
   // might be more complicated in that case.
   if (STOCOpcode && !IndexReg && TM.getSubtargetImpl()->hasLoadStoreOnCond()) {
     if (Invert)
-      CCMask = CCMask ^ SystemZ::CCMASK_ANY;
+      CCMask ^= CCValid;
     BuildMI(*MBB, MI, DL, TII->get(STOCOpcode))
       .addReg(SrcReg).addOperand(Base).addImm(Disp).addImm(CCMask);
     MI->eraseFromParent();
@@ -1770,7 +1781,7 @@ SystemZTargetLowering::emitCondStore(MachineInstr *MI,
 
   // Get the condition needed to branch around the store.
   if (!Invert)
-    CCMask = CCMask ^ SystemZ::CCMASK_ANY;
+    CCMask ^= CCValid;
 
   MachineBasicBlock *StartMBB = MBB;
   MachineBasicBlock *JoinMBB  = splitBlockAfter(MI, MBB);
@@ -1780,7 +1791,8 @@ SystemZTargetLowering::emitCondStore(MachineInstr *MI,
   //   BRC CCMask, JoinMBB
   //   # fallthrough to FalseMBB
   MBB = StartMBB;
-  BuildMI(MBB, DL, TII->get(SystemZ::BRC)).addImm(CCMask).addMBB(JoinMBB);
+  BuildMI(MBB, DL, TII->get(SystemZ::BRC))
+    .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB);
   MBB->addSuccessor(JoinMBB);
   MBB->addSuccessor(FalseMBB);
 
@@ -1812,7 +1824,6 @@ SystemZTargetLowering::emitAtomicLoadBinary(MachineInstr *MI,
   const SystemZInstrInfo *TII = TM.getInstrInfo();
   MachineFunction &MF = *MBB->getParent();
   MachineRegisterInfo &MRI = MF.getRegInfo();
-  unsigned MaskNE = CCMaskForCondCode(ISD::SETNE);
   bool IsSubWord = (BitSize < 32);
 
   // Extract the operands.  Base can be a register or a frame index.
@@ -1912,7 +1923,8 @@ SystemZTargetLowering::emitAtomicLoadBinary(MachineInstr *MI,
       .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0);
   BuildMI(MBB, DL, TII->get(CSOpcode), Dest)
     .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp);
-  BuildMI(MBB, DL, TII->get(SystemZ::BRC)).addImm(MaskNE).addMBB(LoopMBB);
+  BuildMI(MBB, DL, TII->get(SystemZ::BRC))
+    .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB);
   MBB->addSuccessor(LoopMBB);
   MBB->addSuccessor(DoneMBB);
 
@@ -1935,7 +1947,6 @@ SystemZTargetLowering::emitAtomicLoadMinMax(MachineInstr *MI,
   const SystemZInstrInfo *TII = TM.getInstrInfo();
   MachineFunction &MF = *MBB->getParent();
   MachineRegisterInfo &MRI = MF.getRegInfo();
-  unsigned MaskNE = CCMaskForCondCode(ISD::SETNE);
   bool IsSubWord = (BitSize < 32);
 
   // Extract the operands.  Base can be a register or a frame index.
@@ -2000,7 +2011,7 @@ SystemZTargetLowering::emitAtomicLoadMinMax(MachineInstr *MI,
   BuildMI(MBB, DL, TII->get(CompareOpcode))
     .addReg(RotatedOldVal).addReg(Src2);
   BuildMI(MBB, DL, TII->get(SystemZ::BRC))
-    .addImm(KeepOldMask).addMBB(UpdateMBB);
+    .addImm(SystemZ::CCMASK_ICMP).addImm(KeepOldMask).addMBB(UpdateMBB);
   MBB->addSuccessor(UpdateMBB);
   MBB->addSuccessor(UseAltMBB);
 
@@ -2030,7 +2041,8 @@ SystemZTargetLowering::emitAtomicLoadMinMax(MachineInstr *MI,
       .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0);
   BuildMI(MBB, DL, TII->get(CSOpcode), Dest)
     .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp);
-  BuildMI(MBB, DL, TII->get(SystemZ::BRC)).addImm(MaskNE).addMBB(LoopMBB);
+  BuildMI(MBB, DL, TII->get(SystemZ::BRC))
+    .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB);
   MBB->addSuccessor(LoopMBB);
   MBB->addSuccessor(DoneMBB);
 
@@ -2046,7 +2058,6 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr *MI,
   const SystemZInstrInfo *TII = TM.getInstrInfo();
   MachineFunction &MF = *MBB->getParent();
   MachineRegisterInfo &MRI = MF.getRegInfo();
-  unsigned MaskNE = CCMaskForCondCode(ISD::SETNE);
 
   // Extract the operands.  Base can be a register or a frame index.
   unsigned Dest        = MI->getOperand(0).getReg();
@@ -2122,7 +2133,8 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr *MI,
   BuildMI(MBB, DL, TII->get(SystemZ::CR))
     .addReg(Dest).addReg(RetryCmpVal);
   BuildMI(MBB, DL, TII->get(SystemZ::BRC))
-    .addImm(MaskNE).addMBB(DoneMBB);
+    .addImm(SystemZ::CCMASK_ICMP)
+    .addImm(SystemZ::CCMASK_CMP_NE).addMBB(DoneMBB);
   MBB->addSuccessor(DoneMBB);
   MBB->addSuccessor(SetMBB);
 
@@ -2142,7 +2154,8 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr *MI,
     .addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize);
   BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal)
     .addReg(OldVal).addReg(StoreVal).addOperand(Base).addImm(Disp);
-  BuildMI(MBB, DL, TII->get(SystemZ::BRC)).addImm(MaskNE).addMBB(LoopMBB);
+  BuildMI(MBB, DL, TII->get(SystemZ::BRC))
+    .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB);
   MBB->addSuccessor(LoopMBB);
   MBB->addSuccessor(DoneMBB);
 
diff --git a/lib/Target/SystemZ/SystemZInstrFormats.td b/lib/Target/SystemZ/SystemZInstrFormats.td
index 1c55da4f58..c0bb7b73c7 100644
--- a/lib/Target/SystemZ/SystemZInstrFormats.td
+++ b/lib/Target/SystemZ/SystemZInstrFormats.td
@@ -684,7 +684,7 @@ class UnaryRRF<string mnemonic, bits<16> opcode, RegisterOperand cls1,
 // is added as an implicit use.
 class CondUnaryRRF<string mnemonic, bits<16> opcode, RegisterOperand cls1,
                    RegisterOperand cls2>
-  : InstRRF<opcode, (outs cls1:$R1), (ins cls2:$R2, cond4:$R3),
+  : InstRRF<opcode, (outs cls1:$R1), (ins cls2:$R2, cond4:$valid, cond4:$R3),
             mnemonic#"r$R3\t$R1, $R2", []>,
     Requires<[FeatureLoadStoreOnCond]>;
 
@@ -1256,8 +1256,10 @@ class Pseudo<dag outs, dag ins, list<dag> pattern>
 // Implements "$dst = $cc & (8 >> CC) ? $src1 : $src2", where CC is
 // the value of the PSW's 2-bit condition code field.
 class SelectWrapper<RegisterOperand cls>
-  : Pseudo<(outs cls:$dst), (ins cls:$src1, cls:$src2, i8imm:$cc),
-           [(set cls:$dst, (z_select_ccmask cls:$src1, cls:$src2, imm:$cc))]> {
+  : Pseudo<(outs cls:$dst),
+           (ins cls:$src1, cls:$src2, uimm8zx4:$valid, uimm8zx4:$cc),
+           [(set cls:$dst, (z_select_ccmask cls:$src1, cls:$src2,
+                                            uimm8zx4:$valid, uimm8zx4:$cc))]> {
   let usesCustomInserter = 1;
   // Although the instructions used by these nodes do not in themselves
   // change CC, the insertion requires new blocks, and CC cannot be live
@@ -1270,12 +1272,16 @@ class SelectWrapper<RegisterOperand cls>
 multiclass CondStores<RegisterOperand cls, SDPatternOperator store,
                       SDPatternOperator load, AddressingMode mode> {
   let Defs = [CC], Uses = [CC], usesCustomInserter = 1 in {
-    def "" : Pseudo<(outs), (ins cls:$new, mode:$addr, uimm8zx4:$cc),
+    def "" : Pseudo<(outs),
+                    (ins cls:$new, mode:$addr, uimm8zx4:$valid, uimm8zx4:$cc),
                     [(store (z_select_ccmask cls:$new, (load mode:$addr),
-                                             uimm8zx4:$cc), mode:$addr)]>;
-    def Inv : Pseudo<(outs), (ins cls:$new, mode:$addr, uimm8zx4:$cc),
+                                             uimm8zx4:$valid, uimm8zx4:$cc),
+                            mode:$addr)]>;
+    def Inv : Pseudo<(outs),
+                     (ins cls:$new, mode:$addr, uimm8zx4:$valid, uimm8zx4:$cc),
                      [(store (z_select_ccmask (load mode:$addr), cls:$new,
-                                              uimm8zx4:$cc), mode:$addr)]>;
+                                              uimm8zx4:$valid, uimm8zx4:$cc),
+                              mode:$addr)]>;
   }
 }
 
diff --git a/lib/Target/SystemZ/SystemZInstrInfo.cpp b/lib/Target/SystemZ/SystemZInstrInfo.cpp
index dfb5c0983c..2b604a99fd 100644
--- a/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -201,13 +201,13 @@ bool SystemZInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
       // FIXME: add X86-style branch swap
       FBB = TBB;
       TBB = Branch.Target->getMBB();
+      Cond.push_back(MachineOperand::CreateImm(Branch.CCValid));
       Cond.push_back(MachineOperand::CreateImm(Branch.CCMask));
       continue;
     }
 
     // Handle subsequent conditional branches.
-    assert(Cond.size() == 1);
-    assert(TBB);
+    assert(Cond.size() == 2 && TBB && "Should have seen a conditional branch");
 
     // Only handle the case where all conditional branches branch to the same
     // destination.
@@ -215,11 +215,13 @@ bool SystemZInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
       return true;
 
     // If the conditions are the same, we can leave them alone.
-    unsigned OldCond = Cond[0].getImm();
-    if (OldCond == Branch.CCMask)
+    unsigned OldCCValid = Cond[0].getImm();
+    unsigned OldCCMask = Cond[1].getImm();
+    if (OldCCValid == Branch.CCValid && OldCCMask == Branch.CCMask)
       continue;
 
     // FIXME: Try combining conditions like X86 does.  Should be easy on Z!
+    return false;
   }
 
   return false;
@@ -247,6 +249,13 @@ unsigned SystemZInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
   return Count;
 }
 
+bool SystemZInstrInfo::
+ReverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
+  assert(Cond.size() == 2 && "Invalid condition");
+  Cond[1].setImm(Cond[1].getImm() ^ Cond[0].getImm());
+  return false;
+}
+
 unsigned
 SystemZInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
                                MachineBasicBlock *FBB,
@@ -258,7 +267,7 @@ SystemZInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
 
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
-  assert((Cond.size() == 1 || Cond.size() == 0) &&
+  assert((Cond.size() == 2 || Cond.size() == 0) &&
          "SystemZ branch conditions have one component!");
 
   if (Cond.empty()) {
@@ -270,8 +279,10 @@ SystemZInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
 
   // Conditional branch.
   unsigned Count = 0;
-  unsigned CC = Cond[0].getImm();
-  BuildMI(&MBB, DL, get(SystemZ::BRC)).addImm(CC).addMBB(TBB);
+  unsigned CCValid = Cond[0].getImm();
+  unsigned CCMask = Cond[1].getImm();
+  BuildMI(&MBB, DL, get(SystemZ::BRC))
+    .addImm(CCValid).addImm(CCMask).addMBB(TBB);
   ++Count;
 
   if (FBB) {
@@ -321,13 +332,16 @@ isProfitableToIfCvt(MachineBasicBlock &TMBB,
 bool SystemZInstrInfo::
 PredicateInstruction(MachineInstr *MI,
                      const SmallVectorImpl<MachineOperand> &Pred) const {
-  unsigned CCMask = Pred[0].getImm();
+  assert(Pred.size() == 2 && "Invalid condition");
+  unsigned CCValid = Pred[0].getImm();
+  unsigned CCMask = Pred[1].getImm();
   assert(CCMask > 0 && CCMask < 15 && "Invalid predicate");
   unsigned Opcode = MI->getOpcode();
   if (TM.getSubtargetImpl()->hasLoadStoreOnCond()) {
     if (unsigned CondOpcode = getConditionalMove(Opcode)) {
       MI->setDesc(get(CondOpcode));
-      MachineInstrBuilder(*MI->getParent()->getParent(), MI).addImm(CCMask);
+      MachineInstrBuilder(*MI->getParent()->getParent(), MI)
+        .addImm(CCValid).addImm(CCMask);
       return true;
     }
   }
@@ -645,13 +659,6 @@ SystemZInstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const {
   }
 }
 
-bool SystemZInstrInfo::
-ReverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
-  assert(Cond.size() == 1 && "Invalid branch condition!");
-  Cond[0].setImm(Cond[0].getImm() ^ SystemZ::CCMASK_ANY);
-  return false;
-}
-
 uint64_t SystemZInstrInfo::getInstSizeInBytes(const MachineInstr *MI) const {
   if (MI->getOpcode() == TargetOpcode::INLINEASM) {
     const MachineFunction *MF = MI->getParent()->getParent();
@@ -668,22 +675,23 @@ SystemZInstrInfo::getBranchInfo(const MachineInstr *MI) const {
   case SystemZ::J:
   case SystemZ::JG:
     return SystemZII::Branch(SystemZII::BranchNormal, SystemZ::CCMASK_ANY,
-                             &MI->getOperand(0));
+                             SystemZ::CCMASK_ANY, &MI->getOperand(0));
 
   case SystemZ::BRC:
   case SystemZ::BRCL:
     return SystemZII::Branch(SystemZII::BranchNormal,
-                             MI->getOperand(0).getImm(), &MI->getOperand(1));
+                             MI->getOperand(0).getImm(),
+                             MI->getOperand(1).getImm(), &MI->getOperand(2));
 
   case SystemZ::CIJ:
   case SystemZ::CRJ:
-    return SystemZII::Branch(SystemZII::BranchC, MI->getOperand(2).getImm(),
-                             &MI->getOperand(3));
+    return SystemZII::Branch(SystemZII::BranchC, SystemZ::CCMASK_ICMP,
+                             MI->getOperand(2).getImm(), &MI->getOperand(3));
 
   case SystemZ::CGIJ:
   case SystemZ::CGRJ:
-    return SystemZII::Branch(SystemZII::BranchCG, MI->getOperand(2).getImm(),
-                             &MI->getOperand(3));
+    return SystemZII::Branch(SystemZII::BranchCG, SystemZ::CCMASK_ICMP,
+                             MI->getOperand(2).getImm(), &MI->getOperand(3));
 
   default:
     llvm_unreachable("Unrecognized branch opcode");
diff --git a/lib/Target/SystemZ/SystemZInstrInfo.h b/lib/Target/SystemZ/SystemZInstrInfo.h
index 7617d03ddc..917ac6e348 100644
--- a/lib/Target/SystemZ/SystemZInstrInfo.h
+++ b/lib/Target/SystemZ/SystemZInstrInfo.h
@@ -66,14 +66,18 @@ namespace SystemZII {
     // The type of the branch.
     BranchType Type;
 
+    // CCMASK_<N> is set if CC might be equal to N.
+    unsigned CCValid;
+
     // CCMASK_<N> is set if the branch should be taken when CC == N.
     unsigned CCMask;
 
     // The target of the branch.
     const MachineOperand *Target;
 
-    Branch(BranchType type, unsigned ccMask, const MachineOperand *target)
-      : Type(type), CCMask(ccMask), Target(target) {}
+    Branch(BranchType type, unsigned ccValid, unsigned ccMask,
+           const MachineOperand *target)
+      : Type(type), CCValid(ccValid), CCMask(ccMask), Target(target) {}
   };
 }
 
diff --git a/lib/Target/SystemZ/SystemZInstrInfo.td b/lib/Target/SystemZ/SystemZInstrInfo.td
index b3ea36db7d..a6efd41d04 100644
--- a/lib/Target/SystemZ/SystemZInstrInfo.td
+++ b/lib/Target/SystemZ/SystemZInstrInfo.td
@@ -58,17 +58,19 @@ let isBranch = 1, isTerminator = 1, isBarrier = 1, R1 = 15 in {
 // in their raw BRC/BRCL form, with the 4-bit condition-code mask being
 // the first operand.  It seems friendlier to use mnemonic forms like
 // JE and JLH when writing out the assembly though.
-multiclass CondBranches<Operand ccmask, string short, string long> {
-  let isBranch = 1, isTerminator = 1, Uses = [CC] in {
-    def "" : InstRI<0xA74, (outs), (ins ccmask:$R1, brtarget16:$I2), short, []>;
-    def L  : InstRIL<0xC04, (outs), (ins ccmask:$R1, brtarget32:$I2), long, []>;
+let isBranch = 1, isTerminator = 1, Uses = [CC] in {
+  let isCodeGenOnly = 1 in {
+    def BRC : InstRI<0xA74, (outs), (ins cond4:$valid, cond4:$R1,
+                                         brtarget16:$I2), "j$R1\t$I2",
+                     [(z_br_ccmask cond4:$valid, cond4:$R1, bb:$I2)]>;
+    def BRCL : InstRIL<0xC04, (outs), (ins cond4:$valid, cond4:$R1,
+                                           brtarget32:$I2), "jg$R1\t$I2", []>;
   }
+  def AsmBRC : InstRI<0xA74, (outs), (ins uimm8zx4:$R1, brtarget16:$I2),
+                      "brc\t$R1, $I2", []>;
+  def AsmBRCL : InstRIL<0xC04, (outs), (ins uimm8zx4:$R1, brtarget32:$I2),
+                        "brcl\t$R1, $I2", []>;
 }
-let isCodeGenOnly = 1 in
-  defm BRC : CondBranches<cond4, "j$R1\t$I2", "jg$R1\t$I2">;
-defm AsmBRC : CondBranches<uimm8zx4, "brc\t$R1, $I2", "brcl\t$R1, $I2">;
-
-def : Pat<(z_br_ccmask cond4:$cond, bb:$dst), (BRC cond4:$cond, bb:$dst)>;
 
 // Fused compare-and-branch instructions.  As for normal branches,
 // we handle these instructions internally in their raw CRJ-like form,
@@ -1136,9 +1138,12 @@ def  : Pat<(sub GR64:$src1, (zextloadi32 bdxaddr20only:$addr)),
 
 // Optimize sign-extended 1/0 selects to -1/0 selects.  This is important
 // for vector legalization.
-def : Pat<(sra (shl (i32 (z_select_ccmask 1, 0, imm:$cc)), (i32 31)), (i32 31)),
-          (Select32 (LHI -1), (LHI 0), imm:$cc)>;
-def : Pat<(sra (shl (i64 (anyext (i32 (z_select_ccmask 1, 0, imm:$cc)))),
+def : Pat<(sra (shl (i32 (z_select_ccmask 1, 0, uimm8zx4:$valid, uimm8zx4:$cc)),
+                         (i32 31)),
+                    (i32 31)),
+          (Select32 (LHI -1), (LHI 0), uimm8zx4:$valid, uimm8zx4:$cc)>;
+def : Pat<(sra (shl (i64 (anyext (i32 (z_select_ccmask 1, 0, uimm8zx4:$valid,
+                                                       uimm8zx4:$cc)))),
                     (i32 63)),
                (i32 63)),
-          (Select64 (LGHI -1), (LGHI 0), imm:$cc)>;
+          (Select64 (LGHI -1), (LGHI 0), uimm8zx4:$valid, uimm8zx4:$cc)>;
diff --git a/lib/Target/SystemZ/SystemZLongBranch.cpp b/lib/Target/SystemZ/SystemZLongBranch.cpp
index 2d2605886b..9b637c01c6 100644
--- a/lib/Target/SystemZ/SystemZLongBranch.cpp
+++ b/lib/Target/SystemZ/SystemZLongBranch.cpp
@@ -297,13 +297,16 @@ bool SystemZLongBranch::fuseCompareAndBranch(MachineInstr *Compare) {
   for (++MBBI; MBBI != MBBE; ++MBBI) {
     if (MBBI->getOpcode() == SystemZ::BRC && !isCCLiveAfter(MBBI, TRI)) {
       // Read the branch mask and target.
-      MachineOperand CCMask(MBBI->getOperand(0));
-      MachineOperand Target(MBBI->getOperand(1));
+      MachineOperand CCMask(MBBI->getOperand(1));
+      MachineOperand Target(MBBI->getOperand(2));
+      assert((CCMask.getImm() & ~SystemZ::CCMASK_ICMP) == 0 &&
+             "Invalid condition-code mask for integer comparison");
 
       // Clear out all current operands.
       int CCUse = MBBI->findRegisterUseOperandIdx(SystemZ::CC, false, TRI);
       assert(CCUse >= 0 && "BRC must use CC");
       MBBI->RemoveOperand(CCUse);
+      MBBI->RemoveOperand(2);
       MBBI->RemoveOperand(1);
       MBBI->RemoveOperand(0);
 
@@ -441,10 +444,11 @@ void SystemZLongBranch::splitCompareBranch(MachineInstr *MI,
     .addOperand(MI->getOperand(0))
     .addOperand(MI->getOperand(1));
   MachineInstr *BRCL = BuildMI(*MBB, MI, DL, TII->get(SystemZ::BRCL))
+    .addImm(SystemZ::CCMASK_ICMP)
     .addOperand(MI->getOperand(2))
     .addOperand(MI->getOperand(3));
   // The implicit use of CC is a killing use.
-  BRCL->getOperand(2).setIsKill();
+  BRCL->addRegisterKilled(SystemZ::CC, &TII->getRegisterInfo());
   MI->eraseFromParent();
 }
 
diff --git a/lib/Target/SystemZ/SystemZOperators.td b/lib/Target/SystemZ/SystemZOperators.td
index 693f3a1e16..6a3af2b890 100644
--- a/lib/Target/SystemZ/SystemZOperators.td
+++ b/lib/Target/SystemZ/SystemZOperators.td
@@ -15,13 +15,15 @@ def SDT_CallSeqEnd          : SDCallSeqEnd<[SDTCisVT<0, i64>,
                                             SDTCisVT<1, i64>]>;
 def SDT_ZCall               : SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>;
 def SDT_ZCmp                : SDTypeProfile<0, 2, [SDTCisSameAs<0, 1>]>;
-def SDT_ZBRCCMask           : SDTypeProfile<0, 2,
+def SDT_ZBRCCMask           : SDTypeProfile<0, 3,
                                             [SDTCisVT<0, i8>,
-                                             SDTCisVT<1, OtherVT>]>;
-def SDT_ZSelectCCMask       : SDTypeProfile<1, 3,
+                                             SDTCisVT<1, i8>,
+                                             SDTCisVT<2, OtherVT>]>;
+def SDT_ZSelectCCMask       : SDTypeProfile<1, 4,
                                             [SDTCisSameAs<0, 1>,
                                              SDTCisSameAs<1, 2>,
-                                             SDTCisVT<3, i8>]>;
+                                             SDTCisVT<3, i8>,
+                                             SDTCisVT<4, i8>]>;
 def SDT_ZWrapPtr            : SDTypeProfile<1, 1,
                                             [SDTCisSameAs<0, 1>,
                                              SDTCisPtrTy<0>]>;
diff --git a/lib/Target/SystemZ/SystemZPatterns.td b/lib/Target/SystemZ/SystemZPatterns.td
index 4e4386a398..5419c2badf 100644
--- a/lib/Target/SystemZ/SystemZPatterns.td
+++ b/lib/Target/SystemZ/SystemZPatterns.td
@@ -58,10 +58,12 @@ multiclass RMWIByte<SDPatternOperator operator, AddressingMode mode,
 // register of class CLS.  The load may trap even if the condition is false.
 multiclass CondLoad<Instruction insn, RegisterOperand cls,
                     SDPatternOperator load> {
-  def : Pat<(z_select_ccmask (load bdaddr20only:$addr), cls:$new, uimm8zx4:$cc),
+  def : Pat<(z_select_ccmask (load bdaddr20only:$addr), cls:$new, uimm8zx4,
+                             uimm8zx4:$cc),
             (insn cls:$new, bdaddr20only:$addr, uimm8zx4:$cc)>,
         Requires<[FeatureLoadStoreOnCond]>;
-  def : Pat<(z_select_ccmask cls:$new, (load bdaddr20only:$addr), uimm8zx4:$cc),
+  def : Pat<(z_select_ccmask cls:$new, (load bdaddr20only:$addr), uimm8zx4,
+                             uimm8zx4:$cc),
             (insn cls:$new, bdaddr20only:$addr, (INVCC uimm8zx4:$cc))>,
         Requires<[FeatureLoadStoreOnCond]>;
 }
diff --git a/test/CodeGen/SystemZ/atomic-store-03.ll b/test/CodeGen/SystemZ/atomic-store-03.ll
index 0954f6f478..846c86fd36 100644
--- a/test/CodeGen/SystemZ/atomic-store-03.ll
+++ b/test/CodeGen/SystemZ/atomic-store-03.ll
@@ -9,7 +9,7 @@ define void @f1(i32 %val, i32 *%src) {
 ; CHECK: l %r0, 0(%r3)
 ; CHECK: [[LABEL:\.[^:]*]]:
 ; CHECK: cs %r0, %r2, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   store atomic i32 %val, i32 *%src seq_cst, align 4
   ret void
diff --git a/test/CodeGen/SystemZ/atomic-store-04.ll b/test/CodeGen/SystemZ/atomic-store-04.ll
index d4182783b7..24615b1156 100644
--- a/test/CodeGen/SystemZ/atomic-store-04.ll
+++ b/test/CodeGen/SystemZ/atomic-store-04.ll
@@ -9,7 +9,7 @@ define void @f1(i64 %val, i64 *%src) {
 ; CHECK: lg %r0, 0(%r3)
 ; CHECK: [[LABEL:\.[^:]*]]:
 ; CHECK: csg %r0, %r2, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   store atomic i64 %val, i64 *%src seq_cst, align 8
   ret void
diff --git a/test/CodeGen/SystemZ/atomicrmw-add-01.ll b/test/CodeGen/SystemZ/atomicrmw-add-01.ll
index 2c8598ddd2..25f71f31ef 100644
--- a/test/CodeGen/SystemZ/atomicrmw-add-01.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-add-01.ll
@@ -22,7 +22,7 @@ define i8 @f1(i8 *%src, i8 %b) {
 ; CHECK: ar [[ROT]], %r3
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -56,7 +56,7 @@ define i8 @f2(i8 *%src) {
 ; CHECK: afi [[ROT]], -2147483648
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0([[NEGSHIFT:%r[1-9]+]])
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK: br %r14
 ;
diff --git a/test/CodeGen/SystemZ/atomicrmw-add-02.ll b/test/CodeGen/SystemZ/atomicrmw-add-02.ll
index 2aff860bd2..cd4e4784c3 100644
--- a/test/CodeGen/SystemZ/atomicrmw-add-02.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-add-02.ll
@@ -22,7 +22,7 @@ define i16 @f1(i16 *%src, i16 %b) {
 ; CHECK: ar [[ROT]], %r3
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -56,7 +56,7 @@ define i16 @f2(i16 *%src) {
 ; CHECK: afi [[ROT]], -2147483648
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0([[NEGSHIFT:%r[1-9]+]])
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK: br %r14
 ;
diff --git a/test/CodeGen/SystemZ/atomicrmw-add-03.ll b/test/CodeGen/SystemZ/atomicrmw-add-03.ll
index 6a919a724a..a81af72d1e 100644
--- a/test/CodeGen/SystemZ/atomicrmw-add-03.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-add-03.ll
@@ -10,7 +10,7 @@ define i32 @f1(i32 %dummy, i32 *%src, i32 %b) {
 ; CHECK: lr %r0, %r2
 ; CHECK: ar %r0, %r4
 ; CHECK: cs %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw add i32 *%src, i32 %b seq_cst
   ret i32 %res
@@ -24,7 +24,7 @@ define i32 @f2(i32 %dummy, i32 *%src) {
 ; CHECK: lr %r0, %r2
 ; CHECK: ahi %r0, 1
 ; CHECK: cs %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw add i32 *%src, i32 1 seq_cst
   ret i32 %res
diff --git a/test/CodeGen/SystemZ/atomicrmw-add-04.ll b/test/CodeGen/SystemZ/atomicrmw-add-04.ll
index 3b5c0d0b95..e7905491f2 100644
--- a/test/CodeGen/SystemZ/atomicrmw-add-04.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-add-04.ll
@@ -10,7 +10,7 @@ define i64 @f1(i64 %dummy, i64 *%src, i64 %b) {
 ; CHECK: lgr %r0, %r2
 ; CHECK: agr %r0, %r4
 ; CHECK: csg %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw add i64 *%src, i64 %b seq_cst
   ret i64 %res
@@ -24,7 +24,7 @@ define i64 @f2(i64 %dummy, i64 *%src) {
 ; CHECK: lgr %r0, %r2
 ; CHECK: aghi %r0, 1
 ; CHECK: csg %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw add i64 *%src, i64 1 seq_cst
   ret i64 %res
diff --git a/test/CodeGen/SystemZ/atomicrmw-and-01.ll b/test/CodeGen/SystemZ/atomicrmw-and-01.ll
index 5cb5aba792..6d2f541c3a 100644
--- a/test/CodeGen/SystemZ/atomicrmw-and-01.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-and-01.ll
@@ -22,7 +22,7 @@ define i8 @f1(i8 *%src, i8 %b) {
 ; CHECK: nr [[ROT]], %r3
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -57,7 +57,7 @@ define i8 @f2(i8 *%src) {
 ; CHECK: nilh [[ROT]], 33023
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0([[NEGSHIFT:%r[1-9]+]])
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK: br %r14
 ;
diff --git a/test/CodeGen/SystemZ/atomicrmw-and-02.ll b/test/CodeGen/SystemZ/atomicrmw-and-02.ll
index bc92db141e..572b22484b 100644
--- a/test/CodeGen/SystemZ/atomicrmw-and-02.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-and-02.ll
@@ -22,7 +22,7 @@ define i16 @f1(i16 *%src, i16 %b) {
 ; CHECK: nr [[ROT]], %r3
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -57,7 +57,7 @@ define i16 @f2(i16 *%src) {
 ; CHECK: nilh [[ROT]], 32768
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0([[NEGSHIFT:%r[1-9]+]])
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK: br %r14
 ;
diff --git a/test/CodeGen/SystemZ/atomicrmw-and-03.ll b/test/CodeGen/SystemZ/atomicrmw-and-03.ll
index 6c7ba23e1b..8d813a1402 100644
--- a/test/CodeGen/SystemZ/atomicrmw-and-03.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-and-03.ll
@@ -10,7 +10,7 @@ define i32 @f1(i32 %dummy, i32 *%src, i32 %b) {
 ; CHECK: lr %r0, %r2
 ; CHECK: nr %r0, %r4
 ; CHECK: cs %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw and i32 *%src, i32 %b seq_cst
   ret i32 %res
@@ -24,7 +24,7 @@ define i32 @f2(i32 %dummy, i32 *%src) {
 ; CHECK: lr %r0, %r2
 ; CHECK: nilf %r0, 1
 ; CHECK: cs %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw and i32 *%src, i32 1 seq_cst
   ret i32 %res
diff --git a/test/CodeGen/SystemZ/atomicrmw-and-04.ll b/test/CodeGen/SystemZ/atomicrmw-and-04.ll
index 649eeebe5d..89899a6a03 100644
--- a/test/CodeGen/SystemZ/atomicrmw-and-04.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-and-04.ll
@@ -10,7 +10,7 @@ define i64 @f1(i64 %dummy, i64 *%src, i64 %b) {
 ; CHECK: lgr %r0, %r2
 ; CHECK: ngr %r0, %r4
 ; CHECK: csg %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw and i64 *%src, i64 %b seq_cst
   ret i64 %res
@@ -33,7 +33,7 @@ define i64 @f3(i64 %dummy, i64 *%src) {
 ; CHECK: [[LABEL:\.[^:]*]]:
 ; CHECK: risbg %r0, %r2, 31, 191, 0
 ; CHECK: csg %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw and i64 *%src, i64 8589934591 seq_cst
   ret i64 %res
@@ -47,7 +47,7 @@ define i64 @f4(i64 %dummy, i64 *%src) {
 ; CHECK: lgr %r0, %r2
 ; CHECK: nihf %r0, 2
 ; CHECK: csg %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw and i64 *%src, i64 12884901887 seq_cst
   ret i64 %res
diff --git a/test/CodeGen/SystemZ/atomicrmw-minmax-01.ll b/test/CodeGen/SystemZ/atomicrmw-minmax-01.ll
index 7368758f61..a15fe57349 100644
--- a/test/CodeGen/SystemZ/atomicrmw-minmax-01.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-minmax-01.ll
@@ -24,7 +24,7 @@ define i8 @f1(i8 *%src, i8 %b) {
 ; CHECK: [[KEEP]]:
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -60,7 +60,7 @@ define i8 @f2(i8 *%src, i8 %b) {
 ; CHECK: [[KEEP]]:
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -97,7 +97,7 @@ define i8 @f3(i8 *%src, i8 %b) {
 ; CHECK: [[KEEP]]:
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -134,7 +134,7 @@ define i8 @f4(i8 *%src, i8 %b) {
 ; CHECK: [[KEEP]]:
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK: br %r14
 ;
diff --git a/test/CodeGen/SystemZ/atomicrmw-minmax-02.ll b/test/CodeGen/SystemZ/atomicrmw-minmax-02.ll
index e5541b5326..c0ae883202 100644
--- a/test/CodeGen/SystemZ/atomicrmw-minmax-02.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-minmax-02.ll
@@ -24,7 +24,7 @@ define i16 @f1(i16 *%src, i16 %b) {
 ; CHECK: [[KEEP]]:
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -60,7 +60,7 @@ define i16 @f2(i16 *%src, i16 %b) {
 ; CHECK: [[KEEP]]:
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -97,7 +97,7 @@ define i16 @f3(i16 *%src, i16 %b) {
 ; CHECK: [[KEEP]]:
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -134,7 +134,7 @@ define i16 @f4(i16 *%src, i16 %b) {
 ; CHECK: [[KEEP]]:
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK: br %r14
 ;
diff --git a/test/CodeGen/SystemZ/atomicrmw-minmax-03.ll b/test/CodeGen/SystemZ/atomicrmw-minmax-03.ll
index 13742b2e41..3a9485af4f 100644
--- a/test/CodeGen/SystemZ/atomicrmw-minmax-03.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-minmax-03.ll
@@ -11,7 +11,7 @@ define i32 @f1(i32 %dummy, i32 *%src, i32 %b) {
 ; CHECK: crjle %r2, %r4, [[KEEP:\..*]]
 ; CHECK: lr [[NEW]], %r4
 ; CHECK: cs %r2, [[NEW]], 0(%r3)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: br %r14
   %res = atomicrmw min i32 *%src, i32 %b seq_cst
   ret i32 %res
@@ -26,7 +26,7 @@ define i32 @f2(i32 %dummy, i32 *%src, i32 %b) {
 ; CHECK: crjhe %r2, %r4, [[KEEP:\..*]]
 ; CHECK: lr [[NEW]], %r4
 ; CHECK: cs %r2, [[NEW]], 0(%r3)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: br %r14
   %res = atomicrmw max i32 *%src, i32 %b seq_cst
   ret i32 %res
@@ -42,7 +42,7 @@ define i32 @f3(i32 %dummy, i32 *%src, i32 %b) {
 ; CHECK: jle [[KEEP:\..*]]
 ; CHECK: lr [[NEW]], %r4
 ; CHECK: cs %r2, [[NEW]], 0(%r3)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: br %r14
   %res = atomicrmw umin i32 *%src, i32 %b seq_cst
   ret i32 %res
@@ -58,7 +58,7 @@ define i32 @f4(i32 %dummy, i32 *%src, i32 %b) {
 ; CHECK: jhe [[KEEP:\..*]]
 ; CHECK: lr [[NEW]], %r4
 ; CHECK: cs %r2, [[NEW]], 0(%r3)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: br %r14
   %res = atomicrmw umax i32 *%src, i32 %b seq_cst
   ret i32 %res
@@ -166,7 +166,7 @@ define i32 @f13(i32 %dummy, i32 *%ptr) {
 ; CHECK: crjle %r2, [[LIMIT]], [[KEEP:\..*]]
 ; CHECK: lhi [[NEW]], 42
 ; CHECK: cs %r2, [[NEW]], 0(%r3)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: br %r14
   %res = atomicrmw min i32 *%ptr, i32 42 seq_cst
   ret i32 %res
diff --git a/test/CodeGen/SystemZ/atomicrmw-minmax-04.ll b/test/CodeGen/SystemZ/atomicrmw-minmax-04.ll
index 9efa16b38f..ebed147586 100644
--- a/test/CodeGen/SystemZ/atomicrmw-minmax-04.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-minmax-04.ll
@@ -11,7 +11,7 @@ define i64 @f1(i64 %dummy, i64 *%src, i64 %b) {
 ; CHECK: cgrjle %r2, %r4, [[KEEP:\..*]]
 ; CHECK: lgr [[NEW]], %r4
 ; CHECK: csg %r2, [[NEW]], 0(%r3)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: br %r14
   %res = atomicrmw min i64 *%src, i64 %b seq_cst
   ret i64 %res
@@ -26,7 +26,7 @@ define i64 @f2(i64 %dummy, i64 *%src, i64 %b) {
 ; CHECK: cgrjhe %r2, %r4, [[KEEP:\..*]]
 ; CHECK: lgr [[NEW]], %r4
 ; CHECK: csg %r2, [[NEW]], 0(%r3)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: br %r14
   %res = atomicrmw max i64 *%src, i64 %b seq_cst
   ret i64 %res
@@ -42,7 +42,7 @@ define i64 @f3(i64 %dummy, i64 *%src, i64 %b) {
 ; CHECK: jle [[KEEP:\..*]]
 ; CHECK: lgr [[NEW]], %r4
 ; CHECK: csg %r2, [[NEW]], 0(%r3)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: br %r14
   %res = atomicrmw umin i64 *%src, i64 %b seq_cst
   ret i64 %res
@@ -58,7 +58,7 @@ define i64 @f4(i64 %dummy, i64 *%src, i64 %b) {
 ; CHECK: jhe [[KEEP:\..*]]
 ; CHECK: lgr [[NEW]], %r4
 ; CHECK: csg %r2, [[NEW]], 0(%r3)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: br %r14
   %res = atomicrmw umax i64 *%src, i64 %b seq_cst
   ret i64 %res
@@ -133,7 +133,7 @@ define i64 @f10(i64 %dummy, i64 *%ptr) {
 ; CHECK: cgrjle %r2, [[LIMIT]], [[KEEP:\..*]]
 ; CHECK: lghi [[NEW]], 42
 ; CHECK: csg %r2, [[NEW]], 0(%r3)
-; CHECK: jlh [[LOOP]]
+; CHECK: jl [[LOOP]]
 ; CHECK: br %r14
   %res = atomicrmw min i64 *%ptr, i64 42 seq_cst
   ret i64 %res
diff --git a/test/CodeGen/SystemZ/atomicrmw-nand-01.ll b/test/CodeGen/SystemZ/atomicrmw-nand-01.ll
index ec81af2108..db5bb8ff9e 100644
--- a/test/CodeGen/SystemZ/atomicrmw-nand-01.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-nand-01.ll
@@ -23,7 +23,7 @@ define i8 @f1(i8 *%src, i8 %b) {
 ; CHECK: xilf [[ROT]], 4278190080
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -59,7 +59,7 @@ define i8 @f2(i8 *%src) {
 ; CHECK: xilf [[ROT]], 4278190080
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0([[NEGSHIFT:%r[1-9]+]])
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK: br %r14
 ;
diff --git a/test/CodeGen/SystemZ/atomicrmw-nand-02.ll b/test/CodeGen/SystemZ/atomicrmw-nand-02.ll
index 3396f68fa7..6141543e0d 100644
--- a/test/CodeGen/SystemZ/atomicrmw-nand-02.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-nand-02.ll
@@ -23,7 +23,7 @@ define i16 @f1(i16 *%src, i16 %b) {
 ; CHECK: xilf [[ROT]], 4294901760
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -59,7 +59,7 @@ define i16 @f2(i16 *%src) {
 ; CHECK: xilf [[ROT]], 4294901760
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0([[NEGSHIFT:%r[1-9]+]])
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK: br %r14
 ;
diff --git a/test/CodeGen/SystemZ/atomicrmw-nand-03.ll b/test/CodeGen/SystemZ/atomicrmw-nand-03.ll
index c511bd608f..c7a6691083 100644
--- a/test/CodeGen/SystemZ/atomicrmw-nand-03.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-nand-03.ll
@@ -11,7 +11,7 @@ define i32 @f1(i32 %dummy, i32 *%src, i32 %b) {
 ; CHECK: nr %r0, %r4
 ; CHECK: xilf %r0, 4294967295
 ; CHECK: cs %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw nand i32 *%src, i32 %b seq_cst
   ret i32 %res
@@ -26,7 +26,7 @@ define i32 @f2(i32 %dummy, i32 *%src) {
 ; CHECK: nilf %r0, 1
 ; CHECK: xilf %r0, 4294967295
 ; CHECK: cs %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw nand i32 *%src, i32 1 seq_cst
   ret i32 %res
diff --git a/test/CodeGen/SystemZ/atomicrmw-nand-04.ll b/test/CodeGen/SystemZ/atomicrmw-nand-04.ll
index cda7aa2673..91fe639cd7 100644
--- a/test/CodeGen/SystemZ/atomicrmw-nand-04.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-nand-04.ll
@@ -12,7 +12,7 @@ define i64 @f1(i64 %dummy, i64 *%src, i64 %b) {
 ; CHECK: lcgr %r0, %r0
 ; CHECK: aghi %r0, -1
 ; CHECK: csg %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw nand i64 *%src, i64 %b seq_cst
   ret i64 %res
@@ -37,7 +37,7 @@ define i64 @f3(i64 %dummy, i64 *%src) {
 ; CHECK: lcgr %r0, %r0
 ; CHECK: aghi %r0, -1
 ; CHECK: csg %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw nand i64 *%src, i64 8589934591 seq_cst
   ret i64 %res
@@ -53,7 +53,7 @@ define i64 @f4(i64 %dummy, i64 *%src) {
 ; CHECK: lcgr %r0, %r0
 ; CHECK: aghi %r0, -1
 ; CHECK: csg %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw nand i64 *%src, i64 12884901887 seq_cst
   ret i64 %res
diff --git a/test/CodeGen/SystemZ/atomicrmw-or-01.ll b/test/CodeGen/SystemZ/atomicrmw-or-01.ll
index bf4be43e31..caba621add 100644
--- a/test/CodeGen/SystemZ/atomicrmw-or-01.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-or-01.ll
@@ -22,7 +22,7 @@ define i8 @f1(i8 *%src, i8 %b) {
 ; CHECK: or [[ROT]], %r3
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -56,7 +56,7 @@ define i8 @f2(i8 *%src) {
 ; CHECK: oilh [[ROT]], 32768
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0([[NEGSHIFT:%r[1-9]+]])
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK: br %r14
 ;
diff --git a/test/CodeGen/SystemZ/atomicrmw-or-02.ll b/test/CodeGen/SystemZ/atomicrmw-or-02.ll
index 726d9aaa35..877c642a35 100644
--- a/test/CodeGen/SystemZ/atomicrmw-or-02.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-or-02.ll
@@ -22,7 +22,7 @@ define i16 @f1(i16 *%src, i16 %b) {
 ; CHECK: or [[ROT]], %r3
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -56,7 +56,7 @@ define i16 @f2(i16 *%src) {
 ; CHECK: oilh [[ROT]], 32768
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0([[NEGSHIFT:%r[1-9]+]])
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK: br %r14
 ;
diff --git a/test/CodeGen/SystemZ/atomicrmw-or-03.ll b/test/CodeGen/SystemZ/atomicrmw-or-03.ll
index 692b11c4cf..9a0aa86feb 100644
--- a/test/CodeGen/SystemZ/atomicrmw-or-03.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-or-03.ll
@@ -10,7 +10,7 @@ define i32 @f1(i32 %dummy, i32 *%src, i32 %b) {
 ; CHECK: lr %r0, %r2
 ; CHECK: or %r0, %r4
 ; CHECK: cs %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw or i32 *%src, i32 %b seq_cst
   ret i32 %res
@@ -24,7 +24,7 @@ define i32 @f2(i32 %dummy, i32 *%src) {
 ; CHECK: lr %r0, %r2
 ; CHECK: oill %r0, 1
 ; CHECK: cs %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw or i32 *%src, i32 1 seq_cst
   ret i32 %res
diff --git a/test/CodeGen/SystemZ/atomicrmw-or-04.ll b/test/CodeGen/SystemZ/atomicrmw-or-04.ll
index 4782768765..dbc0f11cc7 100644
--- a/test/CodeGen/SystemZ/atomicrmw-or-04.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-or-04.ll
@@ -10,7 +10,7 @@ define i64 @f1(i64 %dummy, i64 *%src, i64 %b) {
 ; CHECK: lgr %r0, %r2
 ; CHECK: ogr %r0, %r4
 ; CHECK: csg %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw or i64 *%src, i64 %b seq_cst
   ret i64 %res
@@ -24,7 +24,7 @@ define i64 @f2(i64 %dummy, i64 *%src) {
 ; CHECK: lgr %r0, %r2
 ; CHECK: oill %r0, 1
 ; CHECK: csg %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw or i64 *%src, i64 1 seq_cst
   ret i64 %res
diff --git a/test/CodeGen/SystemZ/atomicrmw-sub-01.ll b/test/CodeGen/SystemZ/atomicrmw-sub-01.ll
index 6d42545631..2c08ebd9f5 100644
--- a/test/CodeGen/SystemZ/atomicrmw-sub-01.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-sub-01.ll
@@ -22,7 +22,7 @@ define i8 @f1(i8 *%src, i8 %b) {
 ; CHECK: sr [[ROT]], %r3
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -56,7 +56,7 @@ define i8 @f2(i8 *%src) {
 ; CHECK: afi [[ROT]], -2147483648
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0([[NEGSHIFT:%r[1-9]+]])
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK: br %r14
 ;
diff --git a/test/CodeGen/SystemZ/atomicrmw-sub-02.ll b/test/CodeGen/SystemZ/atomicrmw-sub-02.ll
index 6f8e26e5c0..f82ebd9aaa 100644
--- a/test/CodeGen/SystemZ/atomicrmw-sub-02.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-sub-02.ll
@@ -22,7 +22,7 @@ define i16 @f1(i16 *%src, i16 %b) {
 ; CHECK: sr [[ROT]], %r3
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -56,7 +56,7 @@ define i16 @f2(i16 *%src) {
 ; CHECK: afi [[ROT]], -2147483648
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0([[NEGSHIFT:%r[1-9]+]])
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK: br %r14
 ;
diff --git a/test/CodeGen/SystemZ/atomicrmw-sub-03.ll b/test/CodeGen/SystemZ/atomicrmw-sub-03.ll
index 64ec603fae..a3031c6806 100644
--- a/test/CodeGen/SystemZ/atomicrmw-sub-03.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-sub-03.ll
@@ -10,7 +10,7 @@ define i32 @f1(i32 %dummy, i32 *%src, i32 %b) {
 ; CHECK: lr %r0, %r2
 ; CHECK: sr %r0, %r4
 ; CHECK: cs %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw sub i32 *%src, i32 %b seq_cst
   ret i32 %res
@@ -24,7 +24,7 @@ define i32 @f2(i32 %dummy, i32 *%src) {
 ; CHECK: lr %r0, %r2
 ; CHECK: ahi %r0, -1
 ; CHECK: cs %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw sub i32 *%src, i32 1 seq_cst
   ret i32 %res
diff --git a/test/CodeGen/SystemZ/atomicrmw-sub-04.ll b/test/CodeGen/SystemZ/atomicrmw-sub-04.ll
index 18929b8c49..911648b613 100644
--- a/test/CodeGen/SystemZ/atomicrmw-sub-04.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-sub-04.ll
@@ -10,7 +10,7 @@ define i64 @f1(i64 %dummy, i64 *%src, i64 %b) {
 ; CHECK: lgr %r0, %r2
 ; CHECK: sgr %r0, %r4
 ; CHECK: csg %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw sub i64 *%src, i64 %b seq_cst
   ret i64 %res
@@ -24,7 +24,7 @@ define i64 @f2(i64 %dummy, i64 *%src) {
 ; CHECK: lgr %r0, %r2
 ; CHECK: aghi %r0, -1
 ; CHECK: csg %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw sub i64 *%src, i64 1 seq_cst
   ret i64 %res
diff --git a/test/CodeGen/SystemZ/atomicrmw-xchg-01.ll b/test/CodeGen/SystemZ/atomicrmw-xchg-01.ll
index 238ce336ac..52575c6349 100644
--- a/test/CodeGen/SystemZ/atomicrmw-xchg-01.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-xchg-01.ll
@@ -20,7 +20,7 @@ define i8 @f1(i8 *%src, i8 %b) {
 ; CHECK: risbg [[ROT]], %r3, 32, 39, 24
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK: br %r14
 ;
diff --git a/test/CodeGen/SystemZ/atomicrmw-xchg-02.ll b/test/CodeGen/SystemZ/atomicrmw-xchg-02.ll
index 635d9d440d..04be623ada 100644
--- a/test/CodeGen/SystemZ/atomicrmw-xchg-02.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-xchg-02.ll
@@ -20,7 +20,7 @@ define i16 @f1(i16 *%src, i16 %b) {
 ; CHECK: risbg [[ROT]], %r3, 32, 47, 16
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK: br %r14
 ;
diff --git a/test/CodeGen/SystemZ/atomicrmw-xchg-03.ll b/test/CodeGen/SystemZ/atomicrmw-xchg-03.ll
index e5ba4d5900..a602a02a18 100644
--- a/test/CodeGen/SystemZ/atomicrmw-xchg-03.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-xchg-03.ll
@@ -8,7 +8,7 @@ define i32 @f1(i32 %dummy, i32 *%src, i32 %b) {
 ; CHECK: l %r2, 0(%r3)
 ; CHECK: [[LABEL:\.[^:]*]]:
 ; CHECK: cs %r2, %r4, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw xchg i32 *%src, i32 %b seq_cst
   ret i32 %res
@@ -114,7 +114,7 @@ define i32 @f10(i32 %dummy, i32 *%src) {
 ; CHECK: l %r2, 0(%r3)
 ; CHECK: [[LABEL:\.[^:]*]]:
 ; CHECK: cs %r2, [[VALUE]], 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw xchg i32 *%src, i32 40000 seq_cst
   ret i32 %res
diff --git a/test/CodeGen/SystemZ/atomicrmw-xchg-04.ll b/test/CodeGen/SystemZ/atomicrmw-xchg-04.ll
index c19b34d9fe..80c0eeb712 100644
--- a/test/CodeGen/SystemZ/atomicrmw-xchg-04.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-xchg-04.ll
@@ -8,7 +8,7 @@ define i64 @f1(i64 %dummy, i64 *%src, i64 %b) {
 ; CHECK: lg %r2, 0(%r3)
 ; CHECK: [[LABEL:\.[^:]*]]:
 ; CHECK: csg %r2, %r4, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw xchg i64 *%src, i64 %b seq_cst
   ret i64 %res
@@ -81,7 +81,7 @@ define i64 @f7(i64 %dummy, i64 *%ptr) {
 ; CHECK: lg %r2, 0(%r3)
 ; CHECK: [[LABEL:\.[^:]*]]:
 ; CHECK: csg %r2, [[VALUE]], 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw xchg i64 *%ptr, i64 3000000000 seq_cst
   ret i64 %res
diff --git a/test/CodeGen/SystemZ/atomicrmw-xor-01.ll b/test/CodeGen/SystemZ/atomicrmw-xor-01.ll
index 969e3ef87c..e8fef2d31d 100644
--- a/test/CodeGen/SystemZ/atomicrmw-xor-01.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-xor-01.ll
@@ -22,7 +22,7 @@ define i8 @f1(i8 *%src, i8 %b) {
 ; CHECK: xr [[ROT]], %r3
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -56,7 +56,7 @@ define i8 @f2(i8 *%src) {
 ; CHECK: xilf [[ROT]], 2147483648
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0([[NEGSHIFT:%r[1-9]+]])
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK: br %r14
 ;
diff --git a/test/CodeGen/SystemZ/atomicrmw-xor-02.ll b/test/CodeGen/SystemZ/atomicrmw-xor-02.ll
index 10d2e3f592..9405c2ec0c 100644
--- a/test/CodeGen/SystemZ/atomicrmw-xor-02.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-xor-02.ll
@@ -22,7 +22,7 @@ define i16 @f1(i16 *%src, i16 %b) {
 ; CHECK: xr [[ROT]], %r3
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK: br %r14
 ;
@@ -56,7 +56,7 @@ define i16 @f2(i16 *%src) {
 ; CHECK: xilf [[ROT]], 2147483648
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0([[NEGSHIFT:%r[1-9]+]])
 ; CHECK: cs [[OLD]], [[NEW]], 0(%r2)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK: br %r14
 ;
diff --git a/test/CodeGen/SystemZ/atomicrmw-xor-03.ll b/test/CodeGen/SystemZ/atomicrmw-xor-03.ll
index 05754e7a0e..d719d0bd71 100644
--- a/test/CodeGen/SystemZ/atomicrmw-xor-03.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-xor-03.ll
@@ -10,7 +10,7 @@ define i32 @f1(i32 %dummy, i32 *%src, i32 %b) {
 ; CHECK: lr %r0, %r2
 ; CHECK: xr %r0, %r4
 ; CHECK: cs %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw xor i32 *%src, i32 %b seq_cst
   ret i32 %res
@@ -24,7 +24,7 @@ define i32 @f2(i32 %dummy, i32 *%src) {
 ; CHECK: lr %r0, %r2
 ; CHECK: xilf %r0, 1
 ; CHECK: cs %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw xor i32 *%src, i32 1 seq_cst
   ret i32 %res
diff --git a/test/CodeGen/SystemZ/atomicrmw-xor-04.ll b/test/CodeGen/SystemZ/atomicrmw-xor-04.ll
index 1e438bd2dc..c17a879f37 100644
--- a/test/CodeGen/SystemZ/atomicrmw-xor-04.ll
+++ b/test/CodeGen/SystemZ/atomicrmw-xor-04.ll
@@ -10,7 +10,7 @@ define i64 @f1(i64 %dummy, i64 *%src, i64 %b) {
 ; CHECK: lgr %r0, %r2
 ; CHECK: xgr %r0, %r4
 ; CHECK: csg %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw xor i64 *%src, i64 %b seq_cst
   ret i64 %res
@@ -24,7 +24,7 @@ define i64 @f2(i64 %dummy, i64 *%src) {
 ; CHECK: lgr %r0, %r2
 ; CHECK: xilf %r0, 1
 ; CHECK: csg %r2, %r0, 0(%r3)
-; CHECK: jlh [[LABEL]]
+; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw xor i64 *%src, i64 1 seq_cst
   ret i64 %res
diff --git a/test/CodeGen/SystemZ/branch-08.ll b/test/CodeGen/SystemZ/branch-08.ll
index ee3f5b5f53..c4dc4673dc 100644
--- a/test/CodeGen/SystemZ/branch-08.ll
+++ b/test/CodeGen/SystemZ/branch-08.ll
@@ -9,7 +9,7 @@ declare void @foo() noreturn
 define i32 @f1(i32 %a, i32 %b) {
 ; CHECK-LABEL: f1:
 ; CHECK: clr %r2, %r3
-; CHECK: jnhe .L[[LABEL:.*]]
+; CHECK: jl .L[[LABEL:.*]]
 ; CHECK: br %r14
 ; CHECK: .L[[LABEL]]:
 ; CHECK: brasl %r14, foo@PLT
@@ -28,7 +28,7 @@ return:
 ; Same again with a fused compare and branch.
 define i32 @f2(i32 %a) {
 ; CHECK-LABEL: f2:
-; CHECK: cijnlh %r2, 0, .L[[LABEL:.*]]
+; CHECK: cije %r2, 0, .L[[LABEL:.*]]
 ; CHECK: br %r14
 ; CHECK: .L[[LABEL]]:
 ; CHECK: brasl %r14, foo@PLT
diff --git a/test/CodeGen/SystemZ/cmpxchg-01.ll b/test/CodeGen/SystemZ/cmpxchg-01.ll
index 467fe68be3..d5ea977869 100644
--- a/test/CodeGen/SystemZ/cmpxchg-01.ll
+++ b/test/CodeGen/SystemZ/cmpxchg-01.ll
@@ -22,7 +22,7 @@ define i8 @f1(i8 %dummy, i8 *%src, i8 %cmp, i8 %swap) {
 ; CHECK-MAIN: risbg %r5, %r2, 32, 55, 0
 ; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -8({{%r[1-9]+}})
 ; CHECK-MAIN: cs [[OLD]], [[NEW]], 0(%r3)
-; CHECK-MAIN: jlh [[LOOP]]
+; CHECK-MAIN: jl [[LOOP]]
 ; CHECK-MAIN: [[EXIT]]:
 ; CHECK-MAIN-NOT: %r2
 ; CHECK-MAIN: br %r14
diff --git a/test/CodeGen/SystemZ/cmpxchg-02.ll b/test/CodeGen/SystemZ/cmpxchg-02.ll
index c7ee898ce2..08c79d717c 100644
--- a/test/CodeGen/SystemZ/cmpxchg-02.ll
+++ b/test/CodeGen/SystemZ/cmpxchg-02.ll
@@ -22,7 +22,7 @@ define i16 @f1(i16 %dummy, i16 *%src, i16 %cmp, i16 %swap) {
 ; CHECK-MAIN: risbg %r5, %r2, 32, 47, 0
 ; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -16({{%r[1-9]+}})
 ; CHECK-MAIN: cs [[OLD]], [[NEW]], 0(%r3)
-; CHECK-MAIN: jlh [[LOOP]]
+; CHECK-MAIN: jl [[LOOP]]
 ; CHECK-MAIN: [[EXIT]]:
 ; CHECK-MAIN-NOT: %r2
 ; CHECK-MAIN: br %r14
diff --git a/test/CodeGen/SystemZ/cond-move-01.ll b/test/CodeGen/SystemZ/cond-move-01.ll
index 3ddc820dc1..088dee0232 100644
--- a/test/CodeGen/SystemZ/cond-move-01.ll
+++ b/test/CodeGen/SystemZ/cond-move-01.ll
@@ -6,7 +6,7 @@
 define i32 @f1(i32 %a, i32 %b, i32 %limit) {
 ; CHECK-LABEL: f1:
 ; CHECK: clfi %r4, 42
-; CHECK: locrnl %r2, %r3
+; CHECK: locrhe %r2, %r3
 ; CHECK: br %r14
   %cond = icmp ult i32 %limit, 42
   %res = select i1 %cond, i32 %a, i32 %b
@@ -17,9 +17,32 @@ define i32 @f1(i32 %a, i32 %b, i32 %limit) {
 define i64 @f2(i64 %a, i64 %b, i64 %limit) {
 ; CHECK-LABEL: f2:
 ; CHECK: clgfi %r4, 42
-; CHECK: locgrnl %r2, %r3
+; CHECK: locgrhe %r2, %r3
 ; CHECK: br %r14
   %cond = icmp ult i64 %limit, 42
   %res = select i1 %cond, i64 %a, i64 %b
   ret i64 %res
 }
+
+; Test LOCR in a case that could use COMPARE AND BRANCH.  We prefer using
+; LOCR if possible.
+define i32 @f3(i32 %a, i32 %b, i32 %limit) {
+; CHECK-LABEL: f3:
+; CHECK: chi %r4, 42
+; CHECK: locrlh %r2, %r3
+; CHECK: br %r14
+  %cond = icmp eq i32 %limit, 42
+  %res = select i1 %cond, i32 %a, i32 %b
+  ret i32 %res
+}
+
+; ...and again for LOCGR.
+define i64 @f4(i64 %a, i64 %b, i64 %limit) {
+; CHECK-LABEL: f4:
+; CHECK: cghi %r4, 42
+; CHECK: locgrlh %r2, %r3
+; CHECK: br %r14
+  %cond = icmp eq i64 %limit, 42
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
diff --git a/test/CodeGen/SystemZ/cond-store-01.ll b/test/CodeGen/SystemZ/cond-store-01.ll
index a734eeeb41..80e6d91614 100644
--- a/test/CodeGen/SystemZ/cond-store-01.ll
+++ b/test/CodeGen/SystemZ/cond-store-01.ll
@@ -24,7 +24,7 @@ define void @f1(i8 *%ptr, i8 %alt, i32 %limit) {
 define void @f2(i8 *%ptr, i8 %alt, i32 %limit) {
 ; CHECK-LABEL: f2:
 ; CHECK-NOT: %r2
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK-NOT: %r2
 ; CHECK: stc %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
@@ -59,7 +59,7 @@ define void @f3(i8 *%ptr, i32 %alt, i32 %limit) {
 define void @f4(i8 *%ptr, i32 %alt, i32 %limit) {
 ; CHECK-LABEL: f4:
 ; CHECK-NOT: %r2
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK-NOT: %r2
 ; CHECK: stc %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
@@ -96,7 +96,7 @@ define void @f5(i8 *%ptr, i32 %alt, i32 %limit) {
 define void @f6(i8 *%ptr, i32 %alt, i32 %limit) {
 ; CHECK-LABEL: f6:
 ; CHECK-NOT: %r2
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK-NOT: %r2
 ; CHECK: stc %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
@@ -133,7 +133,7 @@ define void @f7(i8 *%ptr, i64 %alt, i32 %limit) {
 define void @f8(i8 *%ptr, i64 %alt, i32 %limit) {
 ; CHECK-LABEL: f8:
 ; CHECK-NOT: %r2
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK-NOT: %r2
 ; CHECK: stc %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
@@ -170,7 +170,7 @@ define void @f9(i8 *%ptr, i64 %alt, i32 %limit) {
 define void @f10(i8 *%ptr, i64 %alt, i32 %limit) {
 ; CHECK-LABEL: f10:
 ; CHECK-NOT: %r2
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK-NOT: %r2
 ; CHECK: stc %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
@@ -327,7 +327,7 @@ define void @f18(i8 *%ptr, i8 %alt, i32 %limit) {
 ; ...likewise stores.  In this case we should have a conditional load into %r3.
 define void @f19(i8 *%ptr, i8 %alt, i32 %limit) {
 ; CHECK-LABEL: f19:
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK: lb %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
 ; CHECK: stc %r3, 0(%r2)
@@ -347,6 +347,7 @@ define void @f20(i8 *%ptr, i8 %alt, i32 %limit) {
 ; FIXME: should use a normal load instead of CS.
 ; CHECK-LABEL: f20:
 ; CHECK: cs {{%r[0-9]+}},
+; CHECK: jl
 ; CHECK: {{jl|jnl}} [[LABEL:[^ ]*]]
 ; CHECK: [[LABEL]]:
 ; CHECK: stc {{%r[0-9]+}},
@@ -362,7 +363,7 @@ define void @f20(i8 *%ptr, i8 %alt, i32 %limit) {
 define void @f21(i8 *%ptr, i8 %alt, i32 %limit) {
 ; FIXME: should use a normal store instead of CS.
 ; CHECK-LABEL: f21:
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK: lb %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
 ; CHECK: cs {{%r[0-9]+}},
diff --git a/test/CodeGen/SystemZ/cond-store-02.ll b/test/CodeGen/SystemZ/cond-store-02.ll
index 58550bf192..e01a8531bc 100644
--- a/test/CodeGen/SystemZ/cond-store-02.ll
+++ b/test/CodeGen/SystemZ/cond-store-02.ll
@@ -24,7 +24,7 @@ define void @f1(i16 *%ptr, i16 %alt, i32 %limit) {
 define void @f2(i16 *%ptr, i16 %alt, i32 %limit) {
 ; CHECK-LABEL: f2:
 ; CHECK-NOT: %r2
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK-NOT: %r2
 ; CHECK: sth %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
@@ -59,7 +59,7 @@ define void @f3(i16 *%ptr, i32 %alt, i32 %limit) {
 define void @f4(i16 *%ptr, i32 %alt, i32 %limit) {
 ; CHECK-LABEL: f4:
 ; CHECK-NOT: %r2
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK-NOT: %r2
 ; CHECK: sth %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
@@ -96,7 +96,7 @@ define void @f5(i16 *%ptr, i32 %alt, i32 %limit) {
 define void @f6(i16 *%ptr, i32 %alt, i32 %limit) {
 ; CHECK-LABEL: f6:
 ; CHECK-NOT: %r2
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK-NOT: %r2
 ; CHECK: sth %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
@@ -133,7 +133,7 @@ define void @f7(i16 *%ptr, i64 %alt, i32 %limit) {
 define void @f8(i16 *%ptr, i64 %alt, i32 %limit) {
 ; CHECK-LABEL: f8:
 ; CHECK-NOT: %r2
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK-NOT: %r2
 ; CHECK: sth %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
@@ -170,7 +170,7 @@ define void @f9(i16 *%ptr, i64 %alt, i32 %limit) {
 define void @f10(i16 *%ptr, i64 %alt, i32 %limit) {
 ; CHECK-LABEL: f10:
 ; CHECK-NOT: %r2
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK-NOT: %r2
 ; CHECK: sth %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
@@ -327,7 +327,7 @@ define void @f18(i16 *%ptr, i16 %alt, i32 %limit) {
 ; ...likewise stores.  In this case we should have a conditional load into %r3.
 define void @f19(i16 *%ptr, i16 %alt, i32 %limit) {
 ; CHECK-LABEL: f19:
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK: lh %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
 ; CHECK: sth %r3, 0(%r2)
@@ -347,6 +347,7 @@ define void @f20(i16 *%ptr, i16 %alt, i32 %limit) {
 ; FIXME: should use a normal load instead of CS.
 ; CHECK-LABEL: f20:
 ; CHECK: cs {{%r[0-9]+}},
+; CHECK: jl
 ; CHECK: {{jl|jnl}} [[LABEL:[^ ]*]]
 ; CHECK: [[LABEL]]:
 ; CHECK: sth {{%r[0-9]+}},
@@ -362,7 +363,7 @@ define void @f20(i16 *%ptr, i16 %alt, i32 %limit) {
 define void @f21(i16 *%ptr, i16 %alt, i32 %limit) {
 ; FIXME: should use a normal store instead of CS.
 ; CHECK-LABEL: f21:
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK: lh %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
 ; CHECK: cs {{%r[0-9]+}},
diff --git a/test/CodeGen/SystemZ/cond-store-03.ll b/test/CodeGen/SystemZ/cond-store-03.ll
index e2aeecfd57..e122bc26ac 100644
--- a/test/CodeGen/SystemZ/cond-store-03.ll
+++ b/test/CodeGen/SystemZ/cond-store-03.ll
@@ -24,7 +24,7 @@ define void @f1(i32 *%ptr, i32 %alt, i32 %limit) {
 define void @f2(i32 *%ptr, i32 %alt, i32 %limit) {
 ; CHECK-LABEL: f2:
 ; CHECK-NOT: %r2
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK-NOT: %r2
 ; CHECK: st %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
@@ -59,7 +59,7 @@ define void @f3(i32 *%ptr, i64 %alt, i32 %limit) {
 define void @f4(i32 *%ptr, i64 %alt, i32 %limit) {
 ; CHECK-LABEL: f4:
 ; CHECK-NOT: %r2
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK-NOT: %r2
 ; CHECK: st %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
@@ -96,7 +96,7 @@ define void @f5(i32 *%ptr, i64 %alt, i32 %limit) {
 define void @f6(i32 *%ptr, i64 %alt, i32 %limit) {
 ; CHECK-LABEL: f6:
 ; CHECK-NOT: %r2
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK-NOT: %r2
 ; CHECK: st %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
@@ -253,7 +253,7 @@ define void @f14(i32 *%ptr, i32 %alt, i32 %limit) {
 ; ...likewise stores.  In this case we should have a conditional load into %r3.
 define void @f15(i32 *%ptr, i32 %alt, i32 %limit) {
 ; CHECK-LABEL: f15:
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK: l %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
 ; CHECK: st %r3, 0(%r2)
@@ -288,7 +288,7 @@ define void @f16(i32 *%ptr, i32 %alt, i32 %limit) {
 define void @f17(i32 *%ptr, i32 %alt, i32 %limit) {
 ; FIXME: should use a normal store instead of CS.
 ; CHECK-LABEL: f17:
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK: l %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
 ; CHECK: cs {{%r[0-5]}}, %r3, 0(%r2)
diff --git a/test/CodeGen/SystemZ/cond-store-04.ll b/test/CodeGen/SystemZ/cond-store-04.ll
index 1830f27ebd..4ed23a364a 100644
--- a/test/CodeGen/SystemZ/cond-store-04.ll
+++ b/test/CodeGen/SystemZ/cond-store-04.ll
@@ -24,7 +24,7 @@ define void @f1(i64 *%ptr, i64 %alt, i32 %limit) {
 define void @f2(i64 *%ptr, i64 %alt, i32 %limit) {
 ; CHECK-LABEL: f2:
 ; CHECK-NOT: %r2
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK-NOT: %r2
 ; CHECK: stg %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
@@ -145,7 +145,7 @@ define void @f8(i64 *%ptr, i64 %alt, i32 %limit) {
 ; ...likewise stores.  In this case we should have a conditional load into %r3.
 define void @f9(i64 *%ptr, i64 %alt, i32 %limit) {
 ; CHECK-LABEL: f9:
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK: lg %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
 ; CHECK: stg %r3, 0(%r2)
@@ -180,7 +180,7 @@ define void @f10(i64 *%ptr, i64 %alt, i32 %limit) {
 define void @f11(i64 *%ptr, i64 %alt, i32 %limit) {
 ; FIXME: should use a normal store instead of CSG.
 ; CHECK-LABEL: f11:
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK: lg %r3, 0(%r2)
 ; CHECK: [[LABEL]]:
 ; CHECK: csg {{%r[0-5]}}, %r3, 0(%r2)
diff --git a/test/CodeGen/SystemZ/cond-store-05.ll b/test/CodeGen/SystemZ/cond-store-05.ll
index 131d1020cc..e41c8fe4f6 100644
--- a/test/CodeGen/SystemZ/cond-store-05.ll
+++ b/test/CodeGen/SystemZ/cond-store-05.ll
@@ -24,7 +24,7 @@ define void @f1(float *%ptr, float %alt, i32 %limit) {
 define void @f2(float *%ptr, float %alt, i32 %limit) {
 ; CHECK-LABEL: f2:
 ; CHECK-NOT: %r2
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK-NOT: %r2
 ; CHECK: ste %f0, 0(%r2)
 ; CHECK: [[LABEL]]:
@@ -179,7 +179,7 @@ define void @f10(float *%ptr, float %alt, i32 %limit) {
 ; ...likewise stores.  In this case we should have a conditional load into %f0.
 define void @f11(float *%ptr, float %alt, i32 %limit) {
 ; CHECK-LABEL: f11:
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK: le %f0, 0(%r2)
 ; CHECK: [[LABEL]]:
 ; CHECK: ste %f0, 0(%r2)
diff --git a/test/CodeGen/SystemZ/cond-store-06.ll b/test/CodeGen/SystemZ/cond-store-06.ll
index 1c2d716e5f..759a3e0561 100644
--- a/test/CodeGen/SystemZ/cond-store-06.ll
+++ b/test/CodeGen/SystemZ/cond-store-06.ll
@@ -24,7 +24,7 @@ define void @f1(double *%ptr, double %alt, i32 %limit) {
 define void @f2(double *%ptr, double %alt, i32 %limit) {
 ; CHECK-LABEL: f2:
 ; CHECK-NOT: %r2
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK-NOT: %r2
 ; CHECK: std %f0, 0(%r2)
 ; CHECK: [[LABEL]]:
@@ -179,7 +179,7 @@ define void @f10(double *%ptr, double %alt, i32 %limit) {
 ; ...likewise stores.  In this case we should have a conditional load into %f0.
 define void @f11(double *%ptr, double %alt, i32 %limit) {
 ; CHECK-LABEL: f11:
-; CHECK: jnl [[LABEL:[^ ]*]]
+; CHECK: jhe [[LABEL:[^ ]*]]
 ; CHECK: ld %f0, 0(%r2)
 ; CHECK: [[LABEL]]:
 ; CHECK: std %f0, 0(%r2)
diff --git a/test/CodeGen/SystemZ/cond-store-07.ll b/test/CodeGen/SystemZ/cond-store-07.ll
index 291360b96a..b1df525566 100644
--- a/test/CodeGen/SystemZ/cond-store-07.ll
+++ b/test/CodeGen/SystemZ/cond-store-07.ll
@@ -8,7 +8,7 @@ declare void @foo(i32 *)
 define void @f1(i32 *%ptr, i32 %alt, i32 %limit) {
 ; CHECK-LABEL: f1:
 ; CHECK: clfi %r4, 42
-; CHECK: stocnl %r3, 0(%r2)
+; CHECK: stoche %r3, 0(%r2)
 ; CHECK: br %r14
   %cond = icmp ult i32 %limit, 42
   %orig = load i32 *%ptr
@@ -35,7 +35,7 @@ define void @f2(i32 *%ptr, i32 %alt, i32 %limit) {
 define void @f3(i32 *%ptr, i64 %alt, i32 %limit) {
 ; CHECK-LABEL: f3:
 ; CHECK: clfi %r4, 42
-; CHECK: stocnl %r3, 0(%r2)
+; CHECK: stoche %r3, 0(%r2)
 ; CHECK: br %r14
   %cond = icmp ult i32 %limit, 42
   %orig = load i32 *%ptr
@@ -66,7 +66,7 @@ define void @f4(i32 *%ptr, i64 %alt, i32 %limit) {
 define void @f5(i32 *%ptr, i64 %alt, i32 %limit) {
 ; CHECK-LABEL: f5:
 ; CHECK: clfi %r4, 42
-; CHECK: stocnl %r3, 0(%r2)
+; CHECK: stoche %r3, 0(%r2)
 ; CHECK: br %r14
   %cond = icmp ult i32 %limit, 42
   %orig = load i32 *%ptr
@@ -96,7 +96,7 @@ define void @f6(i32 *%ptr, i64 %alt, i32 %limit) {
 define void @f7(i32 *%base, i32 %alt, i32 %limit) {
 ; CHECK-LABEL: f7:
 ; CHECK: clfi %r4, 42
-; CHECK: stocnl %r3, 524284(%r2)
+; CHECK: stoche %r3, 524284(%r2)
 ; CHECK: br %r14
   %ptr = getelementptr i32 *%base, i64 131071
   %cond = icmp ult i32 %limit, 42
@@ -111,7 +111,7 @@ define void @f8(i32 *%base, i32 %alt, i32 %limit) {
 ; CHECK-LABEL: f8:
 ; CHECK: agfi %r2, 524288
 ; CHECK: clfi %r4, 42
-; CHECK: stocnl %r3, 0(%r2)
+; CHECK: stoche %r3, 0(%r2)
 ; CHECK: br %r14
   %ptr = getelementptr i32 *%base, i64 131072
   %cond = icmp ult i32 %limit, 42
@@ -125,7 +125,7 @@ define void @f8(i32 *%base, i32 %alt, i32 %limit) {
 define void @f9(i32 *%base, i32 %alt, i32 %limit) {
 ; CHECK-LABEL: f9:
 ; CHECK: clfi %r4, 42
-; CHECK: stocnl %r3, -524288(%r2)
+; CHECK: stoche %r3, -524288(%r2)
 ; CHECK: br %r14
   %ptr = getelementptr i32 *%base, i64 -131072
   %cond = icmp ult i32 %limit, 42
@@ -140,7 +140,7 @@ define void @f10(i32 *%base, i32 %alt, i32 %limit) {
 ; CHECK-LABEL: f10:
 ; CHECK: agfi %r2, -524292
 ; CHECK: clfi %r4, 42
-; CHECK: stocnl %r3, 0(%r2)
+; CHECK: stoche %r3, 0(%r2)
 ; CHECK: br %r14
   %ptr = getelementptr i32 *%base, i64 -131073
   %cond = icmp ult i32 %limit, 42
@@ -154,7 +154,7 @@ define void @f10(i32 *%base, i32 %alt, i32 %limit) {
 define void @f11(i32 %alt, i32 %limit) {
 ; CHECK-LABEL: f11:
 ; CHECK: brasl %r14, foo@PLT
-; CHECK: stocnl {{%r[0-9]+}}, {{[0-9]+}}(%r15)
+; CHECK: stoche {{%r[0-9]+}}, {{[0-9]+}}(%r15)
 ; CHECK: brasl %r14, foo@PLT
 ; CHECK: br %r14
   %ptr = alloca i32
diff --git a/test/CodeGen/SystemZ/cond-store-08.ll b/test/CodeGen/SystemZ/cond-store-08.ll
index d67281c5af..56dc7ee777 100644
--- a/test/CodeGen/SystemZ/cond-store-08.ll
+++ b/test/CodeGen/SystemZ/cond-store-08.ll
@@ -8,7 +8,7 @@ declare void @foo(i64 *)
 define void @f1(i64 *%ptr, i64 %alt, i32 %limit) {
 ; CHECK-LABEL: f1:
 ; CHECK: clfi %r4, 42
-; CHECK: stocgnl %r3, 0(%r2)
+; CHECK: stocghe %r3, 0(%r2)
 ; CHECK: br %r14
   %cond = icmp ult i32 %limit, 42
   %orig = load i64 *%ptr
@@ -34,7 +34,7 @@ define void @f2(i64 *%ptr, i64 %alt, i32 %limit) {
 define void @f3(i64 *%base, i64 %alt, i32 %limit) {
 ; CHECK-LABEL: f3:
 ; CHECK: clfi %r4, 42
-; CHECK: stocgnl %r3, 524280(%r2)
+; CHECK: stocghe %r3, 524280(%r2)
 ; CHECK: br %r14
   %ptr = getelementptr i64 *%base, i64 65535
   %cond = icmp ult i32 %limit, 42
@@ -49,7 +49,7 @@ define void @f4(i64 *%base, i64 %alt, i32 %limit) {
 ; CHECK-LABEL: f4:
 ; CHECK: agfi %r2, 524288
 ; CHECK: clfi %r4, 42
-; CHECK: stocgnl %r3, 0(%r2)
+; CHECK: stocghe %r3, 0(%r2)
 ; CHECK: br %r14
   %ptr = getelementptr i64 *%base, i64 65536
   %cond = icmp ult i32 %limit, 42
@@ -63,7 +63,7 @@ define void @f4(i64 *%base, i64 %alt, i32 %limit) {
 define void @f5(i64 *%base, i64 %alt, i32 %limit) {
 ; CHECK-LABEL: f5:
 ; CHECK: clfi %r4, 42
-; CHECK: stocgnl %r3, -524288(%r2)
+; CHECK: stocghe %r3, -524288(%r2)
 ; CHECK: br %r14
   %ptr = getelementptr i64 *%base, i64 -65536
   %cond = icmp ult i32 %limit, 42
@@ -78,7 +78,7 @@ define void @f6(i64 *%base, i64 %alt, i32 %limit) {
 ; CHECK-LABEL: f6:
 ; CHECK: agfi %r2, -524296
 ; CHECK: clfi %r4, 42
-; CHECK: stocgnl %r3, 0(%r2)
+; CHECK: stocghe %r3, 0(%r2)
 ; CHECK: br %r14
   %ptr = getelementptr i64 *%base, i64 -65537
   %cond = icmp ult i32 %limit, 42
@@ -92,7 +92,7 @@ define void @f6(i64 *%base, i64 %alt, i32 %limit) {
 define void @f7(i64 %alt, i32 %limit) {
 ; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, foo@PLT
-; CHECK: stocgnl {{%r[0-9]+}}, {{[0-9]+}}(%r15)
+; CHECK: stocghe {{%r[0-9]+}}, {{[0-9]+}}(%r15)
 ; CHECK: brasl %r14, foo@PLT
 ; CHECK: br %r14
   %ptr = alloca i64
diff --git a/test/CodeGen/SystemZ/int-cmp-02.ll b/test/CodeGen/SystemZ/int-cmp-02.ll
index a77711931a..26e1391434 100644
--- a/test/CodeGen/SystemZ/int-cmp-02.ll
+++ b/test/CodeGen/SystemZ/int-cmp-02.ll
@@ -166,11 +166,11 @@ define double @f11(double %a, double %b, i32 %i1, i64 %base, i64 %index) {
 ; critical edge %entry->%while.body, which lost the kills information for CC.
 define void @f12(i32 %a, i32 %b) {
 ; CHECK-LABEL: f12:
-; CHECK: crje %r2,
+; CHECK: cije %r2, 0
 ; CHECK: crjlh %r2,
 ; CHECK: br %r14
 entry:
-  %cmp11 = icmp eq i32 %a, %b
+  %cmp11 = icmp eq i32 %a, 0
   br i1 %cmp11, label %while.end, label %while.body
 
 while.body: