R600/SI: Custom lower SI_IF and SI_ELSE to avoid machine verifier errors

SI_IF and SI_ELSE are terminators which also produce a value.  For
these instructions ISel always inserts a COPY to move their value
to another basic block.  This COPY ends up between SI_(IF|ELSE)
and the S_BRANCH* instruction at the end of the block.

This breaks MachineBasicBlock::getFirstTerminator() and also the
machine verifier which assumes that terminators are grouped together at
the end of blocks.

To solve this we coalesce the copy away right after ISel to make sure
there are no instructions in between terminators at the end of blocks.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@207591 91177308-0d34-0410-b5e6-96231b3b80d8

2 files changed, 63 insertions, 15 deletions

diff --git a/lib/Target/R600/SIISelLowering.cpp b/lib/Target/R600/SIISelLowering.cpp
index a4c808ce20..8c686c9150 100644
--- a/lib/Target/R600/SIISelLowering.cpp
+++ b/lib/Target/R600/SIISelLowering.cpp
@@ -444,19 +444,48 @@ SDValue SITargetLowering::LowerFormalArguments(
   return Chain;
 }
 
+/// Usually ISel will insert a copy between terminator insturction that output
+/// a value and the S_BRANCH* at the end of the block.  This causes
+/// MachineBasicBlock::getFirstTerminator() to return the incorrect value,
+/// so we want to make sure there are no copies between terminators at the
+/// end of blocks.
+static void LowerTerminatorWithOutput(unsigned Opcode, MachineBasicBlock *BB,
+                                      MachineInstr *MI,
+                                      const TargetInstrInfo *TII,
+                                      MachineRegisterInfo &MRI) {
+  unsigned DstReg = MI->getOperand(0).getReg();
+  // Usually ISel will insert a copy between the SI_IF_NON_TERM instruction
+  // and the S_BRANCH* terminator.  We want to replace SI_IF_NO_TERM with
+  // SI_IF and we can't have any instructions between S_BRANCH* and SI_IF,
+  // since they are both terminators
+  assert(MRI.hasOneUse(DstReg));
+  MachineOperand &Use = *MRI.use_begin(DstReg);
+  MachineInstr *UseMI = Use.getParent();
+  assert(UseMI->getOpcode() == AMDGPU::COPY);
+
+  MRI.replaceRegWith(UseMI->getOperand(0).getReg(), DstReg);
+  UseMI->eraseFromParent();
+  BuildMI(*BB, BB->getFirstTerminator(), MI->getDebugLoc(),
+          TII->get(Opcode))
+          .addOperand(MI->getOperand(0))
+          .addOperand(MI->getOperand(1))
+          .addOperand(MI->getOperand(2));
+  MI->eraseFromParent();
+}
+
 MachineBasicBlock * SITargetLowering::EmitInstrWithCustomInserter(
     MachineInstr * MI, MachineBasicBlock * BB) const {
 
   MachineBasicBlock::iterator I = *MI;
+  const SIInstrInfo *TII =
+    static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
+  MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
 
   switch (MI->getOpcode()) {
   default:
     return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB);
   case AMDGPU::BRANCH: return BB;
   case AMDGPU::SI_ADDR64_RSRC: {
-    const SIInstrInfo *TII =
-      static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
-    MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
     unsigned SuperReg = MI->getOperand(0).getReg();
     unsigned SubRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass);
     unsigned SubRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass);
@@ -481,9 +510,13 @@ MachineBasicBlock * SITargetLowering::EmitInstrWithCustomInserter(
     MI->eraseFromParent();
     break;
   }
-  case AMDGPU::V_SUB_F64: {
-    const SIInstrInfo *TII =
-      static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
+  case AMDGPU::SI_IF_NON_TERM:
+    LowerTerminatorWithOutput(AMDGPU::SI_IF, BB, MI, TII, MRI);
+    break;
+  case AMDGPU::SI_ELSE_NON_TERM:
+    LowerTerminatorWithOutput(AMDGPU::SI_ELSE, BB, MI, TII, MRI);
+    break;
+  case AMDGPU::V_SUB_F64:
     BuildMI(*BB, I, MI->getDebugLoc(), TII->get(AMDGPU::V_ADD_F64),
             MI->getOperand(0).getReg())
             .addReg(MI->getOperand(1).getReg())
@@ -495,11 +528,9 @@ MachineBasicBlock * SITargetLowering::EmitInstrWithCustomInserter(
             .addImm(2); /* NEG */
     MI->eraseFromParent();
     break;
-  }
+
   case AMDGPU::SI_RegisterStorePseudo: {
     MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
-    const SIInstrInfo *TII =
-      static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
     unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
     MachineInstrBuilder MIB =
         BuildMI(*BB, I, MI->getDebugLoc(), TII->get(AMDGPU::SI_RegisterStore),
diff --git a/lib/Target/R600/SIInstructions.td b/lib/Target/R600/SIInstructions.td
index a8aefc2287..00f9be61e2 100644
--- a/lib/Target/R600/SIInstructions.td
+++ b/lib/Target/R600/SIInstructions.td
@@ -1411,21 +1411,38 @@ def LOAD_CONST : AMDGPUShaderInst <
 let mayLoad = 1, mayStore = 1, hasSideEffects = 1,
     Uses = [EXEC], Defs = [EXEC] in {
 
+let usesCustomInserter = 1 in {
+
+def SI_IF_NON_TERM : InstSI <
+  (outs SReg_64:$dst),
+  (ins SReg_64:$vcc, brtarget:$target), "",
+  [(set i64:$dst, (int_SI_if i1:$vcc, bb:$target))]
+>;
+
+def SI_ELSE_NON_TERM : InstSI <
+  (outs SReg_64:$dst),
+  (ins SReg_64:$src, brtarget:$target),
+  "",
+  [(set i64:$dst, (int_SI_else i64:$src, bb:$target))]
+> {
+  let Constraints = "$src = $dst";
+}
+
+} // usesCustomInserter = 1
+
 let isBranch = 1, isTerminator = 1 in {
 
-def SI_IF : InstSI <
+def SI_IF: InstSI <
   (outs SReg_64:$dst),
   (ins SReg_64:$vcc, brtarget:$target),
-  "SI_IF $dst, $vcc, $target",
-  [(set i64:$dst, (int_SI_if i1:$vcc, bb:$target))]
+  "", []
 >;
 
 def SI_ELSE : InstSI <
   (outs SReg_64:$dst),
   (ins SReg_64:$src, brtarget:$target),
-  "SI_ELSE $dst, $src, $target",
-  [(set i64:$dst, (int_SI_else i64:$src, bb:$target))]> {
-
+  "", []
+> {
   let Constraints = "$src = $dst";
 }