Move the point at which FastISel taps into the SelectionDAGISel

process up to a higher level. This allows FastISel to leverage
more of SelectionDAGISel's infastructure, such as updating Machine
PHI nodes.

Also, implement transitioning from SDISel back to FastISel in
the middle of a block, so it's now possible to go back and
forth. This allows FastISel to hand individual CallInsts and other
complicated things off to SDISel to handle, while handling the rest
of the block itself.

To help support this, reorganize the SelectionDAG class so that it
is allocated once and reused throughout a function, instead of
being completely reallocated for each block.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@55219 91177308-0d34-0410-b5e6-96231b3b80d8

14 files changed, 366 insertions, 332 deletions

diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h
index a19765a644..01aba3f985 100644
--- a/include/llvm/CodeGen/SelectionDAG.h
+++ b/include/llvm/CodeGen/SelectionDAG.h
@@ -36,13 +36,6 @@ class MachineFunction;
 class MachineConstantPoolValue;
 class FunctionLoweringInfo;
 
-/// NodeAllocatorType - The AllocatorType for allocating SDNodes. We use
-/// pool allocation with recycling.
-///
-typedef RecyclingAllocator<BumpPtrAllocator, SDNode, sizeof(LargestSDNode),
-                           AlignOf<MostAlignedSDNode>::Alignment>
-  NodeAllocatorType;
-
 template<> class ilist_traits<SDNode> : public ilist_default_traits<SDNode> {
   mutable SDNode Sentinel;
 public:
@@ -77,21 +70,31 @@ class SelectionDAG {
   FunctionLoweringInfo &FLI;
   MachineModuleInfo *MMI;
 
-  /// Root - The root of the entire DAG.  EntryNode - The starting token.
-  SDValue Root, EntryNode;
+  /// EntryNode - The starting token.
+  SDNode EntryNode;
+
+  /// Root - The root of the entire DAG.
+  SDValue Root;
 
   /// AllNodes - A linked list of nodes in the current DAG.
   ilist<SDNode> AllNodes;
 
-  /// NodeAllocator - Pool allocation for nodes. The allocator isn't
-  /// allocated inside this class because we want to reuse a single
-  /// recycler across multiple SelectionDAG runs.
-  NodeAllocatorType &NodeAllocator;
+  /// NodeAllocatorType - The AllocatorType for allocating SDNodes. We use
+  /// pool allocation with recycling.
+  typedef RecyclingAllocator<BumpPtrAllocator, SDNode, sizeof(LargestSDNode),
+                             AlignOf<MostAlignedSDNode>::Alignment>
+    NodeAllocatorType;
+
+  /// NodeAllocator - Pool allocation for nodes.
+  NodeAllocatorType NodeAllocator;
 
   /// CSEMap - This structure is used to memoize nodes, automatically performing
   /// CSE with existing nodes with a duplicate is requested.
   FoldingSet<SDNode> CSEMap;
 
+  /// OperandAllocator - Pool allocation for machine-opcode SDNode operands.
+  BumpPtrAllocator OperandAllocator;
+
   /// Allocator - Pool allocation for misc. objects that are created once per
   /// SelectionDAG.
   BumpPtrAllocator Allocator;
@@ -101,10 +104,14 @@ class SelectionDAG {
 
 public:
   SelectionDAG(TargetLowering &tli, MachineFunction &mf, 
-               FunctionLoweringInfo &fli, MachineModuleInfo *mmi,
-               NodeAllocatorType &nodeallocator);
+               FunctionLoweringInfo &fli, MachineModuleInfo *mmi);
   ~SelectionDAG();
 
+  /// reset - Clear state and free memory necessary to make this
+  /// SelectionDAG ready to process a new block.
+  ///
+  void reset();
+
   MachineFunction &getMachineFunction() const { return MF; }
   const TargetMachine &getTarget() const;
   TargetLowering &getTargetLoweringInfo() const { return TLI; }
@@ -152,7 +159,9 @@ public:
 
   /// getEntryNode - Return the token chain corresponding to the entry of the
   /// function.
-  const SDValue &getEntryNode() const { return EntryNode; }
+  SDValue getEntryNode() const {
+    return SDValue(const_cast<SDNode *>(&EntryNode), 0);
+  }
 
   /// setRoot - Set the current root tag of the SelectionDAG.
   ///
@@ -721,6 +730,8 @@ private:
   void DeleteNodeNotInCSEMaps(SDNode *N);
 
   unsigned getMVTAlignment(MVT MemoryVT) const;
+
+  void allnodes_clear();
   
   // List of non-single value types.
   std::vector<SDVTList> VTList;
diff --git a/include/llvm/CodeGen/SelectionDAGISel.h b/include/llvm/CodeGen/SelectionDAGISel.h
index 951efe1975..64c3b25591 100644
--- a/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/include/llvm/CodeGen/SelectionDAGISel.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_CODEGEN_SELECTIONDAG_ISEL_H
 #define LLVM_CODEGEN_SELECTIONDAG_ISEL_H
 
+#include "llvm/BasicBlock.h"
 #include "llvm/Pass.h"
 #include "llvm/Constant.h"
 #include "llvm/CodeGen/SelectionDAG.h"
@@ -58,7 +59,7 @@ public:
   unsigned MakeReg(MVT VT);
 
   virtual void EmitFunctionEntryCode(Function &Fn, MachineFunction &MF) {}
-  virtual void InstructionSelect(SelectionDAG &SD) = 0;
+  virtual void InstructionSelect() = 0;
   virtual void InstructionSelectPostProcessing() {}
   
   void SelectRootInit() {
@@ -72,8 +73,7 @@ public:
   /// OutOps vector.
   virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
                                             char ConstraintCode,
-                                            std::vector<SDValue> &OutOps,
-                                            SelectionDAG &DAG) {
+                                            std::vector<SDValue> &OutOps) {
     return true;
   }
 
@@ -168,8 +168,7 @@ protected:
 
   /// SelectInlineAsmMemoryOperands - Calls to this are automatically generated
   /// by tblgen.  Others should not call it.
-  void SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops,
-                                     SelectionDAG &DAG);
+  void SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops);
 
   // Calls to these predicates are generated by tblgen.
   bool CheckAndMask(SDValue LHS, ConstantSDNode *RHS,
@@ -180,26 +179,28 @@ protected:
 private:
   void SelectAllBasicBlocks(Function &Fn, MachineFunction &MF,
                             FunctionLoweringInfo &FuncInfo);
-  void SelectBasicBlock(BasicBlock *BB, MachineFunction &MF,
-                        FunctionLoweringInfo &FuncInfo,
-             std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
-                        NodeAllocatorType &NodeAllocator);
-  void FinishBasicBlock(BasicBlock *BB, MachineFunction &MF,
-                        FunctionLoweringInfo &FuncInfo,
-             std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
-                        NodeAllocatorType &NodeAllocator);
+  void FinishBasicBlock(FunctionLoweringInfo &FuncInfo,
+             std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate);
 
-  void BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB,
+  void SelectBasicBlock(BasicBlock *LLVMBB,
+                        BasicBlock::iterator Begin,
+                        BasicBlock::iterator End,
+                        bool DoArgs,
              std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
-                         FunctionLoweringInfo &FuncInfo);
-  void CodeGenAndEmitDAG(SelectionDAG &DAG);
+                        FunctionLoweringInfo &FuncInfo);
+  void CodeGenAndEmitDAG();
   void LowerArguments(BasicBlock *BB, SelectionDAGLowering &SDL);
   
-  void ComputeLiveOutVRegInfo(SelectionDAG &DAG);
+  void ComputeLiveOutVRegInfo();
+
+  void HandlePHINodesInSuccessorBlocks(BasicBlock *LLVMBB,
+                                       FunctionLoweringInfo &FuncInfo,
+             std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
+                                       SelectionDAGLowering &SDL);
 
   /// Pick a safe ordering for instructions for each target node in the
   /// graph.
-  ScheduleDAG *Schedule(SelectionDAG &DAG);
+  ScheduleDAG *Schedule();
 
   /// SwitchCases - Vector of CaseBlock structures used to communicate
   /// SwitchInst code generation information.
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 3e3890142f..9275a21f9f 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -547,11 +547,8 @@ void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
   // Drop all of the operands and decrement used nodes use counts.
   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I)
     I->getVal()->removeUser(std::distance(N->op_begin(), I), N);
-  if (N->OperandsNeedDelete) {
+  if (N->OperandsNeedDelete)
     delete[] N->OperandList;
-  }
-  N->OperandList = 0;
-  N->NumOperands = 0;
   
   AllNodes.remove(N);
 }
@@ -563,6 +560,9 @@ void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
 void SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
   bool Erased = false;
   switch (N->getOpcode()) {
+  case ISD::EntryToken:
+    assert(0 && "EntryToken should not be in CSEMaps!");
+    return;
   case ISD::HANDLENODE: return;  // noop.
   case ISD::CONDCODE:
     assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
@@ -764,24 +764,44 @@ unsigned SelectionDAG::getMVTAlignment(MVT VT) const {
 }
 
 SelectionDAG::SelectionDAG(TargetLowering &tli, MachineFunction &mf,
-                           FunctionLoweringInfo &fli, MachineModuleInfo *mmi,
-                           NodeAllocatorType &nodeallocator)
-  : TLI(tli), MF(mf), FLI(fli), MMI(mmi), NodeAllocator(nodeallocator) {
-  EntryNode = Root = getNode(ISD::EntryToken, MVT::Other);
+                           FunctionLoweringInfo &fli, MachineModuleInfo *mmi)
+  : TLI(tli), MF(mf), FLI(fli), MMI(mmi),
+    EntryNode(ISD::EntryToken, getVTList(MVT::Other)), 
+    Root(getEntryNode()) {
+  AllNodes.push_back(&EntryNode);
 }
 
 SelectionDAG::~SelectionDAG() {
+  allnodes_clear();
+}
+
+void SelectionDAG::allnodes_clear() {
   while (!AllNodes.empty()) {
     SDNode *N = AllNodes.remove(AllNodes.begin());
     N->SetNextInBucket(0);
-    if (N->OperandsNeedDelete) {
+    if (N->OperandsNeedDelete)
       delete [] N->OperandList;
-    }
-    N->OperandList = 0;
-    N->NumOperands = 0;
   }
 }
 
+void SelectionDAG::reset() {
+  allnodes_clear();
+  OperandAllocator.Reset();
+  CSEMap.clear();
+
+  ExtendedValueTypeNodes.clear();
+  ExternalSymbols.clear();
+  TargetExternalSymbols.clear();
+  std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
+            static_cast<CondCodeSDNode*>(0));
+  std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
+            static_cast<SDNode*>(0));
+
+  EntryNode.Uses = 0;
+  AllNodes.push_back(&EntryNode);
+  Root = getEntryNode();
+}
+
 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, MVT VT) {
   if (Op.getValueType() == VT) return Op;
   APInt Imm = APInt::getLowBitsSet(Op.getValueSizeInBits(),
@@ -3988,9 +4008,9 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
       delete[] N->OperandList;
     if (N->isMachineOpcode()) {
       // We're creating a final node that will live unmorphed for the
-      // remainder of this SelectionDAG's duration, so we can allocate the
-      // operands directly out of the pool with no recycling metadata.
-      N->OperandList = Allocator.Allocate<SDUse>(NumOps);
+      // remainder of the current SelectionDAG iteration, so we can allocate
+      // the operands directly out of a pool with no recycling metadata.
+      N->OperandList = OperandAllocator.Allocate<SDUse>(NumOps);
       N->OperandsNeedDelete = false;
     } else {
       N->OperandList = new SDUse[NumOps];
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 147f45c8b5..e789c4e41c 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -394,6 +394,11 @@ static bool isUsedOutsideOfDefiningBlock(Instruction *I) {
 /// entry block, return true.  This includes arguments used by switches, since
 /// the switch may expand into multiple basic blocks.
 static bool isOnlyUsedInEntryBlock(Argument *A) {
+  // With FastISel active, we may be splitting blocks, so force creation
+  // of virtual registers for all non-dead arguments.
+  if (EnableFastISel)
+    return A->use_empty();
+
   BasicBlock *Entry = A->getParent()->begin();
   for (Value::use_iterator UI = A->use_begin(), E = A->use_end(); UI != E; ++UI)
     if (cast<Instruction>(*UI)->getParent() != Entry || isa<SwitchInst>(*UI))
@@ -5094,119 +5099,18 @@ static void CheckDAGForTailCallsAndFixThem(SelectionDAG &DAG,
   }
 }
 
-void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB,
-       std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
-                                         FunctionLoweringInfo &FuncInfo) {
-  SelectionDAGLowering SDL(DAG, TLI, *AA, FuncInfo, GFI);
-  BB = FuncInfo.MBBMap[LLVMBB];
-
-  // Before doing SelectionDAG ISel, see if FastISel has been requested.
-  // FastISel doesn't currently support entry blocks, because that
-  // requires special handling for arguments. And it doesn't support EH
-  // landing pads, which also require special handling.
-  // For now, also exclude blocks with terminators that aren't
-  // unconditional branches.
-  BasicBlock::iterator Begin = LLVMBB->begin();
-  if (EnableFastISel &&
-      LLVMBB != &LLVMBB->getParent()->getEntryBlock() &&
-      !BB->isLandingPad() &&
-      isa<BranchInst>(LLVMBB->getTerminator()) &&
-      cast<BranchInst>(LLVMBB->getTerminator())->isUnconditional()) {
-    if (FastISel *F = TLI.createFastISel(FuncInfo.MF)) {
-      Begin = F->SelectInstructions(Begin, LLVMBB->end(),
-                                    FuncInfo.ValueMap, FuncInfo.MBBMap, BB);
-
-      // Clean up the FastISel object. TODO: Reorganize what data is
-      // stored in the FastISel class itself and what is merely passed
-      // to the SelectInstructions method, and then move the creation
-      // and deletion of the FastISel object up so that it is only
-      // done once per MachineFunction.
-      delete F;
-
-      if (Begin == LLVMBB->end())
-        // The "fast" selector selected the entire block, so we're done.
-        return;
-
-      if (!DisableFastISelAbort) {
-        // The "fast" selector couldn't handle something and bailed.
-        // For the purpose of debugging, just abort.
-#ifndef NDEBUG
-        Begin->dump();
-#endif
-        assert(0 && "FastISel didn't select the entire block");
-        abort();
-      }
-    }
-  }
-
-  // Lower any arguments needed in this block if this is the entry block.
-  if (LLVMBB == &LLVMBB->getParent()->getEntryBlock())
-    LowerArguments(LLVMBB, SDL);
-
-  SDL.setCurrentBasicBlock(BB);
-
-  MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
-
-  if (MMI && BB->isLandingPad()) {
-    // Add a label to mark the beginning of the landing pad.  Deletion of the
-    // landing pad can thus be detected via the MachineModuleInfo.
-    unsigned LabelID = MMI->addLandingPad(BB);
-    DAG.setRoot(DAG.getLabel(ISD::EH_LABEL, DAG.getEntryNode(), LabelID));
-
-    // Mark exception register as live in.
-    unsigned Reg = TLI.getExceptionAddressRegister();
-    if (Reg) BB->addLiveIn(Reg);
-
-    // Mark exception selector register as live in.
-    Reg = TLI.getExceptionSelectorRegister();
-    if (Reg) BB->addLiveIn(Reg);
-
-    // FIXME: Hack around an exception handling flaw (PR1508): the personality
-    // function and list of typeids logically belong to the invoke (or, if you
-    // like, the basic block containing the invoke), and need to be associated
-    // with it in the dwarf exception handling tables.  Currently however the
-    // information is provided by an intrinsic (eh.selector) that can be moved
-    // to unexpected places by the optimizers: if the unwind edge is critical,
-    // then breaking it can result in the intrinsics being in the successor of
-    // the landing pad, not the landing pad itself.  This results in exceptions
-    // not being caught because no typeids are associated with the invoke.
-    // This may not be the only way things can go wrong, but it is the only way
-    // we try to work around for the moment.
-    BranchInst *Br = dyn_cast<BranchInst>(LLVMBB->getTerminator());
-
-    if (Br && Br->isUnconditional()) { // Critical edge?
-      BasicBlock::iterator I, E;
-      for (I = LLVMBB->begin(), E = --LLVMBB->end(); I != E; ++I)
-        if (isSelector(I))
-          break;
-
-      if (I == E)
-        // No catch info found - try to extract some from the successor.
-        copyCatchInfo(Br->getSuccessor(0), LLVMBB, MMI, FuncInfo);
-    }
-  }
-
-  // Lower all of the non-terminator instructions.
-  for (BasicBlock::iterator I = Begin, E = --LLVMBB->end();
-       I != E; ++I)
-    SDL.visit(*I);
-
-  // Ensure that all instructions which are used outside of their defining
-  // blocks are available as virtual registers.  Invoke is handled elsewhere.
-  for (BasicBlock::iterator I = Begin, E = LLVMBB->end(); I != E;++I)
-    if (!I->use_empty() && !isa<PHINode>(I) && !isa<InvokeInst>(I)) {
-      DenseMap<const Value*, unsigned>::iterator VMI =FuncInfo.ValueMap.find(I);
-      if (VMI != FuncInfo.ValueMap.end())
-        SDL.CopyValueToVirtualRegister(I, VMI->second);
-    }
-
-  // Handle PHI nodes in successor blocks.  Emit code into the SelectionDAG to
-  // ensure constants are generated when needed.  Remember the virtual registers
-  // that need to be added to the Machine PHI nodes as input.  We cannot just
-  // directly add them, because expansion might result in multiple MBB's for one
-  // BB.  As such, the start of the BB might correspond to a different MBB than
-  // the end.
-  //
+/// Handle PHI nodes in successor blocks.  Emit code into the SelectionDAG to
+/// ensure constants are generated when needed.  Remember the virtual registers
+/// that need to be added to the Machine PHI nodes as input.  We cannot just
+/// directly add them, because expansion might result in multiple MBB's for one
+/// BB.  As such, the start of the BB might correspond to a different MBB than
+/// the end.
+///
+void
+SelectionDAGISel::HandlePHINodesInSuccessorBlocks(BasicBlock *LLVMBB,
+                                                 FunctionLoweringInfo &FuncInfo,
+             std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
+                                                  SelectionDAGLowering &SDL) {
   TerminatorInst *TI = LLVMBB->getTerminator();
 
   // Emit constants only once even if used by multiple PHI nodes.
@@ -5287,22 +5191,101 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB,
   JTCases = SDL.JTCases;
   BitTestCases.clear();
   BitTestCases = SDL.BitTestCases;
+}
+
+void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB,
+                                        BasicBlock::iterator Begin,
+                                        BasicBlock::iterator End,
+                                        bool DoArgs,
+       std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
+                                        FunctionLoweringInfo &FuncInfo) {
+  SelectionDAGLowering SDL(*CurDAG, TLI, *AA, FuncInfo, GFI);
+
+  // Lower any arguments needed in this block if this is the entry block.
+  if (DoArgs)
+    LowerArguments(LLVMBB, SDL);
+
+  SDL.setCurrentBasicBlock(BB);
+
+  MachineModuleInfo *MMI = CurDAG->getMachineModuleInfo();
+
+  if (MMI && BB->isLandingPad()) {
+    // Add a label to mark the beginning of the landing pad.  Deletion of the
+    // landing pad can thus be detected via the MachineModuleInfo.
+    unsigned LabelID = MMI->addLandingPad(BB);
+    CurDAG->setRoot(CurDAG->getLabel(ISD::EH_LABEL,
+                                     CurDAG->getEntryNode(), LabelID));
+
+    // Mark exception register as live in.
+    unsigned Reg = TLI.getExceptionAddressRegister();
+    if (Reg) BB->addLiveIn(Reg);
+
+    // Mark exception selector register as live in.
+    Reg = TLI.getExceptionSelectorRegister();
+    if (Reg) BB->addLiveIn(Reg);
+
+    // FIXME: Hack around an exception handling flaw (PR1508): the personality
+    // function and list of typeids logically belong to the invoke (or, if you
+    // like, the basic block containing the invoke), and need to be associated
+    // with it in the dwarf exception handling tables.  Currently however the
+    // information is provided by an intrinsic (eh.selector) that can be moved
+    // to unexpected places by the optimizers: if the unwind edge is critical,
+    // then breaking it can result in the intrinsics being in the successor of
+    // the landing pad, not the landing pad itself.  This results in exceptions
+    // not being caught because no typeids are associated with the invoke.
+    // This may not be the only way things can go wrong, but it is the only way
+    // we try to work around for the moment.
+    BranchInst *Br = dyn_cast<BranchInst>(LLVMBB->getTerminator());
+
+    if (Br && Br->isUnconditional()) { // Critical edge?
+      BasicBlock::iterator I, E;
+      for (I = LLVMBB->begin(), E = --LLVMBB->end(); I != E; ++I)
+        if (isSelector(I))
+          break;
+
+      if (I == E)
+        // No catch info found - try to extract some from the successor.
+        copyCatchInfo(Br->getSuccessor(0), LLVMBB, MMI, FuncInfo);
+    }
+  }
+
+  // Lower all of the non-terminator instructions.
+  for (BasicBlock::iterator I = Begin; I != End; ++I)
+    if (!isa<TerminatorInst>(I))
+      SDL.visit(*I);
+
+  // Ensure that all instructions which are used outside of their defining
+  // blocks are available as virtual registers.  Invoke is handled elsewhere.
+  for (BasicBlock::iterator I = Begin; I != End; ++I)
+    if (!I->use_empty() && !isa<PHINode>(I) && !isa<InvokeInst>(I)) {
+      DenseMap<const Value*, unsigned>::iterator VMI =FuncInfo.ValueMap.find(I);
+      if (VMI != FuncInfo.ValueMap.end())
+        SDL.CopyValueToVirtualRegister(I, VMI->second);
+    }
+
+  // Handle PHI nodes in successor blocks.
+  if (Begin != End && End == LLVMBB->end())
+    HandlePHINodesInSuccessorBlocks(LLVMBB, FuncInfo, PHINodesToUpdate, SDL);
     
   // Make sure the root of the DAG is up-to-date.
-  DAG.setRoot(SDL.getControlRoot());
+  CurDAG->setRoot(SDL.getControlRoot());
 
   // Check whether calls in this block are real tail calls. Fix up CALL nodes
   // with correct tailcall attribute so that the target can rely on the tailcall
   // attribute indicating whether the call is really eligible for tail call
   // optimization.
-  CheckDAGForTailCallsAndFixThem(DAG, TLI);
+  CheckDAGForTailCallsAndFixThem(*CurDAG, TLI);
+
+  // Final step, emit the lowered DAG as machine code.
+  CodeGenAndEmitDAG();
+  CurDAG->reset();
 }
 
-void SelectionDAGISel::ComputeLiveOutVRegInfo(SelectionDAG &DAG) {
+void SelectionDAGISel::ComputeLiveOutVRegInfo() {
   SmallPtrSet<SDNode*, 128> VisitedNodes;
   SmallVector<SDNode*, 128> Worklist;
   
-  Worklist.push_back(DAG.getRoot().Val);
+  Worklist.push_back(CurDAG->getRoot().Val);
   
   APInt Mask;
   APInt KnownZero;
@@ -5335,14 +5318,14 @@ void SelectionDAGISel::ComputeLiveOutVRegInfo(SelectionDAG &DAG) {
     if (!SrcVT.isInteger() || SrcVT.isVector())
       continue;
     
-    unsigned NumSignBits = DAG.ComputeNumSignBits(Src);
+    unsigned NumSignBits = CurDAG->ComputeNumSignBits(Src);
     Mask = APInt::getAllOnesValue(SrcVT.getSizeInBits());
-    DAG.ComputeMaskedBits(Src, Mask, KnownZero, KnownOne);
+    CurDAG->ComputeMaskedBits(Src, Mask, KnownZero, KnownOne);
     
     // Only install this information if it tells us something.
     if (NumSignBits != 1 || KnownZero != 0 || KnownOne != 0) {
       DestReg -= TargetRegisterInfo::FirstVirtualRegister;
-      FunctionLoweringInfo &FLI = DAG.getFunctionLoweringInfo();
+      FunctionLoweringInfo &FLI = CurDAG->getFunctionLoweringInfo();
       if (DestReg >= FLI.LiveOutRegInfo.size())
         FLI.LiveOutRegInfo.resize(DestReg+1);
       FunctionLoweringInfo::LiveOutInfo &LOI = FLI.LiveOutRegInfo[DestReg];
@@ -5353,102 +5336,102 @@ void SelectionDAGISel::ComputeLiveOutVRegInfo(SelectionDAG &DAG) {
   }
 }
 
-void SelectionDAGISel::CodeGenAndEmitDAG(SelectionDAG &DAG) {
+void SelectionDAGISel::CodeGenAndEmitDAG() {
   std::string GroupName;
   if (TimePassesIsEnabled)
     GroupName = "Instruction Selection and Scheduling";
   std::string BlockName;
   if (ViewDAGCombine1 || ViewLegalizeTypesDAGs || ViewLegalizeDAGs ||
       ViewDAGCombine2 || ViewISelDAGs || ViewSchedDAGs || ViewSUnitDAGs)
-    BlockName = DAG.getMachineFunction().getFunction()->getName() + ':' +
+    BlockName = CurDAG->getMachineFunction().getFunction()->getName() + ':' +
                 BB->getBasicBlock()->getName();
 
   DOUT << "Initial selection DAG:\n";
-  DEBUG(DAG.dump());
+  DEBUG(CurDAG->dump());
 
-  if (ViewDAGCombine1) DAG.viewGraph("dag-combine1 input for " + BlockName);
+  if (ViewDAGCombine1) CurDAG->viewGraph("dag-combine1 input for " + BlockName);
 
   // Run the DAG combiner in pre-legalize mode.
   if (TimePassesIsEnabled) {
     NamedRegionTimer T("DAG Combining 1", GroupName);
-    DAG.Combine(false, *AA, Fast);
+    CurDAG->Combine(false, *AA, Fast);
   } else {
-    DAG.Combine(false, *AA, Fast);
+    CurDAG->Combine(false, *AA, Fast);
   }
   
   DOUT << "Optimized lowered selection DAG:\n";
-  DEBUG(DAG.dump());
+  DEBUG(CurDAG->dump());
   
   // Second step, hack on the DAG until it only uses operations and types that
   // the target supports.
   if (EnableLegalizeTypes) {// Enable this some day.
-    if (ViewLegalizeTypesDAGs) DAG.viewGraph("legalize-types input for " +
-                                             BlockName);
+    if (ViewLegalizeTypesDAGs) CurDAG->viewGraph("legalize-types input for " +
+                                                 BlockName);
 
     if (TimePassesIsEnabled) {
       NamedRegionTimer T("Type Legalization", GroupName);
-      DAG.LegalizeTypes();
+      CurDAG->LegalizeTypes();
     } else {
-      DAG.LegalizeTypes();
+      CurDAG->LegalizeTypes();
     }
 
     DOUT << "Type-legalized selection DAG:\n";
-    DEBUG(DAG.dump());
+    DEBUG(CurDAG->dump());
 
     // TODO: enable a dag combine pass here.
   }
   
-  if (ViewLegalizeDAGs) DAG.viewGraph("legalize input for " + BlockName);
+  if (ViewLegalizeDAGs) CurDAG->viewGraph("legalize input for " + BlockName);
 
   if (TimePassesIsEnabled) {
     NamedRegionTimer T("DAG Legalization", GroupName);
-    DAG.Legalize();
+    CurDAG->Legalize();
   } else {
-    DAG.Legalize();
+    CurDAG->Legalize();
   }
   
   DOUT << "Legalized selection DAG:\n";
-  DEBUG(DAG.dump());
+  DEBUG(CurDAG->dump());
   
-  if (ViewDAGCombine2) DAG.viewGraph("dag-combine2 input for " + BlockName);
+  if (ViewDAGCombine2) CurDAG->viewGraph("dag-combine2 input for " + BlockName);
 
   // Run the DAG combiner in post-legalize mode.
   if (TimePassesIsEnabled) {
     NamedRegionTimer T("DAG Combining 2", GroupName);
-    DAG.Combine(true, *AA, Fast);
+    CurDAG->Combine(true, *AA, Fast);
   } else {
-    DAG.Combine(true, *AA, Fast);
+    CurDAG->Combine(true, *AA, Fast);
   }
   
   DOUT << "Optimized legalized selection DAG:\n";
-  DEBUG(DAG.dump());
+  DEBUG(CurDAG->dump());
 
-  if (ViewISelDAGs) DAG.viewGraph("isel input for " + BlockName);
+  if (ViewISelDAGs) CurDAG->viewGraph("isel input for " + BlockName);
   
   if (!Fast && EnableValueProp)
-    ComputeLiveOutVRegInfo(DAG);
+    ComputeLiveOutVRegInfo();
 
   // Third, instruction select all of the operations to machine code, adding the
   // code to the MachineBasicBlock.
   if (TimePassesIsEnabled) {
     NamedRegionTimer T("Instruction Selection", GroupName);
-    InstructionSelect(DAG);
+    InstructionSelect();
   } else {
-    InstructionSelect(DAG);
+    InstructionSelect();
   }
 
   DOUT << "Selected selection DAG:\n";
-  DEBUG(DAG.dump());
+  DEBUG(CurDAG->dump());
 
-  if (ViewSchedDAGs) DAG.viewGraph("scheduler input for " + BlockName);
+  if (ViewSchedDAGs) CurDAG->viewGraph("scheduler input for " + BlockName);
 
   // Schedule machine code.
   ScheduleDAG *Scheduler;
   if (TimePassesIsEnabled) {
     NamedRegionTimer T("Instruction Scheduling", GroupName);
-    Scheduler = Schedule(DAG);
+    Scheduler = Schedule();
   } else {
-    Scheduler = Schedule(DAG);
+    Scheduler = Schedule();
   }
 
   if (ViewSUnitDAGs) Scheduler->viewGraph();
@@ -5470,57 +5453,94 @@ void SelectionDAGISel::CodeGenAndEmitDAG(SelectionDAG &DAG) {
     delete Scheduler;
   }
 
-  // Perform target specific isel post processing.
-  if (TimePassesIsEnabled) {
-    NamedRegionTimer T("Instruction Selection Post Processing", GroupName);
-    InstructionSelectPostProcessing();
-  } else {
-    InstructionSelectPostProcessing();
-  }
-  
   DOUT << "Selected machine code:\n";
   DEBUG(BB->dump());
 }  
 
 void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn, MachineFunction &MF,
                                             FunctionLoweringInfo &FuncInfo) {
-  // Define NodeAllocator here so that memory allocation is reused for
+  // Define the SelectionDAG here so that memory allocation is reused for
   // each basic block.
-  NodeAllocatorType NodeAllocator;
+  SelectionDAG DAG(TLI, MF, FuncInfo, 
+                   getAnalysisToUpdate<MachineModuleInfo>());
+  CurDAG = &DAG;
 
   std::vector<std::pair<MachineInstr*, unsigned> > PHINodesToUpdate;
   for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) {
     BasicBlock *LLVMBB = &*I;
-    PHINodesToUpdate.clear();
+    BB = FuncInfo.MBBMap[LLVMBB];
+
+    BasicBlock::iterator Begin = LLVMBB->begin();
+    BasicBlock::iterator End = LLVMBB->end();
+    bool DoArgs = LLVMBB == &Fn.getEntryBlock();
+
+    // Before doing SelectionDAG ISel, see if FastISel has been requested.
+    // FastISel doesn't support EH landing pads, which require special handling.
+    if (EnableFastISel && !BB->isLandingPad()) {
+      if (FastISel *F = TLI.createFastISel(FuncInfo.MF)) {
+        while (Begin != End) {
+          Begin = F->SelectInstructions(Begin, End, FuncInfo.ValueMap,
+                                        FuncInfo.MBBMap, BB);
+
+          if (Begin == End)
+            // The "fast" selector selected the entire block, so we're done.
+            break;
+
+          // Handle certain instructions as single-LLVM-Instruction blocks.
+          if (isa<CallInst>(Begin) || isa<LoadInst>(Begin) ||
+              isa<StoreInst>(Begin)) {
+            if (Begin->getType() != Type::VoidTy) {
+              unsigned &R = FuncInfo.ValueMap[Begin];
+              if (!R)
+                R = FuncInfo.CreateRegForValue(Begin);
+            }
+
+            SelectBasicBlock(LLVMBB, Begin, next(Begin), DoArgs,
+                             PHINodesToUpdate, FuncInfo);
+
+            ++Begin;
+            DoArgs = false;
+            continue;
+          }
+
+          if (!DisableFastISelAbort &&
+              // For now, don't abort on non-conditional-branch terminators.
+              (!isa<TerminatorInst>(Begin) ||
+               (isa<BranchInst>(Begin) &&
+                cast<BranchInst>(Begin)->isUnconditional()))) {
+            // The "fast" selector couldn't handle something and bailed.
+            // For the purpose of debugging, just abort.
+#ifndef NDEBUG
+              Begin->dump();
+#endif
+            assert(0 && "FastISel didn't select the entire block");
+          }
+          break;
+        }
+        delete F;
+      }
+    }
+
+    if (Begin != End || DoArgs)
+      SelectBasicBlock(LLVMBB, Begin, End, DoArgs, PHINodesToUpdate, FuncInfo);
 
-    SelectBasicBlock(LLVMBB, MF, FuncInfo, PHINodesToUpdate, NodeAllocator);
-    FinishBasicBlock(LLVMBB, MF, FuncInfo, PHINodesToUpdate, NodeAllocator);
+    FinishBasicBlock(FuncInfo, PHINodesToUpdate);
+    PHINodesToUpdate.clear();
   }
+
+  CurDAG = 0;
 }
 
 void
-SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
-                                   FunctionLoweringInfo &FuncInfo,
-             std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
-                                   NodeAllocatorType &NodeAllocator) {
-  SelectionDAG DAG(TLI, MF, FuncInfo, 
-                   getAnalysisToUpdate<MachineModuleInfo>(),
-                   NodeAllocator);
-  CurDAG = &DAG;
-  
-  // First step, lower LLVM code to some DAG.  This DAG may use operations and
-  // types that are not supported by the target.
-  BuildSelectionDAG(DAG, LLVMBB, PHINodesToUpdate, FuncInfo);
+SelectionDAGISel::FinishBasicBlock(FunctionLoweringInfo &FuncInfo,
+             std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate) {
 
-  // Second step, emit the lowered DAG as machine code.
-  CodeGenAndEmitDAG(DAG);
-}
+  // Perform target specific isel post processing.
+  InstructionSelectPostProcessing();
+  
+  DOUT << "Target-post-processed machine code:\n";
+  DEBUG(BB->dump());
 
-void
-SelectionDAGISel::FinishBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
-                                   FunctionLoweringInfo &FuncInfo,
-             std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
-                                   NodeAllocatorType &NodeAllocator) {
   DOUT << "Total amount of phi nodes to update: "
        << PHINodesToUpdate.size() << "\n";
   DEBUG(for (unsigned i = 0, e = PHINodesToUpdate.size(); i != e; ++i)
@@ -5544,26 +5564,19 @@ SelectionDAGISel::FinishBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
   for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i) {
     // Lower header first, if it wasn't already lowered
     if (!BitTestCases[i].Emitted) {
-      SelectionDAG HSDAG(TLI, MF, FuncInfo, 
-                         getAnalysisToUpdate<MachineModuleInfo>(),
-                         NodeAllocator);
-      CurDAG = &HSDAG;
-      SelectionDAGLowering HSDL(HSDAG, TLI, *AA, FuncInfo, GFI);
+      SelectionDAGLowering HSDL(*CurDAG, TLI, *AA, FuncInfo, GFI);
       // Set the current basic block to the mbb we wish to insert the code into
       BB = BitTestCases[i].Parent;
       HSDL.setCurrentBasicBlock(BB);
       // Emit the code
       HSDL.visitBitTestHeader(BitTestCases[i]);
-      HSDAG.setRoot(HSDL.getRoot());
-      CodeGenAndEmitDAG(HSDAG);
+      CurDAG->setRoot(HSDL.getRoot());
+      CodeGenAndEmitDAG();
+      CurDAG->reset();
     }    
 
     for (unsigned j = 0, ej = BitTestCases[i].Cases.size(); j != ej; ++j) {
-      SelectionDAG BSDAG(TLI, MF, FuncInfo, 
-                         getAnalysisToUpdate<MachineModuleInfo>(),
-                         NodeAllocator);
-      CurDAG = &BSDAG;
-      SelectionDAGLowering BSDL(BSDAG, TLI, *AA, FuncInfo, GFI);
+      SelectionDAGLowering BSDL(*CurDAG, TLI, *AA, FuncInfo, GFI);
       // Set the current basic block to the mbb we wish to insert the code into
       BB = BitTestCases[i].Cases[j].ThisBB;
       BSDL.setCurrentBasicBlock(BB);
@@ -5578,8 +5591,9 @@ SelectionDAGISel::FinishBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
                               BitTestCases[i].Cases[j]);
         
         
-      BSDAG.setRoot(BSDL.getRoot());
-      CodeGenAndEmitDAG(BSDAG);
+      CurDAG->setRoot(BSDL.getRoot());
+      CodeGenAndEmitDAG();
+      CurDAG->reset();
     }
 
     // Update PHI Nodes
@@ -5618,32 +5632,26 @@ SelectionDAGISel::FinishBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
   for (unsigned i = 0, e = JTCases.size(); i != e; ++i) {
     // Lower header first, if it wasn't already lowered
     if (!JTCases[i].first.Emitted) {
-      SelectionDAG HSDAG(TLI, MF, FuncInfo, 
-                         getAnalysisToUpdate<MachineModuleInfo>(),
-                         NodeAllocator);
-      CurDAG = &HSDAG;
-      SelectionDAGLowering HSDL(HSDAG, TLI, *AA, FuncInfo, GFI);
+      SelectionDAGLowering HSDL(*CurDAG, TLI, *AA, FuncInfo, GFI);
       // Set the current basic block to the mbb we wish to insert the code into
       BB = JTCases[i].first.HeaderBB;
       HSDL.setCurrentBasicBlock(BB);
       // Emit the code
       HSDL.visitJumpTableHeader(JTCases[i].second, JTCases[i].first);
-      HSDAG.setRoot(HSDL.getRoot());
-      CodeGenAndEmitDAG(HSDAG);
+      CurDAG->setRoot(HSDL.getRoot());
+      CodeGenAndEmitDAG();
+      CurDAG->reset();
     }
     
-    SelectionDAG JSDAG(TLI, MF, FuncInfo, 
-                       getAnalysisToUpdate<MachineModuleInfo>(),
-                       NodeAllocator);
-    CurDAG = &JSDAG;
-    SelectionDAGLowering JSDL(JSDAG, TLI, *AA, FuncInfo, GFI);
+    SelectionDAGLowering JSDL(*CurDAG, TLI, *AA, FuncInfo, GFI);
     // Set the current basic block to the mbb we wish to insert the code into
     BB = JTCases[i].second.MBB;
     JSDL.setCurrentBasicBlock(BB);
     // Emit the code
     JSDL.visitJumpTable(JTCases[i].second);
-    JSDAG.setRoot(JSDL.getRoot());
-    CodeGenAndEmitDAG(JSDAG);
+    CurDAG->setRoot(JSDL.getRoot());
+    CodeGenAndEmitDAG();
+    CurDAG->reset();
     
     // Update PHI Nodes
     for (unsigned pi = 0, pe = PHINodesToUpdate.size(); pi != pe; ++pi) {
@@ -5682,11 +5690,7 @@ SelectionDAGISel::FinishBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
   // If we generated any switch lowering information, build and codegen any
   // additional DAGs necessary.
   for (unsigned i = 0, e = SwitchCases.size(); i != e; ++i) {
-    SelectionDAG SDAG(TLI, MF, FuncInfo, 
-                      getAnalysisToUpdate<MachineModuleInfo>(),
-                      NodeAllocator);
-    CurDAG = &SDAG;
-    SelectionDAGLowering SDL(SDAG, TLI, *AA, FuncInfo, GFI);
+    SelectionDAGLowering SDL(*CurDAG, TLI, *AA, FuncInfo, GFI);
     
     // Set the current basic block to the mbb we wish to insert the code into
     BB = SwitchCases[i].ThisBB;
@@ -5694,8 +5698,9 @@ SelectionDAGISel::FinishBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
     
     // Emit the code
     SDL.visitSwitchCase(SwitchCases[i]);
-    SDAG.setRoot(SDL.getRoot());
-    CodeGenAndEmitDAG(SDAG);
+    CurDAG->setRoot(SDL.getRoot());
+    CodeGenAndEmitDAG();
+    CurDAG->reset();
     
     // Handle any PHI nodes in successors of this chunk, as if we were coming
     // from the original BB before switch expansion.  Note that PHI nodes can
@@ -5732,7 +5737,7 @@ SelectionDAGISel::FinishBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
 /// Schedule - Pick a safe ordering for instructions for each
 /// target node in the graph.
 ///
-ScheduleDAG *SelectionDAGISel::Schedule(SelectionDAG &DAG) {
+ScheduleDAG *SelectionDAGISel::Schedule() {
   RegisterScheduler::FunctionPassCtor Ctor = RegisterScheduler::getDefault();
   
   if (!Ctor) {
@@ -5740,7 +5745,7 @@ ScheduleDAG *SelectionDAGISel::Schedule(SelectionDAG &DAG) {
     RegisterScheduler::setDefault(Ctor);
   }
   
-  ScheduleDAG *Scheduler = Ctor(this, &DAG, BB, Fast);
+  ScheduleDAG *Scheduler = Ctor(this, CurDAG, BB, Fast);
   Scheduler->Run();
 
   return Scheduler;
@@ -5823,7 +5828,7 @@ bool SelectionDAGISel::CheckOrMask(SDValue LHS, ConstantSDNode *RHS,
 /// SelectInlineAsmMemoryOperands - Calls to this are automatically generated
 /// by tblgen.  Others should not call it.
 void SelectionDAGISel::
-SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops, SelectionDAG &DAG) {
+SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops) {
   std::vector<SDValue> InOps;
   std::swap(InOps, Ops);
 
@@ -5844,15 +5849,15 @@ SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops, SelectionDAG &DAG) {
       assert((Flags >> 3) == 1 && "Memory operand with multiple values?");
       // Otherwise, this is a memory operand.  Ask the target to select it.
       std::vector<SDValue> SelOps;
-      if (SelectInlineAsmMemoryOperand(InOps[i+1], 'm', SelOps, DAG)) {
+      if (SelectInlineAsmMemoryOperand(InOps[i+1], 'm', SelOps)) {
         cerr << "Could not match memory address.  Inline asm failure!\n";
         exit(1);
       }
       
       // Add this to the output node.
-      MVT IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy();
-      Ops.push_back(DAG.getTargetConstant(4/*MEM*/ | (SelOps.size() << 3),
-                                          IntPtrTy));
+      MVT IntPtrTy = CurDAG->getTargetLoweringInfo().getPointerTy();
+      Ops.push_back(CurDAG->getTargetConstant(4/*MEM*/ | (SelOps.size() << 3),
+                                              IntPtrTy));
       Ops.insert(Ops.end(), SelOps.begin(), SelOps.end());
       i += 2;
     }
diff --git a/lib/Target/ARM/ARMISelDAGToDAG.cpp b/lib/Target/ARM/ARMISelDAGToDAG.cpp
index ad1f63d642..d26a5fbd34 100644
--- a/lib/Target/ARM/ARMISelDAGToDAG.cpp
+++ b/lib/Target/ARM/ARMISelDAGToDAG.cpp
@@ -54,7 +54,7 @@ public:
   } 
   
   SDNode *Select(SDValue Op);
-  virtual void InstructionSelect(SelectionDAG &DAG);
+  virtual void InstructionSelect();
   bool SelectAddrMode2(SDValue Op, SDValue N, SDValue &Base,
                        SDValue &Offset, SDValue &Opc);
   bool SelectAddrMode2Offset(SDValue Op, SDValue N,
@@ -91,11 +91,11 @@ public:
 };
 }
 
-void ARMDAGToDAGISel::InstructionSelect(SelectionDAG &DAG) {
+void ARMDAGToDAGISel::InstructionSelect() {
   DEBUG(BB->dump());
 
   SelectRoot();
-  DAG.RemoveDeadNodes();
+  CurDAG->RemoveDeadNodes();
 }
 
 bool ARMDAGToDAGISel::SelectAddrMode2(SDValue Op, SDValue N,
diff --git a/lib/Target/Alpha/AlphaISelDAGToDAG.cpp b/lib/Target/Alpha/AlphaISelDAGToDAG.cpp
index 0ff735d28e..ce6afc3b81 100644
--- a/lib/Target/Alpha/AlphaISelDAGToDAG.cpp
+++ b/lib/Target/Alpha/AlphaISelDAGToDAG.cpp
@@ -163,7 +163,7 @@ namespace {
     
     /// InstructionSelect - This callback is invoked by
     /// SelectionDAGISel when it has created a SelectionDAG for us to codegen.
-    virtual void InstructionSelect(SelectionDAG &DAG);
+    virtual void InstructionSelect();
     
     virtual const char *getPassName() const {
       return "Alpha DAG->DAG Pattern Instruction Selection";
@@ -173,8 +173,7 @@ namespace {
     /// inline asm expressions.
     virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
                                               char ConstraintCode,
-                                              std::vector<SDValue> &OutOps,
-                                              SelectionDAG &DAG) {
+                                              std::vector<SDValue> &OutOps) {
       SDValue Op0;
       switch (ConstraintCode) {
       default: return true;
@@ -232,12 +231,12 @@ SDValue AlphaDAGToDAGISel::getGlobalRetAddr() {
 
 /// InstructionSelect - This callback is invoked by
 /// SelectionDAGISel when it has created a SelectionDAG for us to codegen.
-void AlphaDAGToDAGISel::InstructionSelect(SelectionDAG &DAG) {
+void AlphaDAGToDAGISel::InstructionSelect() {
   DEBUG(BB->dump());
   
   // Select target instructions for the DAG.
   SelectRoot();
-  DAG.RemoveDeadNodes();
+  CurDAG->RemoveDeadNodes();
 }
 
 // Select - Convert the specified operand from a target-independent to a
diff --git a/lib/Target/CellSPU/SPUISelDAGToDAG.cpp b/lib/Target/CellSPU/SPUISelDAGToDAG.cpp
index c8d04b46cb..c2a3e3a8df 100644
--- a/lib/Target/CellSPU/SPUISelDAGToDAG.cpp
+++ b/lib/Target/CellSPU/SPUISelDAGToDAG.cpp
@@ -285,8 +285,7 @@ public:
   /// inline asm expressions.
   virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
                                             char ConstraintCode,
-                                            std::vector<SDValue> &OutOps,
-                                            SelectionDAG &DAG) {
+                                            std::vector<SDValue> &OutOps) {
     SDValue Op0, Op1;
     switch (ConstraintCode) {
     default: return true;
@@ -319,7 +318,7 @@ public:
 
   /// InstructionSelect - This callback is invoked by
   /// SelectionDAGISel when it has created a SelectionDAG for us to codegen.
-  virtual void InstructionSelect(SelectionDAG &DAG);
+  virtual void InstructionSelect();
 
   virtual const char *getPassName() const {
     return "Cell SPU DAG->DAG Pattern Instruction Selection";
@@ -342,13 +341,13 @@ public:
 /// InstructionSelect - This callback is invoked by
 /// SelectionDAGISel when it has created a SelectionDAG for us to codegen.
 void
-SPUDAGToDAGISel::InstructionSelect(SelectionDAG &DAG)
+SPUDAGToDAGISel::InstructionSelect()
 {
   DEBUG(BB->dump());
 
   // Select target instructions for the DAG.
   SelectRoot();
-  DAG.RemoveDeadNodes();
+  CurDAG->RemoveDeadNodes();
 }
 
 /*!
diff --git a/lib/Target/IA64/IA64ISelDAGToDAG.cpp b/lib/Target/IA64/IA64ISelDAGToDAG.cpp
index 8173e96ccb..aa1f6bb015 100644
--- a/lib/Target/IA64/IA64ISelDAGToDAG.cpp
+++ b/lib/Target/IA64/IA64ISelDAGToDAG.cpp
@@ -80,7 +80,7 @@ namespace {
 
     /// InstructionSelect - This callback is invoked by
     /// SelectionDAGISel when it has created a SelectionDAG for us to codegen.
-    virtual void InstructionSelect(SelectionDAG &DAG);
+    virtual void InstructionSelect();
     
     virtual const char *getPassName() const {
       return "IA64 (Itanium) DAG->DAG Instruction Selector";
@@ -96,12 +96,12 @@ private:
 
 /// InstructionSelect - This callback is invoked by
 /// SelectionDAGISel when it has created a SelectionDAG for us to codegen.
-void IA64DAGToDAGISel::InstructionSelect(SelectionDAG &DAG) {
+void IA64DAGToDAGISel::InstructionSelect() {
   DEBUG(BB->dump());
 
   // Select target instructions for the DAG.
   SelectRoot();
-  DAG.RemoveDeadNodes();
+  CurDAG->RemoveDeadNodes();
 }
 
 SDNode *IA64DAGToDAGISel::SelectDIV(SDValue Op) {
diff --git a/lib/Target/Mips/MipsISelDAGToDAG.cpp b/lib/Target/Mips/MipsISelDAGToDAG.cpp
index 94fdb3526d..1e07a8f53d 100644
--- a/lib/Target/Mips/MipsISelDAGToDAG.cpp
+++ b/lib/Target/Mips/MipsISelDAGToDAG.cpp
@@ -66,7 +66,7 @@ public:
   TM(tm), MipsLowering(*TM.getTargetLowering()), 
   Subtarget(tm.getSubtarget<MipsSubtarget>()) {}
   
-  virtual void InstructionSelect(SelectionDAG &SD);
+  virtual void InstructionSelect();
 
   // Pass Name
   virtual const char *getPassName() const {
@@ -103,7 +103,7 @@ private:
 /// InstructionSelect - This callback is invoked by
 /// SelectionDAGISel when it has created a SelectionDAG for us to codegen.
 void MipsDAGToDAGISel::
-InstructionSelect(SelectionDAG &SD) 
+InstructionSelect() 
 {
   DEBUG(BB->dump());
   // Codegen the basic block.
@@ -119,7 +119,7 @@ InstructionSelect(SelectionDAG &SD)
   DOUT << "===== Instruction selection ends:\n";
   #endif
 
-  SD.RemoveDeadNodes();
+  CurDAG->RemoveDeadNodes();
 }
 
 /// getGlobalBaseReg - Output the instructions required to put the
diff --git a/lib/Target/PIC16/PIC16ISelDAGToDAG.cpp b/lib/Target/PIC16/PIC16ISelDAGToDAG.cpp
index f1eaa39155..a9c1d09613 100644
--- a/lib/Target/PIC16/PIC16ISelDAGToDAG.cpp
+++ b/lib/Target/PIC16/PIC16ISelDAGToDAG.cpp
@@ -60,7 +60,7 @@ public:
         SelectionDAGISel(PIC16Lowering),
         TM(tm), PIC16Lowering(*TM.getTargetLowering()) {}
   
-  virtual void InstructionSelect(SelectionDAG &SD);
+  virtual void InstructionSelect();
 
   // Pass Name
   virtual const char *getPassName() const {
@@ -98,7 +98,7 @@ private:
 
 /// InstructionSelect - This callback is invoked by
 /// SelectionDAGISel when it has created a SelectionDAG for us to codegen.
-void PIC16DAGToDAGISel::InstructionSelect(SelectionDAG &SD) 
+void PIC16DAGToDAGISel::InstructionSelect() 
 {
   DEBUG(BB->dump());
   // Codegen the basic block.
@@ -113,7 +113,7 @@ void PIC16DAGToDAGISel::InstructionSelect(SelectionDAG &SD)
 
   DOUT << "===== Instruction selection ends:\n";
 
-  SD.RemoveDeadNodes();
+  CurDAG->RemoveDeadNodes();
 }
 
 
diff --git a/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index c647955434..e655e06e7a 100644
--- a/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -144,8 +144,7 @@ namespace {
     /// inline asm expressions.
     virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
                                               char ConstraintCode,
-                                              std::vector<SDValue> &OutOps,
-                                              SelectionDAG &DAG) {
+                                              std::vector<SDValue> &OutOps) {
       SDValue Op0, Op1;
       switch (ConstraintCode) {
       default: return true;
@@ -175,7 +174,7 @@ namespace {
     
     /// InstructionSelect - This callback is invoked by
     /// SelectionDAGISel when it has created a SelectionDAG for us to codegen.
-    virtual void InstructionSelect(SelectionDAG &DAG);
+    virtual void InstructionSelect();
     
     void InsertVRSaveCode(Function &Fn);
 
@@ -203,12 +202,12 @@ private:
 
 /// InstructionSelect - This callback is invoked by
 /// SelectionDAGISel when it has created a SelectionDAG for us to codegen.
-void PPCDAGToDAGISel::InstructionSelect(SelectionDAG &DAG) {
+void PPCDAGToDAGISel::InstructionSelect() {
   DEBUG(BB->dump());
 
   // Select target instructions for the DAG.
   SelectRoot();
-  DAG.RemoveDeadNodes();
+  CurDAG->RemoveDeadNodes();
 }
 
 /// InsertVRSaveCode - Once the entire function has been instruction selected,
diff --git a/lib/Target/Sparc/SparcISelDAGToDAG.cpp b/lib/Target/Sparc/SparcISelDAGToDAG.cpp
index f62387307d..64e9bcec6a 100644
--- a/lib/Target/Sparc/SparcISelDAGToDAG.cpp
+++ b/lib/Target/Sparc/SparcISelDAGToDAG.cpp
@@ -49,7 +49,7 @@ public:
   
   /// InstructionSelect - This callback is invoked by
   /// SelectionDAGISel when it has created a SelectionDAG for us to codegen.
-  virtual void InstructionSelect(SelectionDAG &DAG);
+  virtual void InstructionSelect();
   
   virtual const char *getPassName() const {
     return "SPARC DAG->DAG Pattern Instruction Selection";
@@ -62,12 +62,12 @@ public:
 
 /// InstructionSelect - This callback is invoked by
 /// SelectionDAGISel when it has created a SelectionDAG for us to codegen.
-void SparcDAGToDAGISel::InstructionSelect(SelectionDAG &DAG) {
+void SparcDAGToDAGISel::InstructionSelect() {
   DEBUG(BB->dump());
   
   // Select target instructions for the DAG.
   SelectRoot();
-  DAG.RemoveDeadNodes();
+  CurDAG->RemoveDeadNodes();
 }
 
 bool SparcDAGToDAGISel::SelectADDRri(SDValue Op, SDValue Addr,
diff --git a/lib/Target/X86/X86ISelDAGToDAG.cpp b/lib/Target/X86/X86ISelDAGToDAG.cpp
index 21837d53d4..29986e50d6 100644
--- a/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -147,7 +147,7 @@ namespace {
 
     /// InstructionSelect - This callback is invoked by
     /// SelectionDAGISel when it has created a SelectionDAG for us to codegen.
-    virtual void InstructionSelect(SelectionDAG &DAG);
+    virtual void InstructionSelect();
 
     /// InstructionSelectPostProcessing - Post processing of selected and
     /// scheduled basic blocks.
@@ -178,15 +178,14 @@ namespace {
     bool TryFoldLoad(SDValue P, SDValue N,
                      SDValue &Base, SDValue &Scale,
                      SDValue &Index, SDValue &Disp);
-    void PreprocessForRMW(SelectionDAG &DAG);
-    void PreprocessForFPConvert(SelectionDAG &DAG);
+    void PreprocessForRMW();
+    void PreprocessForFPConvert();
 
     /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
     /// inline asm expressions.
     virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
                                               char ConstraintCode,
-                                              std::vector<SDValue> &OutOps,
-                                              SelectionDAG &DAG);
+                                              std::vector<SDValue> &OutOps);
     
     void EmitSpecialCodeForMain(MachineBasicBlock *BB, MachineFrameInfo *MFI);
 
@@ -372,7 +371,7 @@ bool X86DAGToDAGISel::CanBeFoldedBy(SDNode *N, SDNode *U, SDNode *Root) const {
 /// MoveBelowTokenFactor - Replace TokenFactor operand with load's chain operand
 /// and move load below the TokenFactor. Replace store's chain operand with
 /// load's chain result.
-static void MoveBelowTokenFactor(SelectionDAG &DAG, SDValue Load,
+static void MoveBelowTokenFactor(SelectionDAG *CurDAG, SDValue Load,
                                  SDValue Store, SDValue TF) {
   std::vector<SDValue> Ops;
   for (unsigned i = 0, e = TF.Val->getNumOperands(); i != e; ++i)
@@ -380,10 +379,10 @@ static void MoveBelowTokenFactor(SelectionDAG &DAG, SDValue Load,
       Ops.push_back(Load.Val->getOperand(0));
     else
       Ops.push_back(TF.Val->getOperand(i));
-  DAG.UpdateNodeOperands(TF, &Ops[0], Ops.size());
-  DAG.UpdateNodeOperands(Load, TF, Load.getOperand(1), Load.getOperand(2));
-  DAG.UpdateNodeOperands(Store, Load.getValue(1), Store.getOperand(1),
-                         Store.getOperand(2), Store.getOperand(3));
+  CurDAG->UpdateNodeOperands(TF, &Ops[0], Ops.size());
+  CurDAG->UpdateNodeOperands(Load, TF, Load.getOperand(1), Load.getOperand(2));
+  CurDAG->UpdateNodeOperands(Store, Load.getValue(1), Store.getOperand(1),
+                             Store.getOperand(2), Store.getOperand(3));
 }
 
 /// isRMWLoad - Return true if N is a load that's part of RMW sub-DAG.
@@ -452,9 +451,9 @@ static bool isRMWLoad(SDValue N, SDValue Chain, SDValue Address,
 ///       \      /
 ///        \    /
 ///       [Store]
-void X86DAGToDAGISel::PreprocessForRMW(SelectionDAG &DAG) {
-  for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
-         E = DAG.allnodes_end(); I != E; ++I) {
+void X86DAGToDAGISel::PreprocessForRMW() {
+  for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
+         E = CurDAG->allnodes_end(); I != E; ++I) {
     if (!ISD::isNON_TRUNCStore(I))
       continue;
     SDValue Chain = I->getOperand(0);
@@ -504,7 +503,7 @@ void X86DAGToDAGISel::PreprocessForRMW(SelectionDAG &DAG) {
     }
 
     if (RModW) {
-      MoveBelowTokenFactor(DAG, Load, SDValue(I, 0), Chain);
+      MoveBelowTokenFactor(CurDAG, Load, SDValue(I, 0), Chain);
       ++NumLoadMoved;
     }
   }
@@ -519,9 +518,9 @@ void X86DAGToDAGISel::PreprocessForRMW(SelectionDAG &DAG) {
 /// hack on these between the call expansion and the node legalization.  As such
 /// this pass basically does "really late" legalization of these inline with the
 /// X86 isel pass.
-void X86DAGToDAGISel::PreprocessForFPConvert(SelectionDAG &DAG) {
-  for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
-       E = DAG.allnodes_end(); I != E; ) {
+void X86DAGToDAGISel::PreprocessForFPConvert() {
+  for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
+       E = CurDAG->allnodes_end(); I != E; ) {
     SDNode *N = I++;  // Preincrement iterator to avoid invalidation issues.
     if (N->getOpcode() != ISD::FP_ROUND && N->getOpcode() != ISD::FP_EXTEND)
       continue;
@@ -553,39 +552,40 @@ void X86DAGToDAGISel::PreprocessForFPConvert(SelectionDAG &DAG) {
     else
       MemVT = SrcIsSSE ? SrcVT : DstVT;
     
-    SDValue MemTmp = DAG.CreateStackTemporary(MemVT);
+    SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
     
     // FIXME: optimize the case where the src/dest is a load or store?
-    SDValue Store = DAG.getTruncStore(DAG.getEntryNode(), N->getOperand(0),
-                                        MemTmp, NULL, 0, MemVT);
-    SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, DstVT, Store, MemTmp,
-                                      NULL, 0, MemVT);
+    SDValue Store = CurDAG->getTruncStore(CurDAG->getEntryNode(),
+                                          N->getOperand(0),
+                                          MemTmp, NULL, 0, MemVT);
+    SDValue Result = CurDAG->getExtLoad(ISD::EXTLOAD, DstVT, Store, MemTmp,
+                                        NULL, 0, MemVT);
 
     // We're about to replace all uses of the FP_ROUND/FP_EXTEND with the
     // extload we created.  This will cause general havok on the dag because
     // anything below the conversion could be folded into other existing nodes.
     // To avoid invalidating 'I', back it up to the convert node.
     --I;
-    DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
+    CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
     
     // Now that we did that, the node is dead.  Increment the iterator to the
     // next node to process, then delete N.
     ++I;
-    DAG.DeleteNode(N);
+    CurDAG->DeleteNode(N);
   }  
 }
 
 /// InstructionSelectBasicBlock - This callback is invoked by SelectionDAGISel
 /// when it has created a SelectionDAG for us to codegen.
-void X86DAGToDAGISel::InstructionSelect(SelectionDAG &DAG) {
+void X86DAGToDAGISel::InstructionSelect() {
   CurBB = BB;  // BB can change as result of isel.
 
   DEBUG(BB->dump());
   if (!Fast)
-    PreprocessForRMW(DAG);
+    PreprocessForRMW();
 
   // FIXME: This should only happen when not -fast.
-  PreprocessForFPConvert(DAG);
+  PreprocessForFPConvert();
 
   // Codegen the basic block.
 #ifndef NDEBUG
@@ -597,7 +597,7 @@ void X86DAGToDAGISel::InstructionSelect(SelectionDAG &DAG) {
   DOUT << "===== Instruction selection ends:\n";
 #endif
 
-  DAG.RemoveDeadNodes();
+  CurDAG->RemoveDeadNodes();
 }
 
 void X86DAGToDAGISel::InstructionSelectPostProcessing() {
@@ -1599,7 +1599,7 @@ SDNode *X86DAGToDAGISel::Select(SDValue N) {
 
 bool X86DAGToDAGISel::
 SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
-                             std::vector<SDValue> &OutOps, SelectionDAG &DAG){
+                             std::vector<SDValue> &OutOps) {
   SDValue Op0, Op1, Op2, Op3;
   switch (ConstraintCode) {
   case 'o':   // offsetable        ??
diff --git a/utils/TableGen/DAGISelEmitter.cpp b/utils/TableGen/DAGISelEmitter.cpp
index cf4d20f9c6..b7778918d0 100644
--- a/utils/TableGen/DAGISelEmitter.cpp
+++ b/utils/TableGen/DAGISelEmitter.cpp
@@ -1864,7 +1864,7 @@ void DAGISelEmitter::EmitInstructionSelector(std::ostream &OS) {
   // Emit boilerplate.
   OS << "SDNode *Select_INLINEASM(SDValue N) {\n"
      << "  std::vector<SDValue> Ops(N.Val->op_begin(), N.Val->op_end());\n"
-     << "  SelectInlineAsmMemoryOperands(Ops, *CurDAG);\n\n"
+     << "  SelectInlineAsmMemoryOperands(Ops);\n\n"
     
      << "  // Ensure that the asm operands are themselves selected.\n"
      << "  for (unsigned j = 0, e = Ops.size(); j != e; ++j)\n"