Adding support and tests for multiple module handling in lli

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@191938 91177308-0d34-0410-b5e6-96231b3b80d8

5 files changed, 304 insertions, 227 deletions

diff --git a/tools/lli/CMakeLists.txt b/tools/lli/CMakeLists.txt
index 578c414cce..5f8c7c9261 100644
--- a/tools/lli/CMakeLists.txt
+++ b/tools/lli/CMakeLists.txt
@@ -21,7 +21,7 @@ endif( LLVM_USE_INTEL_JITEVENTS )
 
 add_llvm_tool(lli
   lli.cpp
-  RecordingMemoryManager.cpp
+  RemoteMemoryManager.cpp
   RemoteTarget.cpp
   RemoteTargetExternal.cpp
   )
diff --git a/tools/lli/RecordingMemoryManager.cpp b/tools/lli/RecordingMemoryManager.cpp
deleted file mode 100644
index d54b8e434d..0000000000
--- a/tools/lli/RecordingMemoryManager.cpp
+++ /dev/null
@@ -1,126 +0,0 @@
-//===- RecordingMemoryManager.cpp - Recording memory manager --------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This memory manager allocates local storage and keeps a record of each
-// allocation. Iterators are provided for all data and code allocations.
-//
-//===----------------------------------------------------------------------===//
-
-#include "RecordingMemoryManager.h"
-using namespace llvm;
-
-RecordingMemoryManager::~RecordingMemoryManager() {
-  for (SmallVectorImpl<Allocation>::iterator
-         I = AllocatedCodeMem.begin(), E = AllocatedCodeMem.end();
-       I != E; ++I)
-    sys::Memory::releaseMappedMemory(I->first);
-  for (SmallVectorImpl<Allocation>::iterator
-         I = AllocatedDataMem.begin(), E = AllocatedDataMem.end();
-       I != E; ++I)
-    sys::Memory::releaseMappedMemory(I->first);
-}
-
-uint8_t *RecordingMemoryManager::
-allocateCodeSection(uintptr_t Size, unsigned Alignment, unsigned SectionID,
-                    StringRef SectionName) {
-  // The recording memory manager is just a local copy of the remote target.
-  // The alignment requirement is just stored here for later use. Regular
-  // heap storage is sufficient here, but we're using mapped memory to work
-  // around a bug in MCJIT.
-  sys::MemoryBlock Block = allocateSection(Size);
-  AllocatedCodeMem.push_back(Allocation(Block, Alignment));
-  return (uint8_t*)Block.base();
-}
-
-uint8_t *RecordingMemoryManager::
-allocateDataSection(uintptr_t Size, unsigned Alignment,
-                    unsigned SectionID, StringRef SectionName,
-                    bool IsReadOnly) {
-  // The recording memory manager is just a local copy of the remote target.
-  // The alignment requirement is just stored here for later use. Regular
-  // heap storage is sufficient here, but we're using mapped memory to work
-  // around a bug in MCJIT.
-  sys::MemoryBlock Block = allocateSection(Size);
-  AllocatedDataMem.push_back(Allocation(Block, Alignment));
-  return (uint8_t*)Block.base();
-}
-
-sys::MemoryBlock RecordingMemoryManager::allocateSection(uintptr_t Size) {
-  error_code ec;
-  sys::MemoryBlock MB = sys::Memory::allocateMappedMemory(Size,
-                                                          &Near,
-                                                          sys::Memory::MF_READ |
-                                                          sys::Memory::MF_WRITE,
-                                                          ec);
-  assert(!ec && MB.base());
-
-  // FIXME: This is part of a work around to keep sections near one another
-  // when MCJIT performs relocations after code emission but before
-  // the generated code is moved to the remote target.
-  // Save this address as the basis for our next request
-  Near = MB;
-  return MB;
-}
-
-void RecordingMemoryManager::setMemoryWritable() { llvm_unreachable("Unexpected!"); }
-void RecordingMemoryManager::setMemoryExecutable() { llvm_unreachable("Unexpected!"); }
-void RecordingMemoryManager::setPoisonMemory(bool poison) { llvm_unreachable("Unexpected!"); }
-void RecordingMemoryManager::AllocateGOT() { llvm_unreachable("Unexpected!"); }
-uint8_t *RecordingMemoryManager::getGOTBase() const {
-  llvm_unreachable("Unexpected!");
-  return 0;
-}
-uint8_t *RecordingMemoryManager::startFunctionBody(const Function *F, uintptr_t &ActualSize){
-  llvm_unreachable("Unexpected!");
-  return 0;
-}
-uint8_t *RecordingMemoryManager::allocateStub(const GlobalValue* F, unsigned StubSize,
-                                              unsigned Alignment) {
-  llvm_unreachable("Unexpected!");
-  return 0;
-}
-void RecordingMemoryManager::endFunctionBody(const Function *F, uint8_t *FunctionStart,
-                                             uint8_t *FunctionEnd) {
-  llvm_unreachable("Unexpected!");
-}
-uint8_t *RecordingMemoryManager::allocateSpace(intptr_t Size, unsigned Alignment) {
-  llvm_unreachable("Unexpected!");
-  return 0;
-}
-uint8_t *RecordingMemoryManager::allocateGlobal(uintptr_t Size, unsigned Alignment) {
-  llvm_unreachable("Unexpected!");
-  return 0;
-}
-void RecordingMemoryManager::deallocateFunctionBody(void *Body) {
-  llvm_unreachable("Unexpected!");
-}
-
-static int jit_noop() {
-  return 0;
-}
-
-void *RecordingMemoryManager::getPointerToNamedFunction(const std::string &Name,
-                                                        bool AbortOnFailure) {
-  // We should not invoke parent's ctors/dtors from generated main()!
-  // On Mingw and Cygwin, the symbol __main is resolved to
-  // callee's(eg. tools/lli) one, to invoke wrong duplicated ctors
-  // (and register wrong callee's dtors with atexit(3)).
-  // We expect ExecutionEngine::runStaticConstructorsDestructors()
-  // is called before ExecutionEngine::runFunctionAsMain() is called.
-  if (Name == "__main") return (void*)(intptr_t)&jit_noop;
-
-  // FIXME: Would it be responsible to provide GOT?
-  if (AbortOnFailure) {
-    if (Name == "_GLOBAL_OFFSET_TABLE_")
-      report_fatal_error("Program used external function '" + Name +
-                         "' which could not be resolved!");
-  }
-
-  return NULL;
-}
diff --git a/tools/lli/RemoteMemoryManager.cpp b/tools/lli/RemoteMemoryManager.cpp
new file mode 100644
index 0000000000..1f86066051
--- /dev/null
+++ b/tools/lli/RemoteMemoryManager.cpp
@@ -0,0 +1,230 @@
+//===---- RemoteMemoryManager.cpp - Recording memory manager --------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This memory manager allocates local storage and keeps a record of each
+// allocation. Iterators are provided for all data and code allocations.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "lli"
+#include "RemoteMemoryManager.h"
+#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/ExecutionEngine/ObjectImage.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Format.h"
+
+using namespace llvm;
+
+RemoteMemoryManager::~RemoteMemoryManager() {
+  for (SmallVector<Allocation, 2>::iterator
+         I = AllocatedSections.begin(), E = AllocatedSections.end();
+       I != E; ++I)
+    sys::Memory::releaseMappedMemory(I->MB);
+}
+
+uint8_t *RemoteMemoryManager::
+allocateCodeSection(uintptr_t Size, unsigned Alignment, unsigned SectionID,
+                    StringRef SectionName) {
+  // The recording memory manager is just a local copy of the remote target.
+  // The alignment requirement is just stored here for later use. Regular
+  // heap storage is sufficient here, but we're using mapped memory to work
+  // around a bug in MCJIT.
+  sys::MemoryBlock Block = allocateSection(Size);
+  AllocatedSections.push_back( Allocation(Block, Alignment, true) );
+  UnmappedSections.push_back( &AllocatedSections.back() );
+  return (uint8_t*)Block.base();
+}
+
+uint8_t *RemoteMemoryManager::
+allocateDataSection(uintptr_t Size, unsigned Alignment,
+                    unsigned SectionID, StringRef SectionName,
+                    bool IsReadOnly) {
+  // The recording memory manager is just a local copy of the remote target.
+  // The alignment requirement is just stored here for later use. Regular
+  // heap storage is sufficient here, but we're using mapped memory to work
+  // around a bug in MCJIT.
+  sys::MemoryBlock Block = allocateSection(Size);
+  AllocatedSections.push_back( Allocation(Block, Alignment, false) );
+  UnmappedSections.push_back( &AllocatedSections.back() );
+  return (uint8_t*)Block.base();
+}
+
+sys::MemoryBlock RemoteMemoryManager::allocateSection(uintptr_t Size) {
+  error_code ec;
+  sys::MemoryBlock MB = sys::Memory::allocateMappedMemory(Size,
+                                                          &Near,
+                                                          sys::Memory::MF_READ |
+                                                          sys::Memory::MF_WRITE,
+                                                          ec);
+  assert(!ec && MB.base());
+
+  // FIXME: This is part of a work around to keep sections near one another
+  // when MCJIT performs relocations after code emission but before
+  // the generated code is moved to the remote target.
+  // Save this address as the basis for our next request
+  Near = MB;
+  return MB;
+}
+
+void RemoteMemoryManager::notifyObjectLoaded(ExecutionEngine *EE,
+                                                const ObjectImage *Obj) {
+  // The client should have called setRemoteTarget() before triggering any
+  // code generation.
+  assert(Target);
+  if (!Target)
+    return;
+
+  // FIXME: Make this function thread safe.
+
+  // Lay out our sections in order, with all the code sections first, then
+  // all the data sections.
+  uint64_t CurOffset = 0;
+  unsigned MaxAlign = Target->getPageAlignment();
+  SmallVector<std::pair<const Allocation*, uint64_t>, 16> Offsets;
+  unsigned NumSections = UnmappedSections.size();
+  // We're going to go through the list twice to separate code and data, but
+  // it's a very small list, so that's OK.
+  for (size_t i = 0, e = NumSections; i != e; ++i) {
+    const Allocation *Section = UnmappedSections[i];
+    assert(Section);
+    if (Section->IsCode) {
+      unsigned Size = Section->MB.size();
+      unsigned Align = Section->Alignment;
+      DEBUG(dbgs() << "code region: size " << Size
+                  << ", alignment " << Align << "\n");
+      // Align the current offset up to whatever is needed for the next
+      // section.
+      CurOffset = (CurOffset + Align - 1) / Align * Align;
+      // Save off the address of the new section and allocate its space.
+      Offsets.push_back(std::pair<const Allocation*,uint64_t>(Section, 
+                                                              CurOffset));
+      CurOffset += Size;
+    }
+  }
+  // Adjust to keep code and data aligned on seperate pages.
+  CurOffset = (CurOffset + MaxAlign - 1) / MaxAlign * MaxAlign;
+  for (size_t i = 0, e = NumSections; i != e; ++i) {
+    const Allocation *Section = UnmappedSections[i];
+    assert(Section);
+    if (!Section->IsCode) {
+      unsigned Size = Section->MB.size();
+      unsigned Align = Section->Alignment;
+      DEBUG(dbgs() << "data region: size " << Size
+                  << ", alignment " << Align << "\n");
+      // Align the current offset up to whatever is needed for the next
+      // section.
+      CurOffset = (CurOffset + Align - 1) / Align * Align;
+      // Save off the address of the new section and allocate its space.
+      Offsets.push_back(std::pair<const Allocation*,uint64_t>(Section, 
+                                                              CurOffset));
+      CurOffset += Size;
+    }
+  }
+
+  // Allocate space in the remote target.
+  uint64_t RemoteAddr;
+  if (Target->allocateSpace(CurOffset, MaxAlign, RemoteAddr))
+    report_fatal_error(Target->getErrorMsg());
+
+  // Map the section addresses so relocations will get updated in the local
+  // copies of the sections.
+  for (unsigned i = 0, e = Offsets.size(); i != e; ++i) {
+    uint64_t Addr = RemoteAddr + Offsets[i].second;
+    EE->mapSectionAddress(const_cast<void*>(Offsets[i].first->MB.base()), Addr);
+
+    DEBUG(dbgs() << "  Mapping local: " << Offsets[i].first->MB.base()
+                 << " to remote: 0x" << format("%llx", Addr) << "\n");
+
+    MappedSections[Addr] = Offsets[i].first;
+  }
+
+  UnmappedSections.clear();
+}
+
+bool RemoteMemoryManager::finalizeMemory(std::string *ErrMsg) {
+  // FIXME: Make this function thread safe.
+  for (DenseMap<uint64_t, const Allocation*>::iterator
+         I = MappedSections.begin(), E = MappedSections.end();
+       I != E; ++I) {
+    uint64_t RemoteAddr = I->first;
+    const Allocation *Section = I->second;
+    if (Section->IsCode) {
+      Target->loadCode(RemoteAddr, Section->MB.base(), Section->MB.size());
+
+      DEBUG(dbgs() << "  loading code: " << Section->MB.base()
+            << " to remote: 0x" << format("%llx", RemoteAddr) << "\n");
+    } else {
+      Target->loadData(RemoteAddr, Section->MB.base(), Section->MB.size());
+
+      DEBUG(dbgs() << "  loading data: " << Section->MB.base()
+            << " to remote: 0x" << format("%llx", RemoteAddr) << "\n");
+    }
+  }
+
+  MappedSections.clear();
+
+  return false;
+}
+
+void RemoteMemoryManager::setMemoryWritable() { llvm_unreachable("Unexpected!"); }
+void RemoteMemoryManager::setMemoryExecutable() { llvm_unreachable("Unexpected!"); }
+void RemoteMemoryManager::setPoisonMemory(bool poison) { llvm_unreachable("Unexpected!"); }
+void RemoteMemoryManager::AllocateGOT() { llvm_unreachable("Unexpected!"); }
+uint8_t *RemoteMemoryManager::getGOTBase() const {
+  llvm_unreachable("Unexpected!");
+  return 0;
+}
+uint8_t *RemoteMemoryManager::startFunctionBody(const Function *F, uintptr_t &ActualSize){
+  llvm_unreachable("Unexpected!");
+  return 0;
+}
+uint8_t *RemoteMemoryManager::allocateStub(const GlobalValue* F, unsigned StubSize,
+                                              unsigned Alignment) {
+  llvm_unreachable("Unexpected!");
+  return 0;
+}
+void RemoteMemoryManager::endFunctionBody(const Function *F, uint8_t *FunctionStart,
+                                             uint8_t *FunctionEnd) {
+  llvm_unreachable("Unexpected!");
+}
+uint8_t *RemoteMemoryManager::allocateSpace(intptr_t Size, unsigned Alignment) {
+  llvm_unreachable("Unexpected!");
+  return 0;
+}
+uint8_t *RemoteMemoryManager::allocateGlobal(uintptr_t Size, unsigned Alignment) {
+  llvm_unreachable("Unexpected!");
+  return 0;
+}
+void RemoteMemoryManager::deallocateFunctionBody(void *Body) {
+  llvm_unreachable("Unexpected!");
+}
+
+static int jit_noop() {
+  return 0;
+}
+
+void *RemoteMemoryManager::getPointerToNamedFunction(const std::string &Name,
+                                                        bool AbortOnFailure) {
+  // We should not invoke parent's ctors/dtors from generated main()!
+  // On Mingw and Cygwin, the symbol __main is resolved to
+  // callee's(eg. tools/lli) one, to invoke wrong duplicated ctors
+  // (and register wrong callee's dtors with atexit(3)).
+  // We expect ExecutionEngine::runStaticConstructorsDestructors()
+  // is called before ExecutionEngine::runFunctionAsMain() is called.
+  if (Name == "__main") return (void*)(intptr_t)&jit_noop;
+
+  // FIXME: Would it be responsible to provide GOT?
+  if (AbortOnFailure) {
+    if (Name == "_GLOBAL_OFFSET_TABLE_")
+      report_fatal_error("Program used external function '" + Name +
+                         "' which could not be resolved!");
+  }
+
+  return NULL;
+}
diff --git a/tools/lli/RecordingMemoryManager.h b/tools/lli/RemoteMemoryManager.h
index 05f4807d04..dc74666672 100644
--- a/tools/lli/RecordingMemoryManager.h
+++ b/tools/lli/RemoteMemoryManager.h
@@ -1,4 +1,4 @@
-//===- RecordingMemoryManager.h - LLI MCJIT recording memory manager ------===//
+//===- RemoteMemoryManager.h - LLI MCJIT recording memory manager ------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -12,24 +12,47 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef RECORDINGMEMORYMANAGER_H
-#define RECORDINGMEMORYMANAGER_H
+#ifndef REMOTEMEMORYMANAGER_H
+#define REMOTEMEMORYMANAGER_H
 
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ExecutionEngine/JITMemoryManager.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Memory.h"
 #include <utility>
 
+#include "RemoteTarget.h"
+
 namespace llvm {
 
-class RecordingMemoryManager : public JITMemoryManager {
+class RemoteMemoryManager : public JITMemoryManager {
 public:
-  typedef std::pair<sys::MemoryBlock, unsigned> Allocation;
+  // Notice that this structure takes ownership of the memory allocated.
+  struct Allocation {
+    Allocation(sys::MemoryBlock mb, unsigned a, bool code)
+      : MB(mb), Alignment(a), IsCode(code) {}
+
+    sys::MemoryBlock  MB;
+    unsigned          Alignment;
+    bool              IsCode;
+  };
 
 private:
-  SmallVector<Allocation, 16> AllocatedDataMem;
-  SmallVector<Allocation, 16> AllocatedCodeMem;
+  // This vector contains Allocation objects for all sections which we have
+  // allocated.  This vector effectively owns the memory associated with the
+  // allocations.
+  SmallVector<Allocation, 2>  AllocatedSections;
+
+  // This vector contains pointers to Allocation objects for any sections we
+  // have allocated locally but have not yet remapped for the remote target.
+  // When we receive notification of a completed module load, we will map
+  // these sections into the remote target.
+  SmallVector<const Allocation *, 2>  UnmappedSections;
+
+  // This map tracks the sections we have remapped for the remote target
+  // but have not yet copied to the target.
+  DenseMap<uint64_t, const Allocation *>  MappedSections;
 
   // FIXME: This is part of a work around to keep sections near one another
   // when MCJIT performs relocations after code emission but before
@@ -37,17 +60,11 @@ private:
   sys::MemoryBlock Near;
   sys::MemoryBlock allocateSection(uintptr_t Size);
 
-public:
-  RecordingMemoryManager() {}
-  virtual ~RecordingMemoryManager();
+  RemoteTarget *Target;
 
-  typedef SmallVectorImpl<Allocation>::const_iterator const_data_iterator;
-  typedef SmallVectorImpl<Allocation>::const_iterator const_code_iterator;
-
-  const_data_iterator data_begin() const { return AllocatedDataMem.begin(); }
-  const_data_iterator   data_end() const { return AllocatedDataMem.end(); }
-  const_code_iterator code_begin() const { return AllocatedCodeMem.begin(); }
-  const_code_iterator   code_end() const { return AllocatedCodeMem.end(); }
+public:
+  RemoteMemoryManager() : Target(NULL) {}
+  virtual ~RemoteMemoryManager();
 
   uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,
                                unsigned SectionID, StringRef SectionName);
@@ -59,7 +76,12 @@ public:
   void *getPointerToNamedFunction(const std::string &Name,
                                   bool AbortOnFailure = true);
 
-  bool finalizeMemory(std::string *ErrMsg) { return false; }
+  void notifyObjectLoaded(ExecutionEngine *EE, const ObjectImage *Obj);
+
+  bool finalizeMemory(std::string *ErrMsg);
+
+  // This is a non-interface function used by lli
+  void setRemoteTarget(RemoteTarget *T) { Target = T; }
 
   // The following obsolete JITMemoryManager calls are stubbed out for
   // this model.
diff --git a/tools/lli/lli.cpp b/tools/lli/lli.cpp
index 1cdd91b693..ac0d219272 100644
--- a/tools/lli/lli.cpp
+++ b/tools/lli/lli.cpp
@@ -15,7 +15,7 @@
 
 #define DEBUG_TYPE "lli"
 #include "llvm/IR/LLVMContext.h"
-#include "RecordingMemoryManager.h"
+#include "RemoteMemoryManager.h"
 #include "RemoteTarget.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Bitcode/ReaderWriter.h"
@@ -131,6 +131,12 @@ namespace {
             cl::value_desc("function"),
             cl::init("main"));
 
+  cl::list<std::string>
+  ExtraModules("extra-modules",
+         cl::CommaSeparated,
+         cl::desc("Extra modules to be loaded"),
+         cl::value_desc("<input bitcode 2>,<input bitcode 3>,..."));
+
   cl::opt<std::string>
   FakeArgv0("fake-argv0",
             cl::desc("Override the 'argv[0]' value passed into the executing"
@@ -222,82 +228,6 @@ static void do_shutdown() {
 #endif
 }
 
-void layoutRemoteTargetMemory(RemoteTarget *T, RecordingMemoryManager *JMM) {
-  // Lay out our sections in order, with all the code sections first, then
-  // all the data sections.
-  uint64_t CurOffset = 0;
-  unsigned MaxAlign = T->getPageAlignment();
-  SmallVector<std::pair<const void*, uint64_t>, 16> Offsets;
-  SmallVector<unsigned, 16> Sizes;
-  for (RecordingMemoryManager::const_code_iterator I = JMM->code_begin(),
-                                                   E = JMM->code_end();
-       I != E; ++I) {
-    DEBUG(dbgs() << "code region: size " << I->first.size()
-                 << ", alignment " << I->second << "\n");
-    // Align the current offset up to whatever is needed for the next
-    // section.
-    unsigned Align = I->second;
-    CurOffset = (CurOffset + Align - 1) / Align * Align;
-    // Save off the address of the new section and allocate its space.
-    Offsets.push_back(std::pair<const void*,uint64_t>(I->first.base(), CurOffset));
-    Sizes.push_back(I->first.size());
-    CurOffset += I->first.size();
-  }
-  // Adjust to keep code and data aligned on seperate pages.
-  CurOffset = (CurOffset + MaxAlign - 1) / MaxAlign * MaxAlign;
-  unsigned FirstDataIndex = Offsets.size();
-  for (RecordingMemoryManager::const_data_iterator I = JMM->data_begin(),
-                                                   E = JMM->data_end();
-       I != E; ++I) {
-    DEBUG(dbgs() << "data region: size " << I->first.size()
-                 << ", alignment " << I->second << "\n");
-    // Align the current offset up to whatever is needed for the next
-    // section.
-    unsigned Align = I->second;
-    CurOffset = (CurOffset + Align - 1) / Align * Align;
-    // Save off the address of the new section and allocate its space.
-    Offsets.push_back(std::pair<const void*,uint64_t>(I->first.base(), CurOffset));
-    Sizes.push_back(I->first.size());
-    CurOffset += I->first.size();
-  }
-
-  // Allocate space in the remote target.
-  uint64_t RemoteAddr;
-  if (T->allocateSpace(CurOffset, MaxAlign, RemoteAddr))
-    report_fatal_error(T->getErrorMsg());
-  // Map the section addresses so relocations will get updated in the local
-  // copies of the sections.
-  for (unsigned i = 0, e = Offsets.size(); i != e; ++i) {
-    uint64_t Addr = RemoteAddr + Offsets[i].second;
-    EE->mapSectionAddress(const_cast<void*>(Offsets[i].first), Addr);
-
-    DEBUG(dbgs() << "  Mapping local: " << Offsets[i].first
-                 << " to remote: 0x" << format("%llx", Addr) << "\n");
-
-  }
-
-  // Trigger application of relocations
-  EE->finalizeObject();
-
-  // Now load it all to the target.
-  for (unsigned i = 0, e = Offsets.size(); i != e; ++i) {
-    uint64_t Addr = RemoteAddr + Offsets[i].second;
-
-    if (i < FirstDataIndex) {
-      T->loadCode(Addr, Offsets[i].first, Sizes[i]);
-
-      DEBUG(dbgs() << "  loading code: " << Offsets[i].first
-            << " to remote: 0x" << format("%llx", Addr) << "\n");
-    } else {
-      T->loadData(Addr, Offsets[i].first, Sizes[i]);
-
-      DEBUG(dbgs() << "  loading data: " << Offsets[i].first
-            << " to remote: 0x" << format("%llx", Addr) << "\n");
-    }
-
-  }
-}
-
 //===----------------------------------------------------------------------===//
 // main Driver function
 //
@@ -370,7 +300,7 @@ int main(int argc, char **argv, char * const *envp) {
   if (UseMCJIT && !ForceInterpreter) {
     builder.setUseMCJIT(true);
     if (RemoteMCJIT)
-      RTDyldMM = new RecordingMemoryManager();
+      RTDyldMM = new RemoteMemoryManager();
     else
       RTDyldMM = new SectionMemoryManager();
     builder.setMCJITMemoryManager(RTDyldMM);
@@ -420,6 +350,16 @@ int main(int argc, char **argv, char * const *envp) {
     exit(1);
   }
 
+  // Load any additional modules specified on the command line.
+  for (unsigned i = 0, e = ExtraModules.size(); i != e; ++i) {
+    Module *XMod = ParseIRFile(ExtraModules[i], Err, Context);
+    if (!XMod) {
+      Err.print(argv[0], errs());
+      return 1;
+    }
+    EE->addModule(XMod);
+  }
+
   // The following functions have no effect if their respective profiling
   // support wasn't enabled in the build configuration.
   EE->RegisterJITEventListener(
@@ -519,7 +459,7 @@ int main(int argc, char **argv, char * const *envp) {
     // it couldn't. This is a limitation of the LLI implemantation, not the
     // MCJIT itself. FIXME.
     //
-    RecordingMemoryManager *MM = static_cast<RecordingMemoryManager*>(RTDyldMM);
+    RemoteMemoryManager *MM = static_cast<RemoteMemoryManager*>(RTDyldMM);
     // Everything is prepared now, so lay out our program for the target
     // address space, assign the section addresses to resolve any relocations,
     // and send it to the target.
@@ -543,19 +483,30 @@ int main(int argc, char **argv, char * const *envp) {
       Target.reset(RemoteTarget::createRemoteTarget());
     }
 
-    // Create the remote target
+    // Give the memory manager a pointer to our remote target interface object.
+    MM->setRemoteTarget(Target.get());
+
+    // Create the remote target.
     Target->create();
 
+// FIXME: Don't commit like this.  I don't think these calls are necessary.
+#if 0
     // Trigger compilation.
     EE->generateCodeForModule(Mod);
 
-    // Layout the target memory.
-    layoutRemoteTargetMemory(Target.get(), MM);
+    // Get everything ready to execute.
+    EE->finalizeModule(Mod);
+#endif
 
     // Since we're executing in a (at least simulated) remote address space,
     // we can't use the ExecutionEngine::runFunctionAsMain(). We have to
     // grab the function address directly here and tell the remote target
     // to execute the function.
+    //
+    // Our memory manager will map generated code into the remote address
+    // space as it is loaded and copy the bits over during the finalizeMemory
+    // operation.
+    //
     // FIXME: argv and envp handling.
     uint64_t Entry = EE->getFunctionAddress(EntryFn->getName().str());