Add support to the JIT for true non-lazy operation. When a call to a function

that has not been JIT'd yet, the callee is put on a list of pending functions
to JIT.  The call is directed through a stub, which is updated with the address
of the function after it has been JIT'd.  A new interface for allocating and
updating empty stubs is provided.

Add support for removing the ModuleProvider the JIT was created with, which
would otherwise invalidate the JIT's PassManager, which is initialized with the
ModuleProvider's Module.

Add support under a new ExecutionEngine flag for emitting the infomration 
necessary to update Function and GlobalVariable stubs after JITing them, by
recording the address of the stub and the name of the GlobalValue.  This allows
code to be copied from one address space to another, where libraries may live
at different virtual addresses, and have the stubs updated with their new
correct target addresses.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@64906 91177308-0d34-0410-b5e6-96231b3b80d8

5 files changed, 225 insertions, 40 deletions

diff --git a/lib/ExecutionEngine/ExecutionEngine.cpp b/lib/ExecutionEngine/ExecutionEngine.cpp
index 4678d0c183..33f29b4e4c 100644
--- a/lib/ExecutionEngine/ExecutionEngine.cpp
+++ b/lib/ExecutionEngine/ExecutionEngine.cpp
@@ -42,6 +42,7 @@ ExecutionEngine::ExecutionEngine(ModuleProvider *P) : LazyFunctionCreator(0) {
   LazyCompilationDisabled = false;
   GVCompilationDisabled   = false;
   SymbolSearchingDisabled = false;
+  DlsymStubsEnabled       = false;
   Modules.push_back(P);
   assert(P && "ModuleProvider is null?");
 }
diff --git a/lib/ExecutionEngine/JIT/JIT.cpp b/lib/ExecutionEngine/JIT/JIT.cpp
index 008c5907d8..6940d85d75 100644
--- a/lib/ExecutionEngine/JIT/JIT.cpp
+++ b/lib/ExecutionEngine/JIT/JIT.cpp
@@ -289,11 +289,28 @@ Module *JIT::removeModuleProvider(ModuleProvider *MP, std::string *E) {
   Module *result = ExecutionEngine::removeModuleProvider(MP, E);
   
   MutexGuard locked(lock);
-  if (Modules.empty()) {
+  
+  if (jitstate->getMP() == MP) {
     delete jitstate;
     jitstate = 0;
   }
   
+  if (!jitstate && !Modules.empty()) {
+    jitstate = new JITState(Modules[0]);
+
+    FunctionPassManager &PM = jitstate->getPM(locked);
+    PM.add(new TargetData(*TM.getTargetData()));
+    
+    // Turn the machine code intermediate representation into bytes in memory
+    // that may be executed.
+    if (TM.addPassesToEmitMachineCode(PM, *MCE, false /*fast*/)) {
+      cerr << "Target does not support machine code emission!\n";
+      abort();
+    }
+    
+    // Initialize passes.
+    PM.doInitialization();
+  }    
   return result;
 }
 
@@ -304,10 +321,28 @@ void JIT::deleteModuleProvider(ModuleProvider *MP, std::string *E) {
   ExecutionEngine::deleteModuleProvider(MP, E);
   
   MutexGuard locked(lock);
-  if (Modules.empty()) {
+  
+  if (jitstate->getMP() == MP) {
     delete jitstate;
     jitstate = 0;
   }
+
+  if (!jitstate && !Modules.empty()) {
+    jitstate = new JITState(Modules[0]);
+    
+    FunctionPassManager &PM = jitstate->getPM(locked);
+    PM.add(new TargetData(*TM.getTargetData()));
+    
+    // Turn the machine code intermediate representation into bytes in memory
+    // that may be executed.
+    if (TM.addPassesToEmitMachineCode(PM, *MCE, false /*fast*/)) {
+      cerr << "Target does not support machine code emission!\n";
+      abort();
+    }
+    
+    // Initialize passes.
+    PM.doInitialization();
+  }    
 }
 
 /// run - Start execution with the specified function and arguments.
@@ -488,14 +523,26 @@ void JIT::runJITOnFunctionUnlocked(Function *F, const MutexGuard &locked) {
   jitstate->getPM(locked).run(*F);
   isAlreadyCodeGenerating = false;
 
-  // If the function referred to a global variable that had not yet been
-  // emitted, it allocates memory for the global, but doesn't emit it yet.  Emit
-  // all of these globals now.
-  while (!jitstate->getPendingGlobals(locked).empty()) {
-    const GlobalVariable *GV = jitstate->getPendingGlobals(locked).back();
-    jitstate->getPendingGlobals(locked).pop_back();
-    EmitGlobalVariable(GV);
+  // If the function referred to another function that had not yet been
+  // read from bitcode, but we are jitting non-lazily, emit it now.
+  while (!jitstate->getPendingFunctions(locked).empty()) {
+    Function *PF = jitstate->getPendingFunctions(locked).back();
+    jitstate->getPendingFunctions(locked).pop_back();
+
+    // JIT the function
+    isAlreadyCodeGenerating = true;
+    jitstate->getPM(locked).run(*PF);
+    isAlreadyCodeGenerating = false;
+    
+    // Now that the function has been jitted, ask the JITEmitter to rewrite
+    // the stub with real address of the function.
+    updateFunctionStub(PF);
   }
+  
+  // If the JIT is configured to emit info so that dlsym can be used to
+  // rewrite stubs to external globals, do so now.
+  if (areDlsymStubsEnabled() && isLazyCompilationDisabled())
+    updateDlsymStubTable();
 }
 
 /// getPointerToFunction - This method is used to get the address of the
@@ -644,3 +691,8 @@ char* JIT::getMemoryForGV(const GlobalVariable* GV) {
     return new char[GVSize];
   }
 }
+
+void JIT::addPendingFunction(Function *F) {
+  MutexGuard locked(lock);
+  jitstate->getPendingFunctions(locked).push_back(F);
+}
diff --git a/lib/ExecutionEngine/JIT/JIT.h b/lib/ExecutionEngine/JIT/JIT.h
index 395e04926d..e0c74f8738 100644
--- a/lib/ExecutionEngine/JIT/JIT.h
+++ b/lib/ExecutionEngine/JIT/JIT.h
@@ -20,8 +20,6 @@
 namespace llvm {
 
 class Function;
-class GlobalValue;
-class Constant;
 class TargetMachine;
 class TargetJITInfo;
 class MachineCodeEmitter;
@@ -29,21 +27,22 @@ class MachineCodeEmitter;
 class JITState {
 private:
   FunctionPassManager PM;  // Passes to compile a function
+  ModuleProvider *MP;      // ModuleProvider used to create the PM
 
-  /// PendingGlobals - Global variables which have had memory allocated for them
-  /// while a function was code generated, but which have not been initialized
-  /// yet.
-  std::vector<const GlobalVariable*> PendingGlobals;
+  /// PendingFunctions - Functions which have not been code generated yet, but
+  /// were called from a function being code generated.
+  std::vector<Function*> PendingFunctions;
 
 public:
-  explicit JITState(ModuleProvider *MP) : PM(MP) {}
+  explicit JITState(ModuleProvider *MP) : PM(MP), MP(MP) {}
 
   FunctionPassManager &getPM(const MutexGuard &L) {
     return PM;
   }
-
-  std::vector<const GlobalVariable*> &getPendingGlobals(const MutexGuard &L) {
-    return PendingGlobals;
+  
+  ModuleProvider *getMP() const { return MP; }
+  std::vector<Function*> &getPendingFunctions(const MutexGuard &L) {
+    return PendingFunctions;
   }
 };
 
@@ -139,6 +138,12 @@ public:
   ///
   void freeMachineCodeForFunction(Function *F);
 
+  /// addPendingFunction - while jitting non-lazily, a called but non-codegen'd
+  /// function was encountered.  Add it to a pending list to be processed after 
+  /// the current function.
+  /// 
+  void addPendingFunction(Function *F);
+  
   /// getCodeEmitter - Return the code emitter this JIT is emitting into.
   MachineCodeEmitter *getCodeEmitter() const { return MCE; }
   
@@ -149,7 +154,9 @@ private:
   static MachineCodeEmitter *createEmitter(JIT &J, JITMemoryManager *JMM);
   void runJITOnFunction(Function *F);
   void runJITOnFunctionUnlocked(Function *F, const MutexGuard &locked);
-
+  void updateFunctionStub(Function *F);
+  void updateDlsymStubTable();
+  
 protected:
 
   /// getMemoryforGV - Allocate memory for a global variable.
diff --git a/lib/ExecutionEngine/JIT/JITEmitter.cpp b/lib/ExecutionEngine/JIT/JITEmitter.cpp
index 1067c2287f..1fbeba923c 100644
--- a/lib/ExecutionEngine/JIT/JITEmitter.cpp
+++ b/lib/ExecutionEngine/JIT/JITEmitter.cpp
@@ -36,6 +36,7 @@
 #include "llvm/System/Memory.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include <algorithm>
 #ifndef NDEBUG
@@ -120,8 +121,9 @@ namespace {
     void *getFunctionStubIfAvailable(Function *F);
 
     /// getFunctionStub - This returns a pointer to a function stub, creating
-    /// one on demand as needed.
-    void *getFunctionStub(Function *F);
+    /// one on demand as needed.  If empty is true, create a function stub
+    /// pointing at address 0, to be filled in later.
+    void *getFunctionStub(Function *F, bool empty = false);
 
     /// getExternalFunctionStub - Return a stub for the function at the
     /// specified address, created lazily on demand.
@@ -140,6 +142,9 @@ namespace {
       state.getStubToFunctionMap(locked)[Location] = F;
       return (void*)(intptr_t)LazyResolverFn;
     }
+    
+    void getRelocatableGVs(SmallVectorImpl<GlobalValue*> &GVs,
+                           SmallVectorImpl<void*> &Ptrs);  
 
     /// getGOTIndexForAddress - Return a new or existing index in the GOT for
     /// an address.  This function only manages slots, it does not manage the
@@ -167,7 +172,7 @@ void *JITResolver::getFunctionStubIfAvailable(Function *F) {
 
 /// getFunctionStub - This returns a pointer to a function stub, creating
 /// one on demand as needed.
-void *JITResolver::getFunctionStub(Function *F) {
+void *JITResolver::getFunctionStub(Function *F, bool empty) {
   MutexGuard locked(TheJIT->lock);
 
   // If we already have a stub for this function, recycle it.
@@ -176,7 +181,7 @@ void *JITResolver::getFunctionStub(Function *F) {
 
   // Call the lazy resolver function unless we already KNOW it is an external
   // function, in which case we just skip the lazy resolution step.
-  void *Actual = (void*)(intptr_t)LazyResolverFn;
+  void *Actual = empty ? (void*)0 : (void*)(intptr_t)LazyResolverFn;
   if (F->isDeclaration() && !F->hasNotBeenReadFromBitcode()) {
     Actual = TheJIT->getPointerToFunction(F);
 
@@ -203,6 +208,12 @@ void *JITResolver::getFunctionStub(Function *F) {
   // Finally, keep track of the stub-to-Function mapping so that the
   // JITCompilerFn knows which function to compile!
   state.getStubToFunctionMap(locked)[Stub] = F;
+  
+  // If this is an "empty" stub, then inform the JIT that it will need to
+  // JIT the function so an address can be provided.
+  if (empty)
+    TheJIT->addPendingFunction(F);
+  
   return Stub;
 }
 
@@ -250,6 +261,28 @@ unsigned JITResolver::getGOTIndexForAddr(void* addr) {
   return idx;
 }
 
+void JITResolver::getRelocatableGVs(SmallVectorImpl<GlobalValue*> &GVs,
+                                    SmallVectorImpl<void*> &Ptrs) {
+  MutexGuard locked(TheJIT->lock);
+  
+  std::map<Function*,void*> &FM = state.getFunctionToStubMap(locked);
+  std::map<GlobalValue*,void*> &GM = state.getGlobalToIndirectSymMap(locked);
+  
+  for (std::map<Function*,void*>::iterator i = FM.begin(), e = FM.end();
+       i != e; ++i) {
+    Function *F = i->first;
+    if (F->isDeclaration() && F->hasExternalLinkage()) {
+      GVs.push_back(i->first);
+      Ptrs.push_back(i->second);
+    }
+  }
+  for (std::map<GlobalValue*,void*>::iterator i = GM.begin(), e = GM.end();
+       i != e; ++i) {
+    GVs.push_back(i->first);
+    Ptrs.push_back(i->second);
+  }
+}
+
 /// JITCompilerFn - This function is called when a lazy compilation stub has
 /// been entered.  It looks up which function this stub corresponds to, compiles
 /// it if necessary, then returns the resultant function pointer.
@@ -399,8 +432,7 @@ static void AddFunctionToSymbolTable(const char *FnName,
     JitSymbolEntry *OldSymbols = SymTabPtr->Symbols;
     
     // Copy the old entries over.
-    memcpy(NewSymbols, OldSymbols,
-           SymTabPtr->NumSymbols*sizeof(OldSymbols[0]));
+    memcpy(NewSymbols, OldSymbols, SymTabPtr->NumSymbols*sizeof(OldSymbols[0]));
     
     // Swap the new symbols in, delete the old ones.
     SymTabPtr->Symbols = NewSymbols;
@@ -538,6 +570,8 @@ namespace {
     
     virtual void startGVStub(const GlobalValue* GV, unsigned StubSize,
                                    unsigned Alignment = 1);
+    virtual void startGVStub(const GlobalValue* GV, void *Buffer,
+                             unsigned StubSize);
     virtual void* finishGVStub(const GlobalValue *GV);
 
     /// allocateSpace - Reserves space in the current block if any, or
@@ -591,6 +625,8 @@ namespace {
     void setMemoryExecutable(void) {
       MemMgr->setMemoryExecutable();
     }
+    
+    JITMemoryManager *getMemMgr(void) const { return MemMgr; }
 
   private:
     void *getPointerToGlobal(GlobalValue *GV, void *Reference, bool NoNeedStub);
@@ -605,11 +641,9 @@ namespace {
 
 void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference,
                                      bool DoesntNeedStub) {
-  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) {
-    /// FIXME: If we straightened things out, this could actually emit the
-    /// global immediately instead of queuing it for codegen later!
+  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V))
     return TheJIT->getOrEmitGlobalVariable(GV);
-  }
+
   if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V))
     return TheJIT->getPointerToGlobal(GA->resolveAliasedGlobal(false));
 
@@ -623,15 +657,18 @@ void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference,
     ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F);
   if (ResultPtr) return ResultPtr;
 
-  if (F->isDeclaration() && !F->hasNotBeenReadFromBitcode()) {
-    // If this is an external function pointer, we can force the JIT to
-    // 'compile' it, which really just adds it to the map.
-    if (DoesntNeedStub)
-      return TheJIT->getPointerToFunction(F);
-
-    return Resolver.getFunctionStub(F);
-  }
-
+  // If this is an external function pointer, we can force the JIT to
+  // 'compile' it, which really just adds it to the map.
+  if (F->isDeclaration() && !F->hasNotBeenReadFromBitcode() && DoesntNeedStub)
+    return TheJIT->getPointerToFunction(F);
+
+  // If we are jitting non-lazily but encounter a function that has not been
+  // jitted yet, we need to allocate a blank stub to call the function
+  // once we JIT it and its address is known.
+  if (TheJIT->isLazyCompilationDisabled())
+    if (!F->isDeclaration() || F->hasNotBeenReadFromBitcode())
+      return Resolver.getFunctionStub(F, true);
+  
   // Okay, the function has not been compiled yet, if the target callback
   // mechanism is capable of rewriting the instruction directly, prefer to do
   // that instead of emitting a stub.
@@ -1175,6 +1212,16 @@ void JITEmitter::startGVStub(const GlobalValue* GV, unsigned StubSize,
   BufferEnd = BufferBegin+StubSize+1;
 }
 
+void JITEmitter::startGVStub(const GlobalValue* GV, void *Buffer,
+                             unsigned StubSize) {
+  SavedBufferBegin = BufferBegin;
+  SavedBufferEnd = BufferEnd;
+  SavedCurBufferPtr = CurBufferPtr;
+  
+  BufferBegin = CurBufferPtr = (unsigned char *)Buffer;
+  BufferEnd = BufferBegin+StubSize+1;
+}
+
 void *JITEmitter::finishGVStub(const GlobalValue* GV) {
   NumBytes += getCurrentPCOffset();
   std::swap(SavedBufferBegin, BufferBegin);
@@ -1248,6 +1295,74 @@ void *JIT::getPointerToFunctionOrStub(Function *F) {
   return JE->getJITResolver().getFunctionStub(F);
 }
 
+void JIT::updateFunctionStub(Function *F) {
+  // Get the empty stub we generated earlier.
+  assert(isa<JITEmitter>(MCE) && "Unexpected MCE?");
+  JITEmitter *JE = cast<JITEmitter>(getCodeEmitter());
+  void *Stub = JE->getJITResolver().getFunctionStub(F);
+
+  // Tell the target jit info to rewrite the stub at the specified address,
+  // rather than creating a new one.
+  void *Addr = getPointerToGlobalIfAvailable(F);
+  getJITInfo().emitFunctionStubAtAddr(F, Addr, Stub, *getCodeEmitter());
+}
+
+/// updateDlsymStubTable - Emit the data necessary to relocate the stubs
+/// that were emitted during code generation.
+///
+void JIT::updateDlsymStubTable() {
+  assert(isa<JITEmitter>(MCE) && "Unexpected MCE?");
+  JITEmitter *JE = cast<JITEmitter>(getCodeEmitter());
+  
+  SmallVector<GlobalValue*, 8> GVs;
+  SmallVector<void*, 8> Ptrs;
+
+  JE->getJITResolver().getRelocatableGVs(GVs, Ptrs);
+
+  // If there are no relocatable stubs, return.
+  if (GVs.empty())
+    return;
+
+  // If there are no new relocatable stubs, return.
+  void *CurTable = JE->getMemMgr()->getDlsymTable();
+  if (CurTable && (*(unsigned *)CurTable == GVs.size()))
+    return;
+  
+  // Calculate the size of the stub info
+  unsigned offset    = 4 + 4 * GVs.size();
+  
+  SmallVector<unsigned, 8> Offsets;
+  for (unsigned i = 0; i != GVs.size(); ++i) {
+    Offsets.push_back(offset);
+    offset += GVs[i]->getName().length() + 1;
+  }
+  
+  // FIXME: This currently allocates new space every time it's called.  A
+  // different data structure could be used to make this unnecessary.
+  JE->startGVStub(0, offset, 4);
+  
+  // Emit the number of records
+  MCE->emitInt32(GVs.size());
+  
+  // Emit the string offsets
+  for (unsigned i = 0; i != GVs.size(); ++i)
+    MCE->emitInt32(Offsets[i]);
+  
+  // Emit the pointers
+  for (unsigned i = 0; i != GVs.size(); ++i)
+    if (sizeof(void *) == 8)
+      MCE->emitInt64((intptr_t)Ptrs[i]);
+    else
+      MCE->emitInt32((intptr_t)Ptrs[i]);
+  
+  // Emit the strings
+  for (unsigned i = 0; i != GVs.size(); ++i)
+    MCE->emitString(GVs[i]->getName());
+  
+  // Tell the JIT memory manager where it is.
+  JE->getMemMgr()->SetDlsymTable(JE->finishGVStub(0));
+}
+
 /// freeMachineCodeForFunction - release machine code memory for given Function.
 ///
 void JIT::freeMachineCodeForFunction(Function *F) {
diff --git a/lib/ExecutionEngine/JIT/JITMemoryManager.cpp b/lib/ExecutionEngine/JIT/JITMemoryManager.cpp
index cc072a896c..0dcc71f837 100644
--- a/lib/ExecutionEngine/JIT/JITMemoryManager.cpp
+++ b/lib/ExecutionEngine/JIT/JITMemoryManager.cpp
@@ -258,6 +258,7 @@ namespace {
     
     unsigned char *CurStubPtr, *StubBase;
     unsigned char *GOTBase;      // Target Specific reserved memory
+    void *DlsymTable;            // Stub external symbol information
 
     // Centralize memory block allocation.
     sys::MemoryBlock getNewMemoryBlock(unsigned size);
@@ -269,7 +270,8 @@ namespace {
     ~DefaultJITMemoryManager();
 
     void AllocateGOT();
-
+    void SetDlsymTable(void *);
+    
     unsigned char *allocateStub(const GlobalValue* F, unsigned StubSize,
                                 unsigned Alignment);
     
@@ -343,6 +345,10 @@ namespace {
       return GOTBase;
     }
     
+    void *getDlsymTable() const {
+      return DlsymTable;
+    }
+    
     /// deallocateMemForFunction - Deallocate all memory for the specified
     /// function body.
     void deallocateMemForFunction(const Function *F) {
@@ -463,6 +469,7 @@ DefaultJITMemoryManager::DefaultJITMemoryManager() {
   FreeMemoryList = Mem0;
 
   GOTBase = NULL;
+  DlsymTable = NULL;
 }
 
 void DefaultJITMemoryManager::AllocateGOT() {
@@ -471,6 +478,9 @@ void DefaultJITMemoryManager::AllocateGOT() {
   HasGOT = true;
 }
 
+void DefaultJITMemoryManager::SetDlsymTable(void *ptr) {
+  DlsymTable = ptr;
+}
 
 DefaultJITMemoryManager::~DefaultJITMemoryManager() {
   for (unsigned i = 0, e = Blocks.size(); i != e; ++i)