Re-apply r196639: Add support for archives and object file caching under MCJIT.

I believe the bot failures on linux systems were due to overestimating the
alignment of object-files within archives, which are only guaranteed to be
two-byte aligned. I have reduced the alignment in
RuntimeDyldELF::createObjectImageFromFile accordingly.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@198737 91177308-0d34-0410-b5e6-96231b3b80d8

9 files changed, 149 insertions, 10 deletions

diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.cpp b/lib/ExecutionEngine/MCJIT/MCJIT.cpp
index 625fc03856..2d5b59f83f 100644
--- a/lib/ExecutionEngine/MCJIT/MCJIT.cpp
+++ b/lib/ExecutionEngine/MCJIT/MCJIT.cpp
@@ -22,6 +22,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/PassManager.h"
+#include "llvm/Object/Archive.h"
 #include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -78,15 +79,24 @@ MCJIT::~MCJIT() {
   Modules.clear();
   Dyld.deregisterEHFrames();
 
-  LoadedObjectMap::iterator it, end = LoadedObjects.end();
-  for (it = LoadedObjects.begin(); it != end; ++it) {
-    ObjectImage *Obj = it->second;
+  LoadedObjectList::iterator it, end;
+  for (it = LoadedObjects.begin(), end = LoadedObjects.end(); it != end; ++it) {
+    ObjectImage *Obj = *it;
     if (Obj) {
       NotifyFreeingObject(*Obj);
       delete Obj;
     }
   }
   LoadedObjects.clear();
+
+
+  SmallVector<object::Archive *, 2>::iterator ArIt, ArEnd;
+  for (ArIt = Archives.begin(), ArEnd = Archives.end(); ArIt != ArEnd; ++ArIt) {
+    object::Archive *A = *ArIt;
+    delete A;
+  }
+  Archives.clear();
+
   delete TM;
 }
 
@@ -102,6 +112,21 @@ bool MCJIT::removeModule(Module *M) {
 
 
 
+void MCJIT::addObjectFile(object::ObjectFile *Obj) {
+  ObjectImage *LoadedObject = Dyld.loadObject(Obj);
+  if (!LoadedObject)
+    report_fatal_error(Dyld.getErrorString());
+
+  LoadedObjects.push_back(LoadedObject);
+
+  NotifyObjectEmitted(*LoadedObject);
+}
+
+void MCJIT::addArchive(object::Archive *A) {
+  Archives.push_back(A);
+}
+
+
 void MCJIT::setObjectCache(ObjectCache* NewCache) {
   MutexGuard locked(lock);
   ObjCache = NewCache;
@@ -171,9 +196,9 @@ void MCJIT::generateCodeForModule(Module *M) {
   }
 
   // Load the object into the dynamic linker.
-  // MCJIT now owns the ObjectImage pointer (via its LoadedObjects map).
+  // MCJIT now owns the ObjectImage pointer (via its LoadedObjects list).
   ObjectImage *LoadedObject = Dyld.loadObject(ObjectToLoad.take());
-  LoadedObjects[M] = LoadedObject;
+  LoadedObjects.push_back(LoadedObject);
   if (!LoadedObject)
     report_fatal_error(Dyld.getErrorString());
 
@@ -271,6 +296,27 @@ uint64_t MCJIT::getSymbolAddress(const std::string &Name,
   if (Addr)
     return Addr;
 
+  SmallVector<object::Archive*, 2>::iterator I, E;
+  for (I = Archives.begin(), E = Archives.end(); I != E; ++I) {
+    object::Archive *A = *I;
+    // Look for our symbols in each Archive
+    object::Archive::child_iterator ChildIt = A->findSym(Name);
+    if (ChildIt != A->end_children()) {
+      OwningPtr<object::Binary> ChildBin;
+      // FIXME: Support nested archives?
+      if (!ChildIt->getAsBinary(ChildBin) && ChildBin->isObject()) {
+        object::ObjectFile *OF = reinterpret_cast<object::ObjectFile *>(
+                                                            ChildBin.take());
+        // This causes the object file to be loaded.
+        addObjectFile(OF);
+        // The address should be here now.
+        Addr = getExistingSymbolAddress(Name);
+        if (Addr)
+          return Addr;
+      }
+    }
+  }
+
   // If it hasn't already been generated, see if it's in one of our modules.
   Module *M = findModuleForSymbol(Name, CheckFunctionsOnly);
   if (!M)
diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.h b/lib/ExecutionEngine/MCJIT/MCJIT.h
index 58381c3c34..44fd6bce47 100644
--- a/lib/ExecutionEngine/MCJIT/MCJIT.h
+++ b/lib/ExecutionEngine/MCJIT/MCJIT.h
@@ -206,8 +206,10 @@ class MCJIT : public ExecutionEngine {
 
   OwningModuleContainer OwnedModules;
 
-  typedef DenseMap<Module *, ObjectImage *> LoadedObjectMap;
-  LoadedObjectMap  LoadedObjects;
+  SmallVector<object::Archive*, 2> Archives;
+
+  typedef SmallVector<ObjectImage *, 2> LoadedObjectList;
+  LoadedObjectList  LoadedObjects;
 
   // An optional ObjectCache to be notified of compiled objects and used to
   // perform lookup of pre-compiled code to avoid re-compilation.
@@ -227,6 +229,8 @@ public:
   /// @name ExecutionEngine interface implementation
   /// @{
   virtual void addModule(Module *M);
+  virtual void addObjectFile(object::ObjectFile *O);
+  virtual void addArchive(object::Archive *O);
   virtual bool removeModule(Module *M);
 
   /// FindFunctionNamed - Search all of the active modules to find the one that
diff --git a/lib/ExecutionEngine/RuntimeDyld/ObjectImageCommon.h b/lib/ExecutionEngine/RuntimeDyld/ObjectImageCommon.h
index 28680044a3..7666a869d6 100644
--- a/lib/ExecutionEngine/RuntimeDyld/ObjectImageCommon.h
+++ b/lib/ExecutionEngine/RuntimeDyld/ObjectImageCommon.h
@@ -20,6 +20,10 @@
 
 namespace llvm {
 
+namespace object {
+  class ObjectFile;
+}
+
 class ObjectImageCommon : public ObjectImage {
   ObjectImageCommon(); // = delete
   ObjectImageCommon(const ObjectImageCommon &other); // = delete
@@ -42,6 +46,8 @@ public:
   {
     ObjFile = object::ObjectFile::createObjectFile(Buffer->getMemBuffer());
   }
+  ObjectImageCommon(object::ObjectFile* Input)
+  : ObjectImage(NULL), ObjFile(Input)  {}
   virtual ~ObjectImageCommon() { delete ObjFile; }
 
   virtual object::symbol_iterator begin_symbols() const
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
index 14e867d7c1..f99d7d0b27 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
@@ -82,12 +82,24 @@ ObjectImage *RuntimeDyldImpl::createObjectImage(ObjectBuffer *InputBuffer) {
   return new ObjectImageCommon(InputBuffer);
 }
 
+ObjectImage *RuntimeDyldImpl::createObjectImageFromFile(ObjectFile *InputObject) {
+  return new ObjectImageCommon(InputObject);
+}
+
+ObjectImage *RuntimeDyldImpl::loadObject(ObjectFile *InputObject) {
+  return loadObject(createObjectImageFromFile(InputObject));
+}
+
 ObjectImage *RuntimeDyldImpl::loadObject(ObjectBuffer *InputBuffer) {
+  return loadObject(createObjectImage(InputBuffer));
+} 
+
+ObjectImage *RuntimeDyldImpl::loadObject(ObjectImage *InputObject) {
   MutexGuard locked(lock);
 
-  OwningPtr<ObjectImage> obj(createObjectImage(InputBuffer));
+  OwningPtr<ObjectImage> obj(InputObject);
   if (!obj)
-    report_fatal_error("Unable to create object image from memory buffer!");
+    return NULL;
 
   // Save information about our target
   Arch = (Triple::ArchType)obj->getArch();
@@ -139,7 +151,7 @@ ObjectImage *RuntimeDyldImpl::loadObject(ObjectBuffer *InputBuffer) {
         if (si == obj->end_sections()) continue;
         Check(si->getContents(SectionData));
         Check(si->isText(IsCode));
-        const uint8_t* SymPtr = (const uint8_t*)InputBuffer->getBufferStart() +
+        const uint8_t* SymPtr = (const uint8_t*)InputObject->getData().data() +
                                 (uintptr_t)FileOffset;
         uintptr_t SectOffset = (uintptr_t)(SymPtr -
                                            (const uint8_t*)SectionData.begin());
@@ -563,6 +575,22 @@ RuntimeDyld::~RuntimeDyld() {
   delete Dyld;
 }
 
+ObjectImage *RuntimeDyld::loadObject(ObjectFile *InputObject) {
+  if (!Dyld) {
+    if (InputObject->isELF())
+      Dyld = new RuntimeDyldELF(MM);
+    else if (InputObject->isMachO())
+      Dyld = new RuntimeDyldMachO(MM);
+    else
+      report_fatal_error("Incompatible object format!");
+  } else {
+    if (!Dyld->isCompatibleFile(InputObject))
+      report_fatal_error("Incompatible object format!");
+  }
+
+  return Dyld->loadObject(InputObject);
+}
+
 ObjectImage *RuntimeDyld::loadObject(ObjectBuffer *InputBuffer) {
   if (!Dyld) {
     sys::fs::file_magic Type =
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
index f2c69fc99c..ec9193120d 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
@@ -25,6 +25,8 @@
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/ELF.h"
+#include "llvm/Support/MemoryBuffer.h"
+
 using namespace llvm;
 using namespace llvm::object;
 
@@ -178,6 +180,39 @@ void RuntimeDyldELF::deregisterEHFrames() {
   RegisteredEHFrameSections.clear();
 }
 
+ObjectImage *RuntimeDyldELF::createObjectImageFromFile(object::ObjectFile *ObjFile) {
+  if (!ObjFile)
+    return NULL;
+
+  error_code ec;
+  MemoryBuffer* Buffer = MemoryBuffer::getMemBuffer(ObjFile->getData(), 
+                                                    "", 
+                                                    false);
+
+  if (ObjFile->getBytesInAddress() == 4 && ObjFile->isLittleEndian()) {
+    DyldELFObject<ELFType<support::little, 2, false> > *Obj =
+      new DyldELFObject<ELFType<support::little, 2, false> >(Buffer, ec);
+    return new ELFObjectImage<ELFType<support::little, 2, false> >(NULL, Obj);
+  }
+  else if (ObjFile->getBytesInAddress() == 4 && !ObjFile->isLittleEndian()) {
+    DyldELFObject<ELFType<support::big, 2, false> > *Obj =
+      new DyldELFObject<ELFType<support::big, 2, false> >(Buffer, ec);
+    return new ELFObjectImage<ELFType<support::big, 2, false> >(NULL, Obj);
+  }
+  else if (ObjFile->getBytesInAddress() == 8 && !ObjFile->isLittleEndian()) {
+    DyldELFObject<ELFType<support::big, 2, true> > *Obj =
+      new DyldELFObject<ELFType<support::big, 2, true> >(Buffer, ec);
+    return new ELFObjectImage<ELFType<support::big, 2, true> >(NULL, Obj);
+  }
+  else if (ObjFile->getBytesInAddress() == 8 && ObjFile->isLittleEndian()) {
+    DyldELFObject<ELFType<support::little, 2, true> > *Obj =
+      new DyldELFObject<ELFType<support::little, 2, true> >(Buffer, ec);
+    return new ELFObjectImage<ELFType<support::little, 2, true> >(NULL, Obj);
+  }
+  else
+    llvm_unreachable("Unexpected ELF format");
+}
+
 ObjectImage *RuntimeDyldELF::createObjectImage(ObjectBuffer *Buffer) {
   if (Buffer->getBufferSize() < ELF::EI_NIDENT)
     llvm_unreachable("Unexpected ELF object size");
@@ -1403,4 +1438,9 @@ bool RuntimeDyldELF::isCompatibleFormat(const ObjectBuffer *Buffer) const {
     return false;
   return (memcmp(Buffer->getBufferStart(), ELF::ElfMagic, strlen(ELF::ElfMagic))) == 0;
 }
+
+bool RuntimeDyldELF::isCompatibleFile(const object::ObjectFile *Obj) const {
+  return Obj->isELF();
+}
+
 } // namespace llvm
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h
index 3adf82706a..181964faa9 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h
@@ -140,7 +140,9 @@ public:
                                     const SymbolTableMap &Symbols,
                                     StubMap &Stubs);
   virtual bool isCompatibleFormat(const ObjectBuffer *Buffer) const;
+  virtual bool isCompatibleFile(const object::ObjectFile *Buffer) const;
   virtual ObjectImage *createObjectImage(ObjectBuffer *InputBuffer);
+  virtual ObjectImage *createObjectImageFromFile(object::ObjectFile *Obj);
   virtual void registerEHFrames();
   virtual void deregisterEHFrames();
   virtual void finalizeLoad(ObjSectionToIDMap &SectionMap);
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
index 634f6cebe1..56970aebef 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
@@ -311,12 +311,18 @@ protected:
   virtual void updateGOTEntries(StringRef Name, uint64_t Addr) {}
 
   virtual ObjectImage *createObjectImage(ObjectBuffer *InputBuffer);
+  virtual ObjectImage *createObjectImageFromFile(object::ObjectFile *InputObject);
+
+  // This is the implementation for the two public overloads
+  ObjectImage *loadObject(ObjectImage *InputObject);
+
 public:
   RuntimeDyldImpl(RTDyldMemoryManager *mm) : MemMgr(mm), HasError(false) {}
 
   virtual ~RuntimeDyldImpl();
 
   ObjectImage *loadObject(ObjectBuffer *InputBuffer);
+  ObjectImage *loadObject(object::ObjectFile *InputObject);
 
   void *getSymbolAddress(StringRef Name) {
     // FIXME: Just look up as a function for now. Overly simple of course.
@@ -354,6 +360,7 @@ public:
   StringRef getErrorString() { return ErrorStr; }
 
   virtual bool isCompatibleFormat(const ObjectBuffer *Buffer) const = 0;
+  virtual bool isCompatibleFile(const ObjectFile *Obj) const = 0;
 
   virtual void registerEHFrames();
 
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp
index 66ee50ebe9..d12e7e5e4d 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp
@@ -455,4 +455,9 @@ bool RuntimeDyldMachO::isCompatibleFormat(
   return false;
 }
 
+bool RuntimeDyldMachO::isCompatibleFile(
+        const object::ObjectFile *Obj) const {
+  return Obj->isMachO();
+}
+
 } // end namespace llvm
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h
index 3e0870da02..d42e2972aa 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h
@@ -94,6 +94,7 @@ public:
                                     const SymbolTableMap &Symbols,
                                     StubMap &Stubs);
   virtual bool isCompatibleFormat(const ObjectBuffer *Buffer) const;
+  virtual bool isCompatibleFile(const object::ObjectFile *Obj) const;
   virtual void registerEHFrames();
   virtual void finalizeLoad(ObjSectionToIDMap &SectionMap);
 };