diff options
author | Shuxin Yang <shuxin.llvm@gmail.com> | 2013-08-12 18:29:43 +0000 |
---|---|---|
committer | Shuxin Yang <shuxin.llvm@gmail.com> | 2013-08-12 18:29:43 +0000 |
commit | 67d135ae40b121a138e334a175d0e02dbb54eeca (patch) | |
tree | 8e7ef892ab3d66802b5f53af56ba6a36a5911742 /tools | |
parent | 353476cbbb80881a754a90abc13851a884a4e26c (diff) | |
download | llvm-67d135ae40b121a138e334a175d0e02dbb54eeca.tar.gz llvm-67d135ae40b121a138e334a175d0e02dbb54eeca.tar.bz2 llvm-67d135ae40b121a138e334a175d0e02dbb54eeca.tar.xz |
Misc enhancements to LTO:
1. Add some helper classes for partitions. They are designed in a
way such that the top-level LTO driver will not see much difference
with or without partitioning.
2. Introduce work-dir. Now all intermediate files generated during
LTO phases will be saved under work-dir. User can specify the workdir
via -lto-workdir=/path/to/dir. By default the work-dir will be
erased before linker exit. To keep the workdir, do -lto-keep, or -lto-keep=1.
TODO: Erase the workdir, if the linker exit prematurely.
We are currently not able to remove directory on signal. The support
routines simply ignore directory.
3. Add one new API lto_codegen_get_files_need_remove().
Linker and LTO plugin will communicate via this API about which files
(including directories) need to removed before linker exit.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@188188 91177308-0d34-0410-b5e6-96231b3b80d8
Diffstat (limited to 'tools')
-rw-r--r-- | tools/gold/gold-plugin.cpp | 32 | ||||
-rw-r--r-- | tools/lto/CMakeLists.txt | 2 | ||||
-rw-r--r-- | tools/lto/LTOCodeGenerator.cpp | 195 | ||||
-rw-r--r-- | tools/lto/LTOCodeGenerator.h | 39 | ||||
-rw-r--r-- | tools/lto/LTOPartition.cpp | 205 | ||||
-rw-r--r-- | tools/lto/LTOPartition.h | 187 | ||||
-rw-r--r-- | tools/lto/LTOPostIPODriver.cpp | 180 | ||||
-rw-r--r-- | tools/lto/LTOPostIPODriver.h | 52 | ||||
-rw-r--r-- | tools/lto/lto.cpp | 13 | ||||
-rw-r--r-- | tools/lto/lto.exports | 1 |
10 files changed, 861 insertions, 45 deletions
diff --git a/tools/gold/gold-plugin.cpp b/tools/gold/gold-plugin.cpp index 77717098d6..7918324702 100644 --- a/tools/gold/gold-plugin.cpp +++ b/tools/gold/gold-plugin.cpp @@ -74,7 +74,6 @@ namespace options { static bool generate_api_file = false; static generate_bc generate_bc_file = BC_NO; static std::string bc_path; - static std::string obj_path; static std::string extra_library_path; static std::string triple; static std::string mcpu; @@ -99,8 +98,6 @@ namespace options { extra_library_path = opt.substr(strlen("extra_library_path=")); } else if (opt.startswith("mtriple=")) { triple = opt.substr(strlen("mtriple=")); - } else if (opt.startswith("obj-path=")) { - obj_path = opt.substr(strlen("obj-path=")); } else if (opt == "emit-llvm") { generate_bc_file = BC_ONLY; } else if (opt == "also-emit-llvm") { @@ -425,6 +422,14 @@ static ld_plugin_status all_symbols_read_hook(void) { (*message)(LDPL_ERROR, "Could not produce a combined object file\n"); } + // Get files that need to be removed in cleanup_hook. + const char *ToRm; + lto_codegen_get_files_need_remove(code_gen, &ToRm); + while (*ToRm) { + Cleanup.push_back(std::string(ToRm)); + ToRm += strlen(ToRm) + 1; + } + lto_codegen_dispose(code_gen); for (std::list<claimed_file>::iterator I = Modules.begin(), E = Modules.end(); I != E; ++I) { @@ -446,17 +451,28 @@ static ld_plugin_status all_symbols_read_hook(void) { return LDPS_ERR; } - if (options::obj_path.empty()) - Cleanup.push_back(objPath); - return LDPS_OK; } static ld_plugin_status cleanup_hook(void) { for (int i = 0, e = Cleanup.size(); i != e; ++i) { - error_code EC = sys::fs::remove(Cleanup[i]); + const char *FN = Cleanup[i].c_str(); + sys::fs::file_status Stat; + error_code EC = sys::fs::status(Twine(FN), Stat); + if (EC) { + (*message)(LDPL_ERROR, "Failed to stat '%s': %s", FN, + EC.message().c_str()); + continue; + } + + uint32_t Dummy; + if (sys::fs::is_directory(FN)) + EC = sys::fs::remove_all(Twine(FN), Dummy); + else + EC = sys::fs::remove(Twine(FN)); + if (EC) - (*message)(LDPL_ERROR, "Failed to delete '%s': %s", Cleanup[i].c_str(), + (*message)(LDPL_ERROR, "Failed to remove '%s': %s", FN, EC.message().c_str()); } diff --git a/tools/lto/CMakeLists.txt b/tools/lto/CMakeLists.txt index 5820b1415b..7667449a34 100644 --- a/tools/lto/CMakeLists.txt +++ b/tools/lto/CMakeLists.txt @@ -9,6 +9,8 @@ set(SOURCES LTODisassembler.cpp lto.cpp LTOModule.cpp + LTOPartition.cpp + LTOPostIPODriver.cpp ) set(LLVM_COMMON_DEPENDS intrinsics_gen) diff --git a/tools/lto/LTOCodeGenerator.cpp b/tools/lto/LTOCodeGenerator.cpp index 3fe7af25af..bcbd01752a 100644 --- a/tools/lto/LTOCodeGenerator.cpp +++ b/tools/lto/LTOCodeGenerator.cpp @@ -14,6 +14,8 @@ #include "LTOCodeGenerator.h" #include "LTOModule.h" +#include "LTOPartition.h" +#include "LTOPostIPODriver.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Analysis/Passes.h" #include "llvm/Analysis/Verifier.h" @@ -35,11 +37,13 @@ #include "llvm/Support/FormattedStream.h" #include "llvm/Support/Host.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Program.h" #include "llvm/Support/Signals.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/TargetSelect.h" #include "llvm/Support/ToolOutputFile.h" #include "llvm/Support/system_error.h" +#include "llvm/Support/SourceMgr.h" #include "llvm/Target/Mangler.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" @@ -47,8 +51,16 @@ #include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/ObjCARC.h" + using namespace llvm; +using namespace lto; +// ///////////////////////////////////////////////////////////////////////////// +// +// Internal options. To avoid collision, most options start with "lto-". +// +// ///////////////////////////////////////////////////////////////////////////// +// static cl::opt<bool> DisableOpt("disable-opt", cl::init(false), cl::desc("Do not run any optimization passes")); @@ -61,6 +73,28 @@ static cl::opt<bool> DisableGVNLoadPRE("disable-gvn-loadpre", cl::init(false), cl::desc("Do not run the GVN load PRE pass")); +// To break merged module into partitions, and compile them independently. +static cl::opt<bool> +EnablePartition("lto-partition", cl::init(false), + cl::desc("Partition program and compile each piece in parallel")); + +// Specify the work-directory for the LTO compilation. All intermeidate +// files will be created immediately under this dir. If it is not +// specified, compiler will create an unique directory under current-dir. +// +static cl::opt<std::string> +TmpWorkDir("lto-workdir", cl::init(""), cl::desc("Specify working directory")); + +static cl::opt<bool> +KeepWorkDir("lto-keep", cl::init(false), cl::desc("Keep working directory")); + + +// ///////////////////////////////////////////////////////////////////////////// +// +// Implementation of LTOCodeGenerator +// +// ///////////////////////////////////////////////////////////////////////////// +// const char* LTOCodeGenerator::getVersionString() { #ifdef LLVM_VERSION_INFO return PACKAGE_NAME " version " PACKAGE_VERSION ", " LLVM_VERSION_INFO; @@ -74,7 +108,8 @@ LTOCodeGenerator::LTOCodeGenerator() _linker(new Module("ld-temp.o", _context)), _target(NULL), _emitDwarfDebugInfo(false), _scopeRestrictionsDone(false), _codeModel(LTO_CODEGEN_PIC_MODEL_DYNAMIC), - _nativeObjectFile(NULL) { + _nativeObjectFile(NULL), PartitionMgr(FileMgr), + OptionsParsed(false) { InitializeAllTargets(); InitializeAllTargetMCs(); InitializeAllAsmPrinters(); @@ -187,35 +222,41 @@ bool LTOCodeGenerator::writeMergedModules(const char *path, return true; } -bool LTOCodeGenerator::compile_to_file(const char** name, std::string& errMsg) { - // make unique temp .o file to put generated object file - SmallString<128> Filename; - int FD; - error_code EC = sys::fs::createTemporaryFile("lto-llvm", "o", FD, Filename); - if (EC) { - errMsg = EC.message(); +// This function is to ensure cl::ParseCommandLineOptions() is called no more +// than once. It would otherwise complain and exit the compilation prematurely. +// +void LTOCodeGenerator::parseOptions() { + if (OptionsParsed) + return; + + if (!_codegenOptions.empty()) + cl::ParseCommandLineOptions(_codegenOptions.size(), + const_cast<char **>(&_codegenOptions[0])); + + OptionsParsed = true; +} + +// Do some prepartion right before compilation starts. +bool LTOCodeGenerator::prepareBeforeCompile(std::string &ErrMsg) { + parseOptions(); + + if (!determineTarget(ErrMsg)) return false; - } - // generate object file - tool_output_file objFile(Filename.c_str(), FD); + FileMgr.setWorkDir(TmpWorkDir.c_str()); + FileMgr.setKeepWorkDir(KeepWorkDir); + return FileMgr.createWorkDir(ErrMsg); +} - bool genResult = generateObjectFile(objFile.os(), errMsg); - objFile.os().close(); - if (objFile.os().has_error()) { - objFile.os().clear_error(); - sys::fs::remove(Twine(Filename)); +bool LTOCodeGenerator::compile_to_file(const char** Name, std::string& ErrMsg) { + if (!prepareBeforeCompile(ErrMsg)) return false; - } - objFile.keep(); - if (!genResult) { - sys::fs::remove(Twine(Filename)); + performIPO(EnablePartition, ErrMsg); + if (!performPostIPO(ErrMsg)) return false; - } - _nativeObjectPath = Filename.c_str(); - *name = _nativeObjectPath.c_str(); + *Name = PartitionMgr.getSinglePartition()->getObjFilePath().c_str(); return true; } @@ -229,21 +270,41 @@ const void* LTOCodeGenerator::compile(size_t* length, std::string& errMsg) { // read .o file into memory buffer OwningPtr<MemoryBuffer> BuffPtr; + const char *BufStart = 0; + if (error_code ec = MemoryBuffer::getFile(name, BuffPtr, -1, false)) { errMsg = ec.message(); - sys::fs::remove(_nativeObjectPath); - return NULL; + _nativeObjectFile = 0; + } else { + if ((_nativeObjectFile = BuffPtr.take())) { + *length = _nativeObjectFile->getBufferSize(); + BufStart = _nativeObjectFile->getBufferStart(); + } } - _nativeObjectFile = BuffPtr.take(); - // remove temp files - sys::fs::remove(_nativeObjectPath); + // Now that the resulting single object file is handed to linker via memory + // buffer, it is safe to remove all intermediate files now. + // + FileMgr.removeAllUnneededFiles(); - // return buffer, unless error - if (_nativeObjectFile == NULL) - return NULL; - *length = _nativeObjectFile->getBufferSize(); - return _nativeObjectFile->getBufferStart(); + return BufStart; +} + +const char *LTOCodeGenerator::getFilesNeedToRemove() { + IPOFileMgr::FileNameVect ToRm; + FileMgr.getFilesNeedToRemove(ToRm); + + ConcatStrings.clear(); + for (IPOFileMgr::FileNameVect::iterator I = ToRm.begin(), E = ToRm.end(); + I != E; I++) { + StringRef S(*I); + ConcatStrings.append(S.begin(), S.end()); + ConcatStrings.push_back('\0'); + } + ConcatStrings.push_back('\0'); + ConcatStrings.push_back('\0'); + + return ConcatStrings.data(); } bool LTOCodeGenerator::determineTarget(std::string &errMsg) { @@ -251,9 +312,7 @@ bool LTOCodeGenerator::determineTarget(std::string &errMsg) { return true; // if options were requested, set them - if (!_codegenOptions.empty()) - cl::ParseCommandLineOptions(_codegenOptions.size(), - const_cast<char **>(&_codegenOptions[0])); + parseOptions(); std::string TripleStr = _linker.getModule()->getTargetTriple(); if (TripleStr.empty()) @@ -384,6 +443,70 @@ void LTOCodeGenerator::applyScopeRestrictions() { _scopeRestrictionsDone = true; } +void LTOCodeGenerator::performIPO(bool ToPartition, std::string &errMsg) { + // Mark which symbols can not be internalized + applyScopeRestrictions(); + + // Instantiate the pass manager to organize the passes. + PassManager Passes; + + // Start off with a verification pass. + Passes.add(createVerifierPass()); + + // Add an appropriate DataLayout instance for this module... + Passes.add(new DataLayout(*_target->getDataLayout())); + _target->addAnalysisPasses(Passes); + + // Enabling internalize here would use its AllButMain variant. It + // keeps only main if it exists and does nothing for libraries. Instead + // we create the pass ourselves with the symbol list provided by the linker. + if (!DisableOpt) + PassManagerBuilder().populateLTOPassManager(Passes, + /*Internalize=*/false, + !DisableInline, + DisableGVNLoadPRE); + // Make sure everything is still good. + Passes.add(createVerifierPass()); + + Module* M = _linker.getModule(); + if (ToPartition) + assert(false && "TBD"); + else { + Passes.run(*M); + + // Create a partition for the merged module. + PartitionMgr.createIPOPart(M); + } +} + +// Perform Post-IPO compilation. If the partition is enabled, there may +// be multiple partitions, and therefore there may be multiple objects. +// In this case, "MergeObjs" indicates to merge all object together (via ld -r) +// and return the path to the merged object via "MergObjPath". +// +bool LTOCodeGenerator::performPostIPO(std::string &ErrMsg, + bool MergeObjs, + const char **MergObjPath) { + // Determine the variant of post-ipo driver + PostIPODriver::VariantTy DrvTy; + if (!EnablePartition) { + assert(!MergeObjs && !MergObjPath && "Invalid parameter"); + DrvTy = PostIPODriver::PIDV_SERIAL; + } else { + DrvTy = PostIPODriver::PIDV_Invalid; + assert(false && "TBD"); + } + + PostIPODriver D(DrvTy, _target, PartitionMgr, FileMgr, MergeObjs); + if (D.Compile(ErrMsg)) { + if (MergeObjs) + *MergObjPath = D.getSingleObjFile()->getPath().c_str(); + return true; + } + + return false; +} + /// Optimize merged modules using various IPO passes bool LTOCodeGenerator::generateObjectFile(raw_ostream &out, std::string &errMsg) { diff --git a/tools/lto/LTOCodeGenerator.h b/tools/lto/LTOCodeGenerator.h index 8f37cf0e1d..5dda5d9c3e 100644 --- a/tools/lto/LTOCodeGenerator.h +++ b/tools/lto/LTOCodeGenerator.h @@ -41,6 +41,7 @@ #include "llvm/Linker.h" #include <string> #include <vector> +#include "LTOPartition.h" namespace llvm { class LLVMContext; @@ -102,16 +103,34 @@ struct LTOCodeGenerator { // const void *compile(size_t *length, std::string &errMsg); + // Return the paths of the intermediate files that linker needs to delete + // before it exits. The paths are delimited by a single '\0', and the last + // path is ended by double '\0's. The file could be a directory. In that + // case, the entire directory should be erased recusively. This function + // must be called after the compilexxx() is successfuly called, because + // only after that moment, compiler is aware which files need to be removed. + // If calling compilexxx() is not successful, it is up to compiler to clean + // up all the intermediate files generated during the compilation process. + // + const char *getFilesNeedToRemove(); + private: void initializeLTOPasses(); + bool determineTarget(std::string &errMsg); + void parseOptions(); + bool prepareBeforeCompile(std::string &ErrMsg); + void performIPO(bool PerformPartition, std::string &ErrMsg); + bool performPostIPO(std::string &ErrMsg, bool MergeObjs = false, + const char **MergObjPath = 0); bool generateObjectFile(llvm::raw_ostream &out, std::string &errMsg); + void applyScopeRestrictions(); void applyRestriction(llvm::GlobalValue &GV, std::vector<const char*> &mustPreserveList, llvm::SmallPtrSet<llvm::GlobalValue*, 8> &asmUsed, llvm::Mangler &mangler); - bool determineTarget(std::string &errMsg); + typedef llvm::StringMap<uint8_t> StringSet; @@ -127,6 +146,24 @@ private: std::vector<char*> _codegenOptions; std::string _mCpu; std::string _nativeObjectPath; + + // To manage the partitions. If partition is not enabled, the whole merged + // module is considered as a single degenerated partition, and the "manager" + // is still active. + lto::IPOPartMgr PartitionMgr; + + // To manage the intermediate files during the compilations. + lto::IPOFileMgr FileMgr; + + // Sometimes we need to return a vector of strings in a "C" way (to work with + // the C-APIs). We encode such C-thinking string vector by concatenating all + // strings tegother with a single '\0' as the delimitor, the last string ended + // by double '\0's. + SmallVector<char, 4> ConcatStrings; + + // Make sure command line is parsed only once. It would otherwise complain + // and quite prematurely. + bool OptionsParsed; }; #endif // LTO_CODE_GENERATOR_H diff --git a/tools/lto/LTOPartition.cpp b/tools/lto/LTOPartition.cpp new file mode 100644 index 0000000000..a056b718f1 --- /dev/null +++ b/tools/lto/LTOPartition.cpp @@ -0,0 +1,205 @@ +//===-- LTOPartition.cpp - Parition Merged Module --------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "LTOPartition.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/Analysis/CallGraph.h" +#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Support/Path.h" +#include "llvm/Transforms/Utils/ValueMapper.h" +#include "llvm/Transforms/Utils/Cloning.h" + +using namespace llvm; +using namespace lto; + +// ///////////////////////////////////////////////////////////////////////////// +// +// Implementation of IPOPartition and IPOPartMgr +// +// ///////////////////////////////////////////////////////////////////////////// +// +IPOPartition::IPOPartition(Module *M, const char *NameWoExt, IPOFileMgr &FM) : + Mod(0), Ctx(0), IRFile(0), ObjFile(0), FileNameWoExt(NameWoExt), FileMgr(FM) { +} + +IPOFile &IPOPartition::getIRFile() const { + if (IRFile) + return *IRFile; + else { + std::string FN(FileNameWoExt + ".bc"); + return *(IRFile = FileMgr.createIRFile(FN.c_str())); + } +} + +IPOFile &IPOPartition::getObjFile() const { + if (ObjFile) + return *ObjFile; + else { + std::string FN(FileNameWoExt + ".o"); + return *(ObjFile = FileMgr.createObjFile(FN.c_str())); + } +} + +bool IPOPartition::saveBitCode() { + if (!Mod) { + // The bit-code have already saved in disk. + return true; + } + + IPOFile &F = getIRFile(); + if (F.errOccur()) + return false; + + raw_fd_ostream OF(F.getPath().c_str(), F.getLastErrStr(), + sys::fs::F_Binary); + WriteBitcodeToFile(Mod, OF); + OF.close(); + + Mod = 0; + delete Ctx; + Ctx = 0; + + return !F.errOccur(); +} + +bool IPOPartition::loadBitCode() { + if (Mod) + return true; + + IPOFile &F = getIRFile(); + if (F.errOccur()) + return false; + + Ctx = new LLVMContext; + + error_code &EC = F.getLastErrCode(); + std::string &ErrMsg = F.getLastErrStr(); + + OwningPtr<MemoryBuffer> Buf; + if (error_code ec = MemoryBuffer::getFile(F.getPath(), Buf, -1, false)) { + EC = ec; + ErrMsg += ec.message(); + return false; + } + + Mod = ParseBitcodeFile(Buf.get(), *Ctx, &ErrMsg); + + return Mod != 0; +} + +IPOPartition *IPOPartMgr::createIPOPart(Module *M) { + std::string PartName; + raw_string_ostream OS(PartName); + OS << "part" << NextPartId++; + + IPOPartition *P = new IPOPartition(M, OS.str().c_str(), FileMgr); + P->Mod = M; + IPOParts.push_back(P); + return P; +} + +// /////////////////////////////////////////////////////////////////////////// +// +// Implementation of IPOFile and IPOFileMgr +// +// /////////////////////////////////////////////////////////////////////////// +// +IPOFile::IPOFile(const char *DirName, const char *BaseName, bool KeepFile) + : Fname(BaseName), Keep(KeepFile) { + // Concatenate dirname and basename + StringRef D(DirName); + SmallVector<char, 64> Path(D.begin(), D.end()); + sys::path::append(Path, Twine(BaseName)); + Fpath = StringRef(Path.data(), Path.size()); +} + +IPOFileMgr::IPOFileMgr() { + IRFiles.reserve(20); + ObjFiles.reserve(20); + OtherFiles.reserve(8); + KeepWorkDir = false; + WorkDirCreated = false; +} + +bool IPOFileMgr::createWorkDir(std::string &ErrorInfo) { + if (WorkDirCreated) + return true; + + error_code EC; + if (WorkDir.empty()) { + // If the workdir is not specified, then create workdir under current + // directory. + // + SmallString<128> D; + if (sys::fs::current_path(D) != error_code::success()) { + ErrorInfo += "fail to get current directory"; + return false; + } + sys::path::append(D, "llvmipo"); + sys::fs::make_absolute(D); + + SmallVector<char, 64> ResPath; + EC = sys::fs::createUniqueDirectory(Twine(StringRef(D.data(), D.size())), + ResPath); + WorkDir = StringRef(ResPath.data(), ResPath.size()); + } else { + bool Exist; + EC = sys::fs::create_directory(Twine(WorkDir), Exist); + } + + if (EC == error_code::success()) { + WorkDirCreated = true; + return true; + } + + return false; +} + +IPOFile *IPOFileMgr::createIRFile(const char *Name) { + IPOFile *F = CreateFile(Name); + IRFiles.push_back(F); + return F; +} + +IPOFile *IPOFileMgr::createObjFile(const char *Name) { + IPOFile *F = CreateFile(Name); + ObjFiles.push_back(F); + return F; +} + +IPOFile *IPOFileMgr::createMakefile(const char *Name) { + IPOFile *F = CreateFile(Name); + OtherFiles.push_back(F); + return F; +} + +void IPOFileMgr::removeAllUnneededFiles() { + FileNameVect ToRm; + getFilesNeedToRemove(ToRm); + + for (SmallVector<const char *, 4>::iterator I = ToRm.begin(), E = ToRm.end(); + I != E; I++) { + const char *FN = *I; + sys::fs::file_status Stat; + if (sys::fs::status(Twine(FN), Stat) != error_code::success()) + continue; + + uint32_t Dummy; + if (sys::fs::is_directory(FN)) + sys::fs::remove_all(Twine(FN), Dummy); + else + sys::fs::remove(Twine(FN)); + } +} diff --git a/tools/lto/LTOPartition.h b/tools/lto/LTOPartition.h new file mode 100644 index 0000000000..e251555abf --- /dev/null +++ b/tools/lto/LTOPartition.h @@ -0,0 +1,187 @@ +//===-------- LTOPartition.h - Partition related classes and functions ---===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file declare the partition related classes and functions. A partition +// is a portion of the merged module. In case partition is disabled, the entire +// merged module is considered as a degenerated partition. +// +// The classes declared in this file are: +// o. IPOPartition : to depicit a partition +// o. IPOFile: It is a "container" collecting miscellaneous information about +// an intermeidate file, including file name, path, last-err-message etc. +// o. IPOPartMgr, IPOFileMgr: as the name suggests, it's the manager of +// IPOPartitions and IPOFiles, respectively. +// +//===----------------------------------------------------------------------===// + +#ifndef LTO_PARTITION_H +#define LTO_PARTITION_H + +#include "llvm/Pass.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/Support/system_error.h" + +using namespace llvm; + +namespace lto { + /// \brief To collect miscellaneous information about an intermdiate file. + /// + /// These informration include file name, path, last error message etc. + /// + class IPOFile { + public: + const std::string &getName() { return Fname; } + const std::string &getPath() { return Fpath; } + + error_code &getLastErrCode() { return LastErr; } + std::string &getLastErrStr() { return LastErrStr; } + + bool errOccur() const { + return LastErr != error_code::success() || !LastErrStr.empty(); + } + + // To keep this file after compilation finish. + void setKeep() { Keep = true; } + bool isKept() const { return Keep; } + + private: + friend class IPOFileMgr; + IPOFile(const char* DirName, const char *BaseName, bool Keep=false); + ~IPOFile(); + + private: + std::string Fname; + std::string Fpath; + error_code LastErr; + std::string LastErrStr; + bool Keep; + }; + + /// \brief To manage IPOFiles, create and remove work-directory. + /// + class IPOFileMgr { + public: + typedef SmallVector<const char *, 4> FileNameVect; + + IPOFileMgr(); + + // NOTE: Do not delete intermeidate in the destructor as we never know + // if these files out-last the class or not. It is safe to let linker's + // clean-up hook to take care these files. + ~IPOFileMgr() {}; + + void setWorkDir(const char* WD) { + assert(!WorkDirCreated /* Too late to change mind */ && + WorkDir.empty() /* don't change back and forth */ && + "Cannot change work dir"); + WorkDir = WD; + } + void setKeepWorkDir(bool Keep) { KeepWorkDir = Keep; } + bool IsToKeepWorkDir() const { return KeepWorkDir; } + const std::string &getWorkDir() { return WorkDir; } + + bool createWorkDir(std::string &ErrorInfo); + + IPOFile *createIRFile(const char *Name); + IPOFile *createObjFile(const char *Name); + IPOFile *createMakefile(const char *Name); + + typedef std::vector<IPOFile *> FileVect; + FileVect &getIRFiles() { return IRFiles; } + FileVect &getObjFiles() { return ObjFiles; } + + // Get all files/dirs that need to removed after the LTO complete. + void getFilesNeedToRemove(FileNameVect &ToRm) { + ToRm.clear(); + if (!IsToKeepWorkDir() && WorkDirCreated) + ToRm.push_back(WorkDir.c_str()); + } + + // Remove all files/dirs returned from getFilesNeedToRemove(). + void removeAllUnneededFiles(); + + private: + IPOFile *CreateFile(const char *Name) { + return new IPOFile(WorkDir.c_str(), Name); + } + + private: + FileVect IRFiles; + FileVect ObjFiles; + FileVect OtherFiles; + std::string WorkDir; + bool KeepWorkDir; + bool WorkDirCreated; + }; + + /// \brief Describe a partition of the merged module. + /// + class IPOPartition { + public: + llvm::Module *getModule() const { return Mod; } + IPOFile &getIRFile() const; + IPOFile &getObjFile() const; + const std::string &getIRFilePath() const { return getIRFile().getPath(); } + const std::string &getObjFilePath() const { return getObjFile().getPath(); } + + // If the bitcode reside in memory or disk + bool isInMemory() const { return Mod != 0; } + + // Load/store bitcode from/to disk file. + bool saveBitCode(); + bool loadBitCode(); + + private: + friend class IPOPartMgr; + IPOPartition(llvm::Module *M, const char *FileNameWoExt, IPOFileMgr &FM); + + // The module associated with this partition + Module *Mod; + LLVMContext *Ctx; + + // The bitcode file and its corresponding object file associated with + // this partition. The names of these two files are different only in + // extension; the "FileNameWoExt" record their (common) name without + // extension. + // + mutable IPOFile *IRFile; + mutable IPOFile *ObjFile; + std::string FileNameWoExt; + + IPOFileMgr &FileMgr; + }; + + /// \brief To manage IPOPartitions + /// + class IPOPartMgr { + public: + IPOPartMgr(IPOFileMgr &IFM) : FileMgr(IFM), NextPartId(1) {} + + typedef std::vector<IPOPartition *> IPOPartsTy; + typedef IPOPartsTy::iterator iterator; + typedef IPOPartsTy::const_iterator const_iterator; + + iterator begin() { return IPOParts.begin(); } + iterator end() { return IPOParts.end(); } + const_iterator begin() const { return IPOParts.begin(); } + const_iterator end() const { return IPOParts.end(); } + + IPOPartition *createIPOPart(Module *); + IPOPartition *getSinglePartition() { + assert(IPOParts.size() == 1 && "Has multiple partition"); + return IPOParts[0]; + } + + private: + IPOPartsTy IPOParts; + IPOFileMgr &FileMgr; + int NextPartId; + }; + +} + +#endif //LTO_PARTITION_H diff --git a/tools/lto/LTOPostIPODriver.cpp b/tools/lto/LTOPostIPODriver.cpp new file mode 100644 index 0000000000..2bb833a5ca --- /dev/null +++ b/tools/lto/LTOPostIPODriver.cpp @@ -0,0 +1,180 @@ +//===---------- LTOPostIPODriver.h - PostIPO Driver -----------------------===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the PostIPODriver class which is the driver for Post-IPO +// compilation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/PassManager.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/FormattedStream.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/ObjCARC.h" +#include "LTOPartition.h" +#include "LTOPostIPODriver.h" + +using namespace llvm; +using namespace lto; + +// ///////////////////////////////////////////////////////////////////////////// +// +// Declare all variants of Post-IPO drivers +// +// ///////////////////////////////////////////////////////////////////////////// +// +namespace { + /// \breif Base class for all driver variants. + /// + class PostIPODrvBaseImpl { + public: + PostIPODrvBaseImpl(TargetMachine *Targ, IPOPartMgr &IPM, IPOFileMgr &IFM, + bool ToMergeObjs): + PartMgr(IPM), FileMgr(IFM), MergedObjFile(0), Target(Targ), + MergeObjs(ToMergeObjs) {} + + virtual ~PostIPODrvBaseImpl() {}; + + IPOPartMgr &getPartitionMgr() { return PartMgr; } + IPOFileMgr &getFileMgr() { return FileMgr; } + + // Implement the PostIPODriver::getSingleObjFile() + virtual IPOFile *getSingleObjFile() const = 0; + + bool IsToMergeObj() const { return MergeObjs; } + + virtual bool Compile(std::string &ErrMsg) = 0; + + protected: + // Populate post-IPO scalar optimization pass manager + bool PopulatePostIPOOptPM(PassManager &PM); + + // Populate post-IPO machine-specific CodeGen pass manager + bool PopulateCodeGenPM(PassManager &PM, formatted_raw_ostream &OutFile, + std::string &Err); + + protected: + IPOPartMgr &PartMgr; + IPOFileMgr &FileMgr; + IPOFile *MergedObjFile; + TargetMachine *Target; + bool MergeObjs; + }; + + /// \breif PostIPO driver for the compiling the entire program without + /// partition. + class PostIPODrvSerial : public PostIPODrvBaseImpl { + public: + PostIPODrvSerial(TargetMachine *T, IPOPartMgr &IPM, IPOFileMgr &IFM, + bool ToMergeObjs) : + PostIPODrvBaseImpl(T, IPM, IFM, ToMergeObjs) {} + + virtual bool Compile(std::string &ErrMsg); + virtual IPOFile *getSingleObjFile() const; + + private: + Module *getModule() const { return (*PartMgr.begin())->getModule(); } + }; +} + +// //////////////////////////////////////////////////////////////////////////// +// +// Implemetation of PostIPODriver +// +// //////////////////////////////////////////////////////////////////////////// +// +PostIPODriver::PostIPODriver(VariantTy V, TargetMachine *TM, IPOPartMgr &IPM, + IPOFileMgr &IFM, bool ToMergeObjs) { + if (V == PIDV_SERIAL) + DrvImpl = new PostIPODrvSerial(TM, IPM, IFM, ToMergeObjs); + else + assert(false && "TBD"); +} + +bool PostIPODriver::Compile(std::string &ErrMsg) { + PostIPODrvBaseImpl *P = static_cast<PostIPODrvBaseImpl *>(DrvImpl); + return P->Compile(ErrMsg); +} + +IPOFile *PostIPODriver::getSingleObjFile() const { + PostIPODrvBaseImpl *P = static_cast<PostIPODrvBaseImpl *>(DrvImpl); + return P->getSingleObjFile(); +} + +// //////////////////////////////////////////////////////////////////////////// +// +// Implemetation of PostIPODrvBaseImpl +// +// //////////////////////////////////////////////////////////////////////////// +// +bool PostIPODrvBaseImpl::PopulatePostIPOOptPM(PassManager &PM) { + (void)PM; + return true; +} + +bool PostIPODrvBaseImpl::PopulateCodeGenPM(PassManager &PM, + formatted_raw_ostream &OutFile, + std::string &Err) { + PM.add(new DataLayout(*Target->getDataLayout())); + Target->addAnalysisPasses(PM); + + // If the bitcode files contain ARC code and were compiled with optimization, + // the ObjCARCContractPass must be run, so do it unconditionally here. + PM.add(createObjCARCContractPass()); + + if (Target->addPassesToEmitFile(PM, OutFile, + TargetMachine::CGFT_ObjectFile)) { + Err = "target file type not supported"; + return false; + } + return true; +} + +// //////////////////////////////////////////////////////////////////////////// +// +// Implemetation of PostIPODrvSerial +// +// //////////////////////////////////////////////////////////////////////////// +// +bool PostIPODrvSerial::Compile(std::string &ErrMsg) { + Module *M = getModule(); + + // Step 1: Run the post-IPO scalar optimizations + { + PassManager SoptPM; + PopulatePostIPOOptPM(SoptPM); + SoptPM.run(*M); + } + + // Step 2: Run the post-IPO machine-specific code-generation passes + { + IPOFile &Obj = (*PartMgr.begin())->getObjFile(); + raw_fd_ostream ros(Obj.getPath().c_str(), Obj.getLastErrStr(), + sys::fs::F_Binary); + formatted_raw_ostream OutFile(ros); + + PassManager CodGenPM; + if (!PopulateCodeGenPM(CodGenPM, OutFile, ErrMsg)) { + ErrMsg += Obj.getLastErrStr(); + return false; + } + + CodGenPM.run(*M); + } + + return true; +} + +IPOFile *PostIPODrvSerial::getSingleObjFile() const { + assert(!MergedObjFile && "No need to *merge* a single object file"); + IPOPartition *P = *PartMgr.begin(); + return &P->getObjFile(); +} diff --git a/tools/lto/LTOPostIPODriver.h b/tools/lto/LTOPostIPODriver.h new file mode 100644 index 0000000000..c1813b9989 --- /dev/null +++ b/tools/lto/LTOPostIPODriver.h @@ -0,0 +1,52 @@ +//===---------- LTOPostIPODriver.h - PostIPO Driver -----------------------===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file declare the PostIPODriver class which is the driver for +// Post-IPO compilation phase. +// +//===----------------------------------------------------------------------===// + +#ifndef LTO_POSTIPO_DRIVER_H +#define LTO_POSTIPO_DRIVER_H + +#include "llvm/Target/TargetMachine.h" + +namespace lto { + class IPOPartMgr; + class IPOFileMgr; + class IPOFile; + + class PostIPODriver { + public: + typedef enum { + PIDV_Invalid, + PIDV_SERIAL, // No partition + PIDV_MultiThread, // Each partition is compiled by a thread + PIDV_MultiProc, // Each partition is compiled by a process + PIDV_MakeUtil // Partitions compilation is driven by a make-utility + } VariantTy; + + PostIPODriver(VariantTy Var, TargetMachine *TM, IPOPartMgr &IPM, + IPOFileMgr &IFM, bool ToMergeObjs = false); + + // Return the single resulting object file. If there is no prior + // compilation failure, this function may return NULL iff: + // 1) Partition is enabled, and + // 2) Multiple partitions are generated, and + // 3) It is not asked to merge together the objects corresponding to the + // the partions. + IPOFile *getSingleObjFile() const; + + bool Compile(std::string &ErrMsg); + + private: + void *DrvImpl; + VariantTy DrvStyle; + }; +} + +#endif // LTO_POSTIPO_DRIVER_H diff --git a/tools/lto/lto.cpp b/tools/lto/lto.cpp index db7147c2bc..c7009d69e1 100644 --- a/tools/lto/lto.cpp +++ b/tools/lto/lto.cpp @@ -207,6 +207,19 @@ bool lto_codegen_compile_to_file(lto_code_gen_t cg, const char **name) { return !cg->compile_to_file(name, sLastErrorString); } +/// Get intermediate files that need to be removed before linker exit. Upon +/// return, the paths of the files need to be removed is written to "paths". The +/// paths are separated by a single '\0', and the last path is ended by double +/// '\0's. A file could be a directory; in this case, the entire directory needs +/// to be removed recursively. +/// +/// It is only necessary to call this function after \p lto_codegen_compile was +/// successfully called. +void +lto_codegen_get_files_need_remove(lto_code_gen_t cg, const char **paths) { + *paths = cg->getFilesNeedToRemove(); +} + /// lto_codegen_debug_options - Used to pass extra options to the code /// generator. void lto_codegen_debug_options(lto_code_gen_t cg, const char *opt) { diff --git a/tools/lto/lto.exports b/tools/lto/lto.exports index 46d0d74c82..41a81dd91f 100644 --- a/tools/lto/lto.exports +++ b/tools/lto/lto.exports @@ -20,6 +20,7 @@ lto_codegen_add_must_preserve_symbol lto_codegen_compile lto_codegen_create lto_codegen_dispose +lto_codegen_get_files_need_remove lto_codegen_set_debug_model lto_codegen_set_pic_model lto_codegen_write_merged_modules |