diff --git a/index-import.cpp b/index-import.cpp index 241a2b1..7afc6fd 100644 --- a/index-import.cpp +++ b/index-import.cpp @@ -1,12 +1,15 @@ #include "clang/Basic/FileManager.h" #include "clang/Index/IndexUnitReader.h" #include "clang/Index/IndexUnitWriter.h" +#include "llvm/ADT/APInt.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/Errc.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" #include +#include #include #include #include @@ -27,7 +30,17 @@ static cl::list PathRemaps("remap", cl::OneOrMore, cl::value_desc("regex=replacement")); static cl::alias PathRemapsAlias("r", cl::aliasopt(PathRemaps)); -static cl::list InputIndexPaths(cl::Positional, cl::OneOrMore, +static cl::opt UnitPathsFile( + "unit-paths-file", + cl::desc("Path to a file containing paths to unit files. If specified " + "record-paths-file must also be specified")); + +static cl::opt RecordPathsFile( + "record-paths-file", + cl::desc("Path to a file containing paths to record files. If specified " + "unit-paths-file must also be specified")); + +static cl::list InputIndexPaths(cl::Positional, cl::ZeroOrMore, cl::desc("")); static cl::opt OutputIndexPath(cl::Positional, cl::Required, @@ -102,12 +115,96 @@ static const FileEntry *getFileEntry(FileManager &fileMgr, StringRef path) { return fileMgr.getVirtualFile(path, /*size*/ 0, /*modtime*/ 0); } -static IndexUnitWriter remapUnit(const std::unique_ptr &reader, - const Remapper &remapper, FileManager &fileMgr, - ModuleNameScope &moduleNames) { +void getUnitPathForOutputFile(StringRef unitsPath, StringRef filePath, + SmallVectorImpl &str, + FileManager &fileMgr) { + str.append(unitsPath.begin(), unitsPath.end()); + str.push_back('/'); + SmallString<256> absPath(filePath); + fileMgr.makeAbsolutePath(absPath); + StringRef fname = sys::path::filename(absPath); + str.append(fname.begin(), fname.end()); + str.push_back('-'); + llvm::hash_code pathHashVal = llvm::hash_value(absPath); + llvm::APInt(64, pathHashVal).toString(str, 36, /*Signed=*/false); +} + +Optional +isUnitUpToDateForOutputFile(StringRef unitsPath, StringRef filePath, + Optional timeCompareFilePath, + FileManager &fileMgr, std::string &error) { + SmallString<256> unitPath; + getUnitPathForOutputFile(unitsPath, filePath, unitPath, fileMgr); + + llvm::sys::fs::file_status unitStat; + if (std::error_code ec = llvm::sys::fs::status(unitPath.c_str(), unitStat)) { + if (ec != llvm::errc::no_such_file_or_directory) { + llvm::raw_string_ostream err(error); + err << "could not access path '" << unitPath << "': " << ec.message(); + return None; + } + return false; + } + + if (!timeCompareFilePath.hasValue()) + return true; + + llvm::sys::fs::file_status compareStat; + if (std::error_code ec = + llvm::sys::fs::status(*timeCompareFilePath, compareStat)) { + if (ec != llvm::errc::no_such_file_or_directory) { + llvm::raw_string_ostream err(error); + err << "could not access path '" << *timeCompareFilePath + << "': " << ec.message(); + return None; + } + return true; + } + + // Return true (unit is up-to-date) if the file to compare is older than the + // unit file. + return compareStat.getLastModificationTime() <= + unitStat.getLastModificationTime(); +} + +// Returns true if the Unit file of given output file already exists and is +// older than the input file. +static bool isUnitUpToDate(StringRef unitsPath, StringRef outputFile, + StringRef inputFile, FileManager &fileMgr) { + std::string error; + auto isUpToDateOpt = isUnitUpToDateForOutputFile(unitsPath, outputFile, + inputFile, fileMgr, error); + if (!isUpToDateOpt.hasValue()) { + errs() << "error: failed file status check:\n" << error << "\n"; + return false; + } + + return *isUpToDateOpt; +} + +// Returns None if the Unit file is already up to date +static Optional +remapUnit(StringRef outputUnitsPath, StringRef inputUnitPath, + const std::unique_ptr &reader, + const Remapper &remapper, FileManager &fileMgr, + ModuleNameScope &moduleNames) { // The set of remapped paths. auto workingDir = remapper.remap(reader->getWorkingDirectory()); auto outputFile = remapper.remap(reader->getOutputFile()); + + // Check if the unit file is already up to date + SmallString<256> remappedOutputFilePath; + if (outputFile[0] != '/') { + // Convert outputFile to absolute path + path::append(remappedOutputFilePath, workingDir, outputFile); + } else { + remappedOutputFilePath = outputFile; + } + if (isUnitUpToDate(outputUnitsPath, remappedOutputFilePath, inputUnitPath, + fileMgr)) { + return None; + } + auto mainFilePath = remapper.remap(reader->getMainFilePath()); auto sysrootPath = remapper.remap(reader->getSysrootPath()); @@ -184,6 +281,17 @@ static bool cloneRecord(StringRef from, StringRef to) { return (failed == 0 || errno == EEXIST); } +static bool createDirectories(StringRef path) { + std::error_code failed = + fs::create_directories(path, /*IgnoreExisting=*/true); + if (failed) { + errs() << "Could not create directory `" << path + << "`: " << failed.message() << "\n"; + return false; + } + return true; +} + static bool cloneRecords(StringRef recordsDirectory, const std::string &inputIndexPath, const std::string &outputIndexPath) { @@ -206,11 +314,8 @@ static bool cloneRecords(StringRef recordsDirectory, path::replace_path_prefix(outputPath, inputIndexPath, outputIndexPath); if (status->type() == fs::file_type::directory_file) { - std::error_code failed = fs::create_directory(outputPath); - if (failed && failed != std::errc::file_exists) { + if (not createDirectories(outputPath)) { success = false; - errs() << "Could not create directory `" << outputPath - << "`: " << failed.message() << "\n"; } } else if (status->type() == fs::file_type::regular_file) { if (not cloneRecord(inputPath, outputPath)) { @@ -230,6 +335,39 @@ static bool cloneRecords(StringRef recordsDirectory, return success; } +static bool cloneRecords(const std::vector &recordPaths, + const std::string &outputIndexPath) { + bool success = true; + + for (auto inputPath : recordPaths) { + auto inputIndexPath = path::parent_path( + path::parent_path(path::parent_path(path::parent_path(inputPath)))); + + SmallString<128> outputPath{inputPath}; + path::replace_path_prefix(outputPath, inputIndexPath, outputIndexPath); + + if (not cloneRecord(inputPath, outputPath)) { + bool innerSuccess = true; + if (errno == ENOENT) { + // We need to create the directories below it + if (not createDirectories(path::parent_path(outputPath))) { + innerSuccess = false; + } else if (not cloneRecord(inputPath, outputPath)) { + innerSuccess = false; + } + } + + if (!innerSuccess) { + success = false; + errs() << "Could not copy record file from `" << inputPath << "` to `" + << outputPath << "`: " << strerror(errno) << "\n"; + } + } + } + + return success; +} + // Normalize a path by removing /./ or // from it. static std::string normalizePath(StringRef Path) { SmallString<128> NormalizedPath; @@ -247,6 +385,8 @@ static bool remapIndex(const Remapper &remapper, path::append(unitDirectory, InputIndexPath, "v5", "units"); SmallString<256> recordsDirectory; path::append(recordsDirectory, InputIndexPath, "v5", "records"); + SmallString<256> outputUnitDirectory; + path::append(outputUnitDirectory, OutputIndexPath, "v5", "units"); if (not fs::is_directory(unitDirectory) || not fs::is_directory(recordsDirectory)) { @@ -284,13 +424,16 @@ static bool remapIndex(const Remapper &remapper, } ModuleNameScope moduleNames; - auto writer = remapUnit(reader, remapper, fileMgr, moduleNames); - - std::string unitWriteError; - if (writer.write(unitWriteError)) { - errs() << "error: failed to write index store; " << unitWriteError - << "\n"; - success = false; + auto writer = remapUnit(outputUnitDirectory, unitPath, reader, remapper, + fileMgr, moduleNames); + + if (writer.hasValue()) { + std::string unitWriteError; + if (writer->write(unitWriteError)) { + errs() << "error: failed to write index store; " << unitWriteError + << "\n"; + success = false; + } } } @@ -302,6 +445,79 @@ static bool remapIndex(const Remapper &remapper, return success; } +static bool remapUnits(const Remapper &remapper, + const std::vector &unitPaths, + const SmallString<256> &outputUnitDirectory) { + FileSystemOptions fsOpts; + FileManager fileMgr{fsOpts}; + + bool success = true; + + for (const auto unitPath : unitPaths) { + std::string unitReadError; + auto reader = IndexUnitReader::createWithFilePath(unitPath, unitReadError); + if (not reader) { + errs() << "error: failed to read unit file " << unitPath << "\n" + << unitReadError; + success = false; + continue; + } + + ModuleNameScope moduleNames; + auto writer = remapUnit(outputUnitDirectory, unitPath, reader, remapper, + fileMgr, moduleNames); + + if (writer.hasValue()) { + std::string unitWriteError; + if (writer->write(unitWriteError)) { + errs() << "error: failed to write index store; " << unitWriteError + << "\n"; + success = false; + } + } + } + + return success; +} + +static bool remapIndex(const Remapper &remapper, + const std::vector &unitPaths, + const std::vector &recordPaths, + const std::string &outputIndexPath) { + SmallString<256> outputUnitDirectory; + path::append(outputUnitDirectory, OutputIndexPath, "v5", "units"); + + __block bool success = true; + + if (not cloneRecords(recordPaths, outputIndexPath)) { + success = false; + } + + if (ParallelStride == 0 || ParallelStride >= unitPaths.size()) { + if (not remapUnits(remapper, unitPaths, outputUnitDirectory)) { + success = false; + } + return success; + } + + // Process the units in groups according to the parallel stride. + const size_t stride = static_cast(ParallelStride); + const size_t length = unitPaths.size(); + const size_t numStrides = ((length - 1) / stride) + 1; + + dispatch_apply(numStrides, DISPATCH_APPLY_AUTO, ^(size_t strideIndex) { + const size_t offset = strideIndex * stride; + auto start = unitPaths.begin() + offset; + auto end = unitPaths.begin() + std::min(offset + stride, length); + std::vector unitPathsSlice(start, end); + if (not remapUnits(remapper, unitPathsSlice, outputUnitDirectory)) { + success = false; + } + }); + + return success; +} + int main(int argc, char **argv) { cl::ParseCommandLineOptions(argc, argv); @@ -340,6 +556,41 @@ int main(int argc, char **argv) { return EXIT_FAILURE; } + // If given the paths to individual unit paths and records, use those instead + // of enumerating directories + if (!UnitPathsFile.empty() || !RecordPathsFile.empty()) { + if (UnitPathsFile.empty()) { + errs() << "record-paths-file set but not unit-paths-file.\n"; + return EXIT_FAILURE; + } + if (RecordPathsFile.empty()) { + errs() << "unit-paths-file set but not record-paths-file.\n"; + return EXIT_FAILURE; + } + + std::ifstream unitPathsStream(UnitPathsFile); + std::vector unitPaths; + std::string unitPath; + while (unitPathsStream >> unitPath) { + unitPaths.push_back(unitPath); + } + + std::ifstream recordPathsStream(RecordPathsFile); + std::vector recordPaths; + std::string recordPath; + while (recordPathsStream >> recordPath) { + recordPaths.push_back(recordPath); + } + + return remapIndex(remapper, unitPaths, recordPaths, OutputIndexPath) + ? EXIT_SUCCESS + : EXIT_FAILURE; + } else if (InputIndexPaths.empty()) { + errs() << "input-indexstores must be set if unit-paths-file and " + "record-paths-file are not.\n"; + return EXIT_FAILURE; + } + if (ParallelStride == 0 || ParallelStride >= InputIndexPaths.size()) { bool success = true; for (auto &InputIndexPath : InputIndexPaths) {