diff --git a/index-import.cpp b/index-import.cpp index 053a55c..d492496 100644 --- a/index-import.cpp +++ b/index-import.cpp @@ -24,7 +24,7 @@ using namespace clang; using namespace clang::index; using namespace clang::index::writer; -static cl::list PathRemaps("remap", cl::OneOrMore, +static cl::list PathRemaps("remap", cl::desc("Path remapping substitution"), cl::value_desc("regex=replacement")); static cl::alias PathRemapsAlias("r", cl::aliasopt(PathRemaps)); @@ -32,6 +32,8 @@ static cl::alias PathRemapsAlias("r", cl::aliasopt(PathRemaps)); static cl::list InputIndexPaths(cl::Positional, cl::OneOrMore, cl::desc("")); +static cl::list RemapFilePaths("import-output-file", + cl::desc("import-output-file=")); static cl::opt OutputIndexPath(cl::Positional, cl::Required, cl::desc("")); @@ -175,9 +177,39 @@ static bool isUnitUpToDate(StringRef unitsPath, StringRef outputFile, return *isUpToDateOpt; } +// Append the path of a record inside of an index +void appendInteriorRecordPath(StringRef RecordName, + SmallVectorImpl &PathBuf) { + // To avoid putting a huge number of files into the records directory, it creates + // subdirectories based on the last 2 characters from the hash. + // Note: the actual record name is a function of the bits in the record + StringRef hash2chars = RecordName.substr(RecordName.size()-2); + sys::path::append(PathBuf, hash2chars); + sys::path::append(PathBuf, RecordName); +} + +static bool cloneRecord(StringRef from, StringRef to) { + // Two record files of the same name are guaranteed to have the same + // contents, because the filename contains a hash of its contents. If the + // destination record file already exists, no action needs to be taken. + if (fs::exists(to)) { + return true; + } + + // With swift-5.1, fs::copy_file supports cloning. Until then, use copyfile. + int failed = + copyfile(from.str().c_str(), to.str().c_str(), nullptr, COPYFILE_CLONE); + + // In parallel mode we might be racing against other threads trying to create + // the same record. To handle this, just silently drop file exists errors. + return (failed == 0 || errno == EEXIST); +} + + // Returns None if the Unit file is already up to date static Optional -remapUnit(StringRef outputUnitsPath, StringRef inputUnitPath, +importUnit(StringRef outputUnitsPath, StringRef inputUnitPath, + StringRef outputRecordsPath, StringRef inputRecordsPath, const std::unique_ptr &reader, const Remapper &remapper, FileManager &fileMgr, ModuleNameScope &moduleNames) { @@ -185,6 +217,9 @@ remapUnit(StringRef outputUnitsPath, StringRef inputUnitPath, auto workingDir = remapper.remap(reader->getWorkingDirectory()); auto outputFile = remapper.remap(reader->getOutputFile()); + // Cloning records when we've got an output records path + const auto cloneDepRecords = !outputRecordsPath.empty(); + if (Incremental) { // Check if the unit file is already up to date SmallString<256> remappedOutputFilePath; @@ -214,6 +249,11 @@ remapUnit(StringRef outputUnitsPath, StringRef inputUnitPath, sysrootPath, moduleNames.getModuleInfo); reader->foreachDependency([&](const IndexUnitReader::DependencyInfo &info) { + SmallString<128> inputRecordPath; + SmallString<128> outputRecordPath; + SmallString<128> outputRecordInterDir; + std::error_code createRecordDirFailed; + const auto name = info.UnitOrRecordName; const auto moduleNameRef = moduleNames.getReference(info.ModuleName); const auto isSystem = info.IsSystem; @@ -237,6 +277,21 @@ remapUnit(StringRef outputUnitsPath, StringRef inputUnitPath, break; } case IndexUnitReader::DependencyKind::Record: + if (cloneDepRecords) { + sys::path::append(outputRecordPath, outputRecordsPath); + appendInteriorRecordPath(info.UnitOrRecordName, outputRecordPath); + + // Compute/create the new interior directory by dropping the file name + outputRecordInterDir = outputRecordPath; + sys::path::remove_filename(outputRecordInterDir); + createRecordDirFailed = fs::create_directory(outputRecordInterDir); + if (createRecordDirFailed && createRecordDirFailed != std::errc::file_exists) { + errs() << "error: failed create output record dir" << outputRecordInterDir << "\n"; + } + sys::path::append(inputRecordPath, inputRecordsPath); + appendInteriorRecordPath(info.UnitOrRecordName, inputRecordPath); + cloneRecord(StringRef(inputRecordPath), StringRef(outputRecordPath)); + } writer.addRecordFile(name, file, isSystem, moduleNameRef); break; case IndexUnitReader::DependencyKind::File: @@ -259,23 +314,6 @@ remapUnit(StringRef outputUnitsPath, StringRef inputUnitPath, return writer; } -static bool cloneRecord(StringRef from, StringRef to) { - // Two record files of the same name are guaranteed to have the same - // contents, because the filename contains a hash of its contents. If the - // destination record file already exists, no action needs to be taken. - if (fs::exists(to)) { - return true; - } - - // With swift-5.1, fs::copy_file supports cloning. Until then, use copyfile. - int failed = - copyfile(from.str().c_str(), to.str().c_str(), nullptr, COPYFILE_CLONE); - - // In parallel mode we might be racing against other threads trying to create - // the same record. To handle this, just silently drop file exists errors. - return (failed == 0 || errno == EEXIST); -} - static bool cloneRecords(StringRef recordsDirectory, const std::string &inputIndexPath, const std::string &outputIndexPath) { @@ -341,45 +379,32 @@ static bool remapIndex(const Remapper &remapper, path::append(recordsDirectory, InputIndexPath, "v5", "records"); SmallString<256> outputUnitDirectory; path::append(outputUnitDirectory, OutputIndexPath, "v5", "units"); + SmallString<256> outputRecordsDirectory; + path::append(outputRecordsDirectory, OutputIndexPath, "v5", "records"); if (not fs::is_directory(unitDirectory) || not fs::is_directory(recordsDirectory)) { errs() << "error: invalid index store directory " << InputIndexPath << "\n"; return false; } - bool success = true; - - if (not cloneRecords(recordsDirectory, InputIndexPath, outputIndexPath)) { - success = false; - } + bool success = true; FileSystemOptions fsOpts; FileManager fileMgr{fsOpts}; - std::error_code dirError; - fs::directory_iterator dir{unitDirectory, dirError}; - fs::directory_iterator end; - while (dir != end && !dirError) { - const auto unitPath = dir->path(); - dir.increment(dirError); - - if (unitPath.empty()) { - // The directory iterator returns a single empty path, ignore it. - continue; - } - + auto handleUnitPath = [&](StringRef unitPath, StringRef outputRecordsPath_) { std::string unitReadError; auto reader = IndexUnitReader::createWithFilePath(unitPath, unitReadError); if (not reader) { errs() << "error: failed to read unit file " << unitPath << "\n" << unitReadError; success = false; - continue; + return; } ModuleNameScope moduleNames; - auto writer = remapUnit(outputUnitDirectory, unitPath, reader, remapper, - fileMgr, moduleNames); + auto writer = importUnit(outputUnitDirectory, unitPath, outputRecordsPath_, + recordsDirectory, reader, remapper, fileMgr, moduleNames); if (writer.hasValue()) { std::string unitWriteError; @@ -389,6 +414,32 @@ static bool remapIndex(const Remapper &remapper, success = false; } } + }; + + // Map over the file paths that the user provided + if (RemapFilePaths.size()) { + for (auto & path : RemapFilePaths) { + SmallString<256> outPath; + getUnitPathForOutputFile(unitDirectory, normalizePath(path), outPath, fileMgr); + handleUnitPath(outPath.c_str(), outputRecordsDirectory); + } + return success; + } + + // This batch clones records in the entire index. If we're importing individual + // ouput files we don't want this. + if (not cloneRecords(recordsDirectory, InputIndexPath, outputIndexPath)) { + success = false; + } + + // Process and map the entire index directory + std::error_code dirError; + fs::directory_iterator dir{unitDirectory, dirError}; + fs::directory_iterator end; + while (dir != end && !dirError) { + const auto unitPath = dir->path(); + dir.increment(dirError); + handleUnitPath(unitPath, ""); } if (dirError) {