diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index d535dca1301..17491055f2d 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -84,11 +84,15 @@ StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) /* Empty version 1 nario, nothing to do. */ break; - case 1: + case 1: { + /* Reuse a string buffer to avoid kernel overhead allocating + memory for large strings. */ + StringSink saved; + /* Non-empty version 1 nario. */ while (true) { /* Extract the NAR from the source. */ - StringSink saved; + saved.s.clear(); TeeSource tee{source, saved}; NullFileSystemObjectSink ether; parseDump(ether, tee); @@ -101,23 +105,26 @@ StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) auto references = CommonProto::Serialise::read(store, CommonProto::ReadConn{.from = source}); auto deriver = readString(source); - auto narHash = hashString(HashAlgorithm::SHA256, saved.s); - - ValidPathInfo info{path, narHash}; - if (deriver != "") - info.deriver = store.parseStorePath(deriver); - info.references = references; - info.narSize = saved.s.size(); // Ignore optional legacy signature. if (readInt(source) == 1) readString(source); - // Can't use underlying source, which would have been exhausted. - auto source2 = StringSource(saved.s); - store.addToStore(info, source2, NoRepair, checkSigs); + if (!store.isValidPath(path)) { + auto narHash = hashString(HashAlgorithm::SHA256, saved.s); - res.push_back(info.path); + ValidPathInfo info{path, narHash}; + if (deriver != "") + info.deriver = store.parseStorePath(deriver); + info.references = references; + info.narSize = saved.s.size(); + + // Can't use underlying source, which would have been exhausted. + auto source2 = StringSource(saved.s); + store.addToStore(info, source2, NoRepair, checkSigs); + } + + res.push_back(path); auto n = readNum(source); if (n == 0) @@ -126,6 +133,7 @@ StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) throw Error("input doesn't look like a nario"); } break; + } case exportMagicV2: while (true) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index cbd3fa6d80d..754f5d4dec0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1049,12 +1049,16 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF bool narRead = false; Finally cleanup = [&]() { if (!narRead) { - NullFileSystemObjectSink sink; - try { - parseDump(sink, source); - } catch (...) { - // TODO: should Interrupted be handled here? - ignoreExceptionInDestructor(); + if (info.narSize) + source.skip(info.narSize); + else { + NullFileSystemObjectSink sink; + try { + parseDump(sink, source); + } catch (...) { + // TODO: should Interrupted be handled here? + ignoreExceptionInDestructor(); + } } } }; diff --git a/src/libutil/include/nix/util/serialise.hh b/src/libutil/include/nix/util/serialise.hh index 8799e128fc4..d6845a494dc 100644 --- a/src/libutil/include/nix/util/serialise.hh +++ b/src/libutil/include/nix/util/serialise.hh @@ -255,6 +255,8 @@ struct StringSource : Source } size_t read(char * data, size_t len) override; + + void skip(size_t len) override; }; /** diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 47a00c8d660..ba153625ee9 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -242,6 +242,16 @@ size_t StringSource::read(char * data, size_t len) return n; } +void StringSource::skip(size_t len) +{ + const size_t remain = s.size() - pos; + if (len > remain) { + pos = s.size(); + throw EndOfFile("end of string reached"); + } + pos += len; +} + std::unique_ptr sourceToSink(std::function fun) { struct SourceToSink : FinishSink