merge queue: embarking 2.29-maintenance (8a3d3b2), #13580, #13599, #13594, #13586 and [#13602 + #13592 + #13596 + #13637 + #13641] together

doc/manual/source/language/operators.md

@@ -196,7 +196,7 @@ All comparison operators are implemented in terms of `<`, and the following equi
 
 ## Logical implication
 
-Equivalent to `!`*b1* `||` *b2*.
+Equivalent to `!`*b1* `||` *b2*  (or `if` *b1* `then` *b2* `else true`)
 
 [Logical implication]: #logical-implication
 

doc/manual/source/store/store-object.md

@@ -18,7 +18,7 @@ In particular, the edge corresponding to a reference is from the store object th
 References other than a self-reference must not form a cycle.
 The graph of references excluding self-references thus forms a [directed acyclic graph].
 
-[directed acyclic graph]: @docroot@/glossary.md#gloss-directed acyclic graph
+[directed acyclic graph]: @docroot@/glossary.md#gloss-directed-acyclic-graph
 
 We can take the [transitive closure] of the references graph, which any pair of store objects have an edge not if there is a single reference from the first to the second, but a path of one or more references from the first to the second.
 The *requisites* of a store object are all store objects reachable by paths of references which start with given store object's references.

nix-meson-build-support/export/meson.build

@@ -11,12 +11,18 @@ endforeach
 requires_public += deps_public
 
 extra_pkg_config_variables = get_variable('extra_pkg_config_variables', {})
+
+extra_cflags = []
+if not meson.project_name().endswith('-c')
+    extra_cflags += ['-std=c++2a']
+endif
+
 import('pkgconfig').generate(
   this_library,
   filebase : meson.project_name(),
   name : 'Nix',
   description : 'Nix Package Manager',
-  extra_cflags : ['-std=c++2a'],
+  extra_cflags : extra_cflags,
   requires : requires_public,
   requires_private : requires_private,
   libraries_private : libraries_private,

packaging/components.nix

@@ -166,6 +166,17 @@ let
     outputs = prevAttrs.outputs or [ "out" ] ++ [ "dev" ];
   };
 
+  fixupStaticLayer = finalAttrs: prevAttrs: {
+    postFixup =
+      prevAttrs.postFixup or ""
+      + lib.optionalString (stdenv.hostPlatform.isStatic) ''
+        # HACK: Otherwise the result will have the entire buildInputs closure
+        # injected by the pkgsStatic stdenv
+        # <https://github.com/NixOS/nixpkgs/issues/83667>
+        rm -f $out/nix-support/propagated-build-inputs
+      '';
+  };
+
   # Work around weird `--as-needed` linker behavior with BSD, see
   # https://github.com/mesonbuild/meson/issues/3593
   bsdNoLinkAsNeeded =
@@ -301,6 +312,7 @@ in
     scope.sourceLayer
     setVersionLayer
     mesonLayer
+    fixupStaticLayer
     scope.mesonComponentOverrides
   ];
   mkMesonExecutable = mkPackageBuilder [
@@ -310,6 +322,7 @@ in
     setVersionLayer
     mesonLayer
     mesonBuildLayer
+    fixupStaticLayer
     scope.mesonComponentOverrides
   ];
   mkMesonLibrary = mkPackageBuilder [
@@ -320,6 +333,7 @@ in
     setVersionLayer
     mesonBuildLayer
     mesonLibraryLayer
+    fixupStaticLayer
     scope.mesonComponentOverrides
   ];
 

scripts/nix-profile-daemon.fish.in

@@ -23,7 +23,33 @@ end
 
 # Set up the per-user profile.
 
-set --local NIX_LINK $HOME/.nix-profile
+set --local NIX_LINK "$HOME/.nix-profile"
+set --local NIX_LINK_NEW
+if test -n "$XDG_STATE_HOME"
+    set NIX_LINK_NEW "$XDG_STATE_HOME/nix/profile"
+else
+    set NIX_LINK_NEW "$HOME/.local/state/nix/profile"
+end
+if test -e "$NIX_LINK_NEW"
+    if test -t 2; and test -e "$NIX_LINK"
+        set --local warning "\033[1;35mwarning:\033[0m "
+        printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2
+
+        if test (realpath "$NIX_LINK") = (realpath "$NIX_LINK_NEW")
+            printf "         Since the profiles match, you can safely delete either of them.\n" 1>&2
+        else
+            # This should be an exceptionally rare occasion: the only way to get it would be to
+            # 1. Update to newer Nix;
+            # 2. Remove .nix-profile;
+            # 3. Set the $NIX_LINK_NEW to something other than the default user profile;
+            # 4. Roll back to older Nix.
+            # If someone did all that, they can probably figure out how to migrate the profile.
+            printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2
+        end
+    end
+
+    set NIX_LINK "$NIX_LINK_NEW"
+end
 
 # Set up environment.
 # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix

scripts/nix-profile.fish.in

@@ -23,7 +23,38 @@ end
 
 # Set up the per-user profile.
 
-set --local NIX_LINK $HOME/.nix-profile
+set --local NIX_LINK
+if test -n "$NIX_STATE_HOME"
+    set NIX_LINK "$NIX_STATE_HOME/.nix-profile"
+else
+    set NIX_LINK "$HOME/.nix-profile"
+    set --local NIX_LINK_NEW
+    if test -n "$XDG_STATE_HOME"
+        set NIX_LINK_NEW "$XDG_STATE_HOME/nix/profile"
+    else
+        set NIX_LINK_NEW "$HOME/.local/state/nix/profile"
+    end
+    if test -e "$NIX_LINK_NEW"
+        if test -t 2; and test -e "$NIX_LINK"
+            set --local warning "\033[1;35mwarning:\033[0m "
+            printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2
+
+            if test (realpath "$NIX_LINK") = (realpath "$NIX_LINK_NEW")
+                printf "         Since the profiles match, you can safely delete either of them.\n" 1>&2
+            else
+                # This should be an exceptionally rare occasion: the only way to get it would be to
+                # 1. Update to newer Nix;
+                # 2. Remove .nix-profile;
+                # 3. Set the $NIX_LINK_NEW to something other than the default user profile;
+                # 4. Roll back to older Nix.
+                # If someone did all that, they can probably figure out how to migrate the profile.
+                printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2
+            end
+        end
+
+        set NIX_LINK "$NIX_LINK_NEW"
+    end
+end
 
 # Set up environment.
 # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix

src/libstore/local-store.cc

@@ -130,7 +130,7 @@ LocalStore::LocalStore(ref<const Config> config)
     Path gcRootsDir = config->stateDir + "/gcroots";
     if (!pathExists(gcRootsDir)) {
         createDirs(gcRootsDir);
-        createSymlink(profilesDir, gcRootsDir + "/profiles");
+        replaceSymlink(profilesDir, gcRootsDir + "/profiles");
     }
 
     for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) {

src/libutil/include/nix/util/lru-cache.hh

@@ -33,6 +33,18 @@ private:
     Data data;
     LRU lru;
 
+    /**
+     * Move this item to the back of the LRU list.
+     */
+    void promote(LRU::iterator it)
+    {
+        /* Think of std::list iterators as stable pointers to the list node,
+         * which never get invalidated. Thus, we can reuse the same lru list
+         * element and just splice it to the back of the list without the need
+         * to update its value in the key -> list iterator map. */
+        lru.splice(/*pos=*/lru.end(), /*other=*/lru, it);
+    }
+
 public:
 
     LRUCache(size_t capacity)
@@ -83,28 +95,40 @@ public:
     /**
      * Look up an item in the cache. If it exists, it becomes the most
      * recently used item.
-     * */
+     *
+     * @returns corresponding cache entry, std::nullopt if it's not in the cache
+     */
     template<typename K>
     std::optional<Value> get(const K & key)
     {
         auto i = data.find(key);
         if (i == data.end())
             return {};
 
-        /**
-         * Move this item to the back of the LRU list.
-         *
-         * Think of std::list iterators as stable pointers to the list node,
-         * which never get invalidated. Thus, we can reuse the same lru list
-         * element and just splice it to the back of the list without the need
-         * to update its value in the key -> list iterator map.
-         */
         auto & [it, value] = i->second;
-        lru.splice(/*pos=*/lru.end(), /*other=*/lru, it.it);
-
+        promote(it.it);
         return value;
     }
 
+    /**
+     * Look up an item in the cache. If it exists, it becomes the most
+     * recently used item.
+     *
+     * @returns mutable pointer to the corresponding cache entry, nullptr if
+     * it's not in the cache
+     */
+    template<typename K>
+    Value * getOrNullptr(const K & key)
+    {
+        auto i = data.find(key);
+        if (i == data.end())
+            return nullptr;
+
+        auto & [it, value] = i->second;
+        promote(it.it);
+        return &value;
+    }
+
     size_t size() const noexcept
     {
         return data.size();

src/libutil/include/nix/util/pos-table.hh

@@ -4,6 +4,7 @@
 #include <cstdint>
 #include <vector>
 
+#include "nix/util/lru-cache.hh"
 #include "nix/util/pos-idx.hh"
 #include "nix/util/position.hh"
 #include "nix/util/sync.hh"
@@ -37,10 +38,20 @@ public:
     };
 
 private:
+    /**
+     * Vector of byte offsets (in the virtual input buffer) of initial line character's position.
+     * Sorted by construction. Binary search over it allows for efficient translation of arbitrary
+     * byte offsets in the virtual input buffer to its line + column position.
+     */
     using Lines = std::vector<uint32_t>;
+    /**
+     * Cache from byte offset in the virtual buffer of Origins -> @ref Lines in that origin.
+     */
+    using LinesCache = LRUCache<uint32_t, Lines>;
 
     std::map<uint32_t, Origin> origins;
-    mutable Sync<std::map<uint32_t, Lines>> lines;
+
+    mutable Sync<LinesCache> linesCache;
 
     const Origin * resolve(PosIdx p) const
     {
@@ -56,6 +67,11 @@ private:
     }
 
 public:
+    PosTable(std::size_t linesCacheCapacity = 65536)
+        : linesCache(linesCacheCapacity)
+    {
+    }
+
     Origin addOrigin(Pos::Origin origin, size_t size)
     {
         uint32_t offset = 0;

src/libutil/pos-table.cc

@@ -15,21 +15,35 @@ Pos PosTable::operator[](PosIdx p) const
     const auto offset = origin->offsetOf(p);
 
     Pos result{0, 0, origin->origin};
-    auto lines = this->lines.lock();
-    auto linesForInput = (*lines)[origin->offset];
-
-    if (linesForInput.empty()) {
-        auto source = result.getSource().value_or("");
-        const char * begin = source.data();
-        for (Pos::LinesIterator it(source), end; it != end; it++)
-            linesForInput.push_back(it->data() - begin);
-        if (linesForInput.empty())
-            linesForInput.push_back(0);
+    auto linesCache = this->linesCache.lock();
+
+    /* Try the origin's line cache */
+    const auto * linesForInput = linesCache->getOrNullptr(origin->offset);
+
+    auto fillCacheForOrigin = [](std::string_view content) {
+        auto contentLines = Lines();
+
+        const char * begin = content.data();
+        for (Pos::LinesIterator it(content), end; it != end; it++)
+            contentLines.push_back(it->data() - begin);
+        if (contentLines.empty())
+            contentLines.push_back(0);
+
+        return contentLines;
+    };
+
+    /* Calculate line offsets and fill the cache */
+    if (!linesForInput) {
+        auto originContent = result.getSource().value_or("");
+        linesCache->upsert(origin->offset, fillCacheForOrigin(originContent));
+        linesForInput = linesCache->getOrNullptr(origin->offset);
     }
-    // as above: the first line starts at byte 0 and is always present
-    auto lineStartOffset = std::prev(std::upper_bound(linesForInput.begin(), linesForInput.end(), offset));
 
-    result.line = 1 + (lineStartOffset - linesForInput.begin());
+    assert(linesForInput);
+
+    // as above: the first line starts at byte 0 and is always present
+    auto lineStartOffset = std::prev(std::upper_bound(linesForInput->begin(), linesForInput->end(), offset));
+    result.line = 1 + (lineStartOffset - linesForInput->begin());
     result.column = 1 + (offset - *lineStartOffset);
     return result;
 }

src/nix/flake.cc

@@ -1019,6 +1019,10 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
 {
     std::string dstUri;
 
+    CheckSigsFlag checkSigs = CheckSigs;
+
+    SubstituteFlag substitute = NoSubstitute;
+
     CmdFlakeArchive()
     {
         addFlag({
@@ -1027,6 +1031,11 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
             .labels = {"store-uri"},
             .handler = {&dstUri},
         });
+        addFlag({
+            .longName = "no-check-sigs",
+            .description = "Do not require that paths are signed by trusted keys.",
+            .handler = {&checkSigs, NoCheckSigs},
+        });
     }
 
     std::string description() override
@@ -1087,7 +1096,8 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
 
         if (!dryRun && !dstUri.empty()) {
             ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
-            copyPaths(*store, *dstStore, sources);
+
+            copyPaths(*store, *dstStore, sources, NoRepair, checkSigs, substitute);
         }
     }
 };

tests/nixos/s3-binary-cache-store.nix

@@ -68,6 +68,7 @@ in
       # Create a binary cache.
       server.wait_for_unit("minio")
       server.wait_for_unit("network-addresses-eth1.service")
+      server.wait_for_open_port(9000)
 
       server.succeed("mc config host add minio http://localhost:9000 ${accessKey} ${secretKey} --api s3v4")
       server.succeed("mc mb minio/my-cache")