feat(libstore/s3-binary-cache-store): implement createMultipartUpload()

doc/manual/package.nix

@@ -35,6 +35,7 @@ mkMesonDerivation (finalAttrs: {
         ../../.version
         # For example JSON
         ../../src/libutil-tests/data/hash
+        ../../src/libstore-tests/data/content-address
         ../../src/libstore-tests/data/derived-path
         # Too many different types of files to filter for now
         ../../doc/manual

doc/manual/source/SUMMARY.md.in

@@ -118,6 +118,7 @@
 - [Formats and Protocols](protocols/index.md)
   - [JSON Formats](protocols/json/index.md)
     - [Hash](protocols/json/hash.md)
+    - [Content Address](protocols/json/content-address.md)
     - [Store Object Info](protocols/json/store-object-info.md)
     - [Derivation](protocols/json/derivation.md)
     - [Deriving Path](protocols/json/deriving-path.md)

doc/manual/source/protocols/json/content-address.md

@@ -0,0 +1,21 @@
+{{#include content-address-v1-fixed.md}}
+
+## Examples
+
+### [Text](@docroot@/store/store-object/content-address.html#method-text) method
+
+```json
+{{#include schema/content-address-v1/text.json}}
+```
+
+### [Nix Archive](@docroot@/store/store-object/content-address.html#method-nix-archive) method
+
+```json
+{{#include schema/content-address-v1/nar.json}}
+```
+
+<!-- need to convert YAML to JSON first
+## Raw Schema
+
+[JSON Schema for Hash v1](schema/content-address-v1.json)
+-->

doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed

@@ -12,3 +12,6 @@ s/\\`/`/g
 # As we have more such relative links, more replacements of this nature
 # should appear below.
 s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g
+s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g
+s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g
+s^\(./content-address-v1.yaml\)^[JSON format for `ContentAddress`](./content-address.html)^g

doc/manual/source/protocols/json/meson.build

@@ -10,6 +10,7 @@ json_schema_config = files('json-schema-for-humans-config.yaml')
 
 schemas = [
   'hash-v1',
+  'content-address-v1',
   'derivation-v3',
   'deriving-path-v1',
 ]

doc/manual/source/protocols/json/schema/content-address-v1

@@ -0,0 +1 @@
+../../../../../../src/libstore-tests/data/content-address

doc/manual/source/protocols/json/schema/content-address-v1.yaml

@@ -0,0 +1,55 @@
+"$schema": "http://json-schema.org/draft-04/schema"
+"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/content-address-v1.json"
+title: Content Address
+description: |
+  This schema describes the JSON representation of Nix's `ContentAddress` type, which conveys information about [content-addressing store objects](@docroot@/store/store-object/content-address.md).
+
+  > **Note**
+  >
+  > For current methods of content addressing, this data type is a bit suspicious, because it is neither simply a content address of a file system object (the `method` is richer), nor simply a content address of a store object (the `hash` doesn't account for the references).
+  > It should thus only be used in contexts where the references are also known / otherwise made tamper-resistant.
+
+  <!--
+  TODO currently `ContentAddress` is used in both of these, and so same rationale applies, but actually in both cases the JSON is currently ad-hoc.
+  That will be fixed, and as each is fixed, the example (along with a more precise link to the field in question) should be become part of the above note, so what is is saying is more clear.
+
+  > For example:
+
+  > - Fixed outputs of derivations are not allowed to have any references, so an empty reference set is statically known by assumption.
+
+  > - [Store object info](./store-object-info.md) includes the set of references along side the (optional) content address.
+
+  > This data type is thus safely used in both of these contexts.
+
+  -->
+
+type: object
+properties:
+  method:
+    "$ref": "#/$defs/method"
+  hash:
+    title: Content Address
+    description: |
+      This would be the content-address itself.
+
+      For all current methods, this is just a content address of the file system object of the store object, [as described in the store chapter](@docroot@/store/file-system-object/content-address.md), and not of the store object as a whole.
+      In particular, the references of the store object are *not* taken into account with this hash (and currently-supported methods).
+    "$ref": "./hash-v1.yaml"
+required:
+- method
+- hash
+additionalProperties: false
+"$defs":
+  method:
+    type: string
+    enum: [flat, nar, text, git]
+    title: Content-Addressing Method
+    description: |
+      A string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen.
+
+      Valid method strings are:
+
+      - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) (provided the contents are a single file)
+      - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
+      - [`text`](@docroot@/store/store-object/content-address.md#method-text)
+      - [`git`](@docroot@/store/store-object/content-address.md#method-git)

doc/manual/source/protocols/json/schema/derivation-v3.yaml

@@ -1,5 +1,5 @@
-"$schema": http://json-schema.org/draft-04/schema#
-"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json
+"$schema": "http://json-schema.org/draft-04/schema"
+"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json"
 title: Derivation
 description: |
   Experimental JSON representation of a Nix derivation (version 3).
@@ -154,19 +154,10 @@ properties:
           The output path, if known in advance.
 
       method:
-        type: string
-        title: Content addressing method
-        enum: [flat, nar, text, git]
+        "$ref": "./content-address-v1.yaml#/$defs/method"
         description: |
           For an output which will be [content addressed](@docroot@/store/derivation/outputs/content-address.md), a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen.
-
-          Valid method strings are:
-
-          - [`flat`](@docroot@/store/store-object/content-address.md#method-flat)
-          - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
-          - [`text`](@docroot@/store/store-object/content-address.md#method-text)
-          - [`git`](@docroot@/store/store-object/content-address.md#method-git)
-
+          See the linked original definition for further details.
       hashAlgo:
         title: Hash algorithm
         "$ref": "./hash-v1.yaml#/$defs/algorithm"

doc/manual/source/protocols/json/schema/deriving-path-v1.yaml

@@ -1,5 +1,5 @@
-"$schema": http://json-schema.org/draft-04/schema#
-"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/deriving-path-v1.json
+"$schema": "http://json-schema.org/draft-04/schema"
+"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/deriving-path-v1.json"
 title: Deriving Path
 description: |
   This schema describes the JSON representation of Nix's [Deriving Path](@docroot@/store/derivation/index.md#deriving-path).

doc/manual/source/protocols/json/schema/hash-v1.yaml

@@ -1,5 +1,5 @@
-"$schema": http://json-schema.org/draft-04/schema#
-"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/hash-v1.json
+"$schema": "http://json-schema.org/draft-04/schema"
+"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/hash-v1.json"
 title: Hash
 description: |
   A cryptographic hash value used throughout Nix for content addressing and integrity verification.

src/json-schema-checks/content-address

@@ -0,0 +1 @@
+../../src/libstore-tests/data/content-address

src/json-schema-checks/meson.build

@@ -30,6 +30,14 @@ schemas = [
       'blake3-base64.json',
     ],
   },
+  {
+    'stem' : 'content-address',
+    'schema' : schema_dir / 'content-address-v1.yaml',
+    'files' : [
+      'text.json',
+      'nar.json',
+    ],
+  },
   {
     'stem' : 'derivation',
     'schema' : schema_dir / 'derivation-v3.yaml',
@@ -73,8 +81,6 @@ foreach schema : schemas
       stem + '-schema-valid',
       jv,
       args : [
-        '--map',
-        './hash-v1.yaml=' + schema_dir / 'hash-v1.yaml',
         'http://json-schema.org/draft-04/schema',
         schema_file,
       ],

src/json-schema-checks/package.nix

@@ -21,6 +21,7 @@ mkMesonDerivation (finalAttrs: {
     ../../.version
     ../../doc/manual/source/protocols/json/schema
     ../../src/libutil-tests/data/hash
+    ../../src/libstore-tests/data/content-address
     ../../src/libstore-tests/data/derivation
     ../../src/libstore-tests/data/derived-path
     ./.

src/libexpr/include/nix/expr/nixexpr.hh

@@ -442,8 +442,14 @@ struct ExprAttrs : Expr
 
 struct ExprList : Expr
 {
-    std::vector<Expr *> elems;
-    ExprList() {};
+    std::span<Expr *> elems;
+
+    ExprList(std::pmr::polymorphic_allocator<char> & alloc, std::vector<Expr *> exprs)
+        : elems({alloc.allocate_object<Expr *>(exprs.size()), exprs.size()})
+    {
+        std::ranges::copy(exprs, elems.begin());
+    };
+
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env) override;
 

src/libexpr/parser.y

@@ -129,7 +129,7 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
 %type <nix::Expr *> start expr expr_function expr_if expr_op
 %type <nix::Expr *> expr_select expr_simple expr_app
 %type <nix::Expr *> expr_pipe_from expr_pipe_into
-%type <nix::ExprList *> expr_list
+%type <std::vector<Expr *>> list
 %type <nix::ExprAttrs *> binds binds1
 %type <nix::Formals *> formals formal_set
 %type <nix::Formal> formal
@@ -334,7 +334,7 @@ expr_simple
     { $2->pos = CUR_POS; $$ = $2; }
   | '{' '}'
     { $$ = new ExprAttrs(CUR_POS); }
-  | '[' expr_list ']' { $$ = $2; }
+  | '[' list ']' { $$ = new ExprList(state->alloc, std::move($2)); }
   ;
 
 string_parts
@@ -484,9 +484,9 @@ string_attr
   | DOLLAR_CURLY expr '}' { $$ = $2; }
   ;
 
-expr_list
-  : expr_list expr_select { $$ = $1; $1->elems.push_back($2); /* !!! dangerous */; $2->warnIfCursedOr(state->symbols, state->positions); }
-  | { $$ = new ExprList; }
+list
+  : list expr_select { $$ = std::move($1); $$.push_back($2); /* !!! dangerous */; $2->warnIfCursedOr(state->symbols, state->positions); }
+  | { }
   ;
 
 formal_set

src/libstore-tests/content-address.cc

@@ -1,40 +1,101 @@
 #include <gtest/gtest.h>
 
 #include "nix/store/content-address.hh"
+#include "nix/util/tests/json-characterization.hh"
 
 namespace nix {
 
 /* ----------------------------------------------------------------------------
  * ContentAddressMethod::parse, ContentAddressMethod::render
  * --------------------------------------------------------------------------*/
 
-TEST(ContentAddressMethod, testRoundTripPrintParse_1)
+static auto methods = ::testing::Values(
+    std::pair{ContentAddressMethod::Raw::Text, "text"},
+    std::pair{ContentAddressMethod::Raw::Flat, "flat"},
+    std::pair{ContentAddressMethod::Raw::NixArchive, "nar"},
+    std::pair{ContentAddressMethod::Raw::Git, "git"});
+
+struct ContentAddressMethodTest : ::testing::Test,
+                                  ::testing::WithParamInterface<std::pair<ContentAddressMethod, std::string_view>>
+{};
+
+TEST_P(ContentAddressMethodTest, testRoundTripPrintParse_1)
 {
-    for (ContentAddressMethod cam : {
-             ContentAddressMethod::Raw::Text,
-             ContentAddressMethod::Raw::Flat,
-             ContentAddressMethod::Raw::NixArchive,
-             ContentAddressMethod::Raw::Git,
-         }) {
-        EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam);
-    }
+    auto & [cam, _] = GetParam();
+    EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam);
 }
 
-TEST(ContentAddressMethod, testRoundTripPrintParse_2)
+TEST_P(ContentAddressMethodTest, testRoundTripPrintParse_2)
 {
-    for (const std::string_view camS : {
-             "text",
-             "flat",
-             "nar",
-             "git",
-         }) {
-        EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS);
-    }
+    auto & [cam, camS] = GetParam();
+    EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS);
 }
 
+INSTANTIATE_TEST_SUITE_P(ContentAddressMethod, ContentAddressMethodTest, methods);
+
 TEST(ContentAddressMethod, testParseContentAddressMethodOptException)
 {
     EXPECT_THROW(ContentAddressMethod::parse("narwhal"), UsageError);
 }
 
+/* ----------------------------------------------------------------------------
+ * JSON
+ * --------------------------------------------------------------------------*/
+
+class ContentAddressTest : public virtual CharacterizationTest
+{
+    std::filesystem::path unitTestData = getUnitTestData() / "content-address";
+
+public:
+
+    /**
+     * We set these in tests rather than the regular globals so we don't have
+     * to worry about race conditions if the tests run concurrently.
+     */
+    ExperimentalFeatureSettings mockXpSettings;
+
+    std::filesystem::path goldenMaster(std::string_view testStem) const override
+    {
+        return unitTestData / testStem;
+    }
+};
+
+using nlohmann::json;
+
+struct ContentAddressJsonTest : ContentAddressTest,
+                                JsonCharacterizationTest<ContentAddress>,
+                                ::testing::WithParamInterface<std::pair<std::string_view, ContentAddress>>
+{};
+
+TEST_P(ContentAddressJsonTest, from_json)
+{
+    auto & [name, expected] = GetParam();
+    readJsonTest(name, expected);
+}
+
+TEST_P(ContentAddressJsonTest, to_json)
+{
+    auto & [name, value] = GetParam();
+    writeJsonTest(name, value);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    ContentAddressJSON,
+    ContentAddressJsonTest,
+    ::testing::Values(
+        std::pair{
+            "text",
+            ContentAddress{
+                .method = ContentAddressMethod::Raw::Text,
+                .hash = hashString(HashAlgorithm::SHA256, "asdf"),
+            },
+        },
+        std::pair{
+            "nar",
+            ContentAddress{
+                .method = ContentAddressMethod::Raw::NixArchive,
+                .hash = hashString(HashAlgorithm::SHA256, "qwer"),
+            },
+        }));
+
 } // namespace nix

src/libstore-tests/data/content-address/nar.json

@@ -0,0 +1,8 @@
+{
+  "hash": {
+    "algorithm": "sha256",
+    "format": "base64",
+    "hash": "9vLqj0XYoFfJVmoz+ZR02i5camYE1zYSFlDicwxvsKM="
+  },
+  "method": "nar"
+}

src/libstore-tests/data/content-address/text.json

@@ -0,0 +1,8 @@
+{
+  "hash": {
+    "algorithm": "sha256",
+    "format": "base64",
+    "hash": "8OTC92xYkW7CWPJGhRvqCR0U1CR6L8PhhpRGGxgW4Ts="
+  },
+  "method": "text"
+}