diff --git a/doc/manual/source/protocols/json/schema/build-result-v1.yaml b/doc/manual/source/protocols/json/schema/build-result-v1.yaml index 7730f3d9399..55e55d3e5cb 100644 --- a/doc/manual/source/protocols/json/schema/build-result-v1.yaml +++ b/doc/manual/source/protocols/json/schema/build-result-v1.yaml @@ -83,7 +83,7 @@ properties: description: | A mapping from output names to their build trace entries. additionalProperties: - "$ref": "build-trace-entry-v2.yaml#/$defs/value" + "$ref": "build-trace-entry-v2.yaml" failure: type: object diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml index 2e8cd07da2a..4340f82388c 100644 --- a/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml @@ -16,21 +16,24 @@ description: | - Version 1: Original format - - Version 2: - - Use `drvPath` not `drvHash` to refer to derivation in a more conventional way. - - Remove `dependentRealisations` - - Separate into `key` and `value` + - Version 2: Remove `dependentRealisations` type: object required: - - key - - value + - id + - outPath + - signatures +allOf: + - "$ref": "#/$defs/key" + - "$ref": "#/$defs/value" properties: - key: - "$ref": "#/$defs/key" - value: - "$ref": "#/$defs/value" -additionalProperties: false + id: {} + outPath: {} + signatures: {} +additionalProperties: + dependentRealisations: + description: deprecated field + type: object "$defs": key: @@ -40,20 +43,23 @@ additionalProperties: false This is the "key" part, refering to a derivation and output. type: object required: - - drvPath - - outputName + - id properties: - drvPath: - "$ref": "store-path-v1.yaml" - title: Derivation Path - description: | - The store path of the derivation that was built. - outputName: + id: type: string - title: Output Name + title: Derivation Output ID + pattern: "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$" description: | - Name of the specific output (e.g., "out", "dev", "doc") - additionalProperties: false + Unique identifier for the derivation output that was built. + + Format: `{hash-quotient-drv}!{output-name}` + + - **hash-quotient-drv**: SHA-256 [hash of the quotient derivation](@docroot@/store/derivation/outputs/input-address.md#hash-quotient-drv). + Begins with `sha256:`. + + - **output-name**: Name of the specific output (e.g., "out", "dev", "doc") + + Example: `"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo"` value: title: Build Trace Value @@ -71,6 +77,13 @@ additionalProperties: false description: | The path to the store object that resulted from building this derivation for the given output name. + patternProperties: + "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$": + "$ref": "store-path-v1.yaml" + title: Dependent Store Path + description: Store path that this dependency resolved to during the build + additionalProperties: false + signatures: type: array title: Build Signatures diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index a4d94616096..a1b525bc51e 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -65,6 +65,9 @@ schemas = [ 'schema' : schema_dir / 'build-trace-entry-v2.yaml', 'files' : [ 'simple.json', + # The field is no longer supported, but we want to show that we + # ignore it during parsing. + 'with-dependent-realisations.json', 'with-signature.json', ], }, diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index f60e4499c3e..fc7f1849384 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -108,16 +108,20 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const overloaded{ [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, [&](const BuiltPath::Built & p) { + auto drvHashes = staticOutputHashes(store, store.readDerivation(p.drvPath->outPath())); for (auto & [outputName, outputPath] : p.outputs) { if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - DrvOutput key{ - .drvPath = p.drvPath->outPath(), - .outputName = outputName, - }; + auto drvOutput = get(drvHashes, outputName); + if (!drvOutput) + throw Error( + "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", + store.printStorePath(p.drvPath->outPath()), + outputName); + DrvOutput key{*drvOutput, outputName}; auto thisRealisation = store.queryRealisation(key); - // We’ve built it, so we must have the realisation. - assert(thisRealisation); - res.insert(Realisation{*thisRealisation, key}); + assert(thisRealisation); // We’ve built it, so we must + // have the realisation + res.insert(Realisation{*thisRealisation, std::move(key)}); } else { res.insert(outputPath); } diff --git a/src/libstore-tests/build-result.cc b/src/libstore-tests/build-result.cc index a1d8ddee641..b7e8f83f9e7 100644 --- a/src/libstore-tests/build-result.cc +++ b/src/libstore-tests/build-result.cc @@ -75,13 +75,25 @@ INSTANTIATE_TEST_SUITE_P( { "foo", { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + DrvOutput{ + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, }, }, { "bar", { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + DrvOutput{ + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, }, }, }, diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index 71f42517f68..f318c8ec631 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -108,6 +108,69 @@ CHARACTERIZATION_TEST( }, })) +CHARACTERIZATION_TEST( + drvOutput, + "drv-output", + (std::tuple{ + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput{ + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +CHARACTERIZATION_TEST( + realisation, + "realisation", + (std::tuple{ + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, + }, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, + })) + +READ_CHARACTERIZATION_TEST( + realisation_with_deps, + "realisation-with-deps", + (std::tuple{ + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, + }, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, + })) + CHARACTERIZATION_TEST( vector, "vector", diff --git a/src/libstore-tests/data/build-result/success.json b/src/libstore-tests/data/build-result/success.json index ec3479d5564..4baadb54775 100644 --- a/src/libstore-tests/data/build-result/success.json +++ b/src/libstore-tests/data/build-result/success.json @@ -1,10 +1,14 @@ { "builtOutputs": { "bar": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "signatures": [] }, "foo": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [] } diff --git a/src/libstore-tests/data/dummy-store/one-realisation.json b/src/libstore-tests/data/dummy-store/one-realisation.json index 576de083817..b5c8b8c5621 100644 --- a/src/libstore-tests/data/dummy-store/one-realisation.json +++ b/src/libstore-tests/data/dummy-store/one-realisation.json @@ -1,7 +1,8 @@ { "buildTrace": { - "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv": { + "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=": { "out": { + "dependentRealisations": {}, "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [] } diff --git a/src/libstore-tests/data/realisation/simple.json b/src/libstore-tests/data/realisation/simple.json index 1e4760b5682..2ccb1e72119 100644 --- a/src/libstore-tests/data/realisation/simple.json +++ b/src/libstore-tests/data/realisation/simple.json @@ -1,10 +1,6 @@ { - "key": { - "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", - "outputName": "foo" - }, - "value": { - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", - "signatures": [] - } + "dependentRealisations": {}, + "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "signatures": [] } diff --git a/src/libstore-tests/data/realisation/with-dependent-realisations.json b/src/libstore-tests/data/realisation/with-dependent-realisations.json new file mode 100644 index 00000000000..a58e0d7fe1c --- /dev/null +++ b/src/libstore-tests/data/realisation/with-dependent-realisations.json @@ -0,0 +1,8 @@ +{ + "dependentRealisations": { + "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" + }, + "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "signatures": [] +} diff --git a/src/libstore-tests/data/realisation/with-signature.json b/src/libstore-tests/data/realisation/with-signature.json index cca29ef3e45..3270f1cdebf 100644 --- a/src/libstore-tests/data/realisation/with-signature.json +++ b/src/libstore-tests/data/realisation/with-signature.json @@ -1,12 +1,8 @@ { - "key": { - "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", - "outputName": "foo" - }, - "value": { - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", - "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" - ] - } + "dependentRealisations": {}, + "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "signatures": [ + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + ] } diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.8.bin b/src/libstore-tests/data/serve-protocol/build-result-2.8.bin deleted file mode 100644 index fa072599578..00000000000 Binary files a/src/libstore-tests/data/serve-protocol/build-result-2.8.bin and /dev/null differ diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.8.json b/src/libstore-tests/data/serve-protocol/build-result-2.8.json deleted file mode 100644 index 9c97678dada..00000000000 --- a/src/libstore-tests/data/serve-protocol/build-result-2.8.json +++ /dev/null @@ -1,37 +0,0 @@ -[ - { - "errorMsg": "no idea why", - "isNonDeterministic": false, - "startTime": 0, - "status": "OutputRejected", - "stopTime": 0, - "success": false, - "timesBuilt": 0 - }, - { - "errorMsg": "no idea why", - "isNonDeterministic": true, - "startTime": 30, - "status": "NotDeterministic", - "stopTime": 50, - "success": false, - "timesBuilt": 3 - }, - { - "builtOutputs": { - "bar": { - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", - "signatures": [] - }, - "foo": { - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", - "signatures": [] - } - }, - "startTime": 30, - "status": "Built", - "stopTime": 50, - "success": true, - "timesBuilt": 1 - } -] diff --git a/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin b/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin deleted file mode 100644 index 5be0b15a345..00000000000 Binary files a/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin and /dev/null differ diff --git a/src/libstore-tests/data/serve-protocol/drv-output-2.8.json b/src/libstore-tests/data/serve-protocol/drv-output-2.8.json deleted file mode 100644 index 6afbf20e83a..00000000000 --- a/src/libstore-tests/data/serve-protocol/drv-output-2.8.json +++ /dev/null @@ -1,10 +0,0 @@ -[ - { - "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "outputName": "baz" - }, - { - "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "outputName": "quux" - } -] diff --git a/src/libstore-tests/data/serve-protocol/realisation-2.8.bin b/src/libstore-tests/data/serve-protocol/realisation-2.8.bin deleted file mode 100644 index 7d1fd462f99..00000000000 Binary files a/src/libstore-tests/data/serve-protocol/realisation-2.8.bin and /dev/null differ diff --git a/src/libstore-tests/data/serve-protocol/realisation-2.8.json b/src/libstore-tests/data/serve-protocol/realisation-2.8.json deleted file mode 100644 index 1977d8485b9..00000000000 --- a/src/libstore-tests/data/serve-protocol/realisation-2.8.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "key": { - "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "outputName": "baz" - }, - "value": { - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", - "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" - ] - } -} diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin deleted file mode 100644 index fce3a25dd88..00000000000 Binary files a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin and /dev/null differ diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json deleted file mode 100644 index 2c2d264f9a1..00000000000 --- a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", - "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" - ] -} diff --git a/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.bin deleted file mode 100644 index 11bec3e6e3b..00000000000 Binary files a/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.bin and /dev/null differ diff --git a/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.json deleted file mode 100644 index 1d566b13bb1..00000000000 --- a/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.json +++ /dev/null @@ -1,39 +0,0 @@ -[ - { - "errorMsg": "no idea why", - "isNonDeterministic": false, - "startTime": 0, - "status": "OutputRejected", - "stopTime": 0, - "success": false, - "timesBuilt": 0 - }, - { - "errorMsg": "no idea why", - "isNonDeterministic": true, - "startTime": 30, - "status": "NotDeterministic", - "stopTime": 50, - "success": false, - "timesBuilt": 3 - }, - { - "builtOutputs": { - "bar": { - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", - "signatures": [] - }, - "foo": { - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", - "signatures": [] - } - }, - "cpuSystem": 604000000, - "cpuUser": 500000000, - "startTime": 30, - "status": "Built", - "stopTime": 50, - "success": true, - "timesBuilt": 1 - } -] diff --git a/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.bin deleted file mode 100644 index 5be0b15a345..00000000000 Binary files a/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.bin and /dev/null differ diff --git a/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.json deleted file mode 100644 index 6afbf20e83a..00000000000 --- a/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.json +++ /dev/null @@ -1,10 +0,0 @@ -[ - { - "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "outputName": "baz" - }, - { - "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "outputName": "quux" - } -] diff --git a/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.bin deleted file mode 100644 index 7d1fd462f99..00000000000 Binary files a/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.bin and /dev/null differ diff --git a/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json deleted file mode 100644 index 1977d8485b9..00000000000 --- a/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "key": { - "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "outputName": "baz" - }, - "value": { - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", - "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" - ] - } -} diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.bin deleted file mode 100644 index fce3a25dd88..00000000000 Binary files a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.bin and /dev/null differ diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json deleted file mode 100644 index 2c2d264f9a1..00000000000 --- a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", - "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" - ] -} diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json index 3271cb98598..7f0f62f876c 100644 --- a/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json +++ b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json @@ -1,7 +1,8 @@ { "buildTrace": { - "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": { + "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { "out": { + "dependentRealisations": {}, "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out", "signatures": [] } diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json index f10653a3a7f..93f4fb22a62 100644 --- a/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json +++ b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json @@ -1,7 +1,8 @@ { "buildTrace": { - "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": { + "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { "out": { + "dependentRealisations": {}, "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out", "signatures": [] } diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json index c4d3901236c..617984a27eb 100644 --- a/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json +++ b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json @@ -1,7 +1,8 @@ { "buildTrace": { - "11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": { + "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": { "out": { + "dependentRealisations": {}, "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out", "signatures": [] } diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json index ecd22821469..a63d6243fae 100644 --- a/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json +++ b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json @@ -1,13 +1,15 @@ { "buildTrace": { - "11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": { + "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": { "out": { + "dependentRealisations": {}, "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out", "signatures": [] } }, - "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { + "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { "out": { + "dependentRealisations": {}, "outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out", "signatures": [] } diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index bc165ad7f58..4a12dcf78c0 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -37,19 +37,20 @@ TEST(DummyStore, realisation_read) return cfg->openDummyStore(); }(); - StorePath drvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}; + auto drvHash = Hash::parseExplicitFormatUnprefixed( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", HashAlgorithm::SHA256, HashFormat::Base16); auto outputName = "foo"; - EXPECT_EQ(store->queryRealisation({drvPath, outputName}), nullptr); + EXPECT_EQ(store->queryRealisation({drvHash, outputName}), nullptr); UnkeyedRealisation value{ - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, }; - store->buildTrace.insert({drvPath, {{outputName, value}}}); + store->buildTrace.insert({drvHash, {{outputName, value}}}); - auto value2 = store->queryRealisation({drvPath, outputName}); + auto value2 = store->queryRealisation({drvHash, outputName}); ASSERT_TRUE(value2); EXPECT_EQ(*value2, value); @@ -130,7 +131,10 @@ INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] { [&] { auto store = writeCfg->openDummyStore(); store->buildTrace.insert_or_assign( - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, + Hash::parseExplicitFormatUnprefixed( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + HashAlgorithm::SHA256, + HashFormat::Base16), std::map{ { "out", diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index 9c0ebbe8abf..d2d7df80f59 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -46,10 +46,13 @@ TEST_P(RealisationJsonTest, to_json) Realisation simple{ { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, }, { - .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, + .drvHash = Hash::parseExplicitFormatUnprefixed( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + HashAlgorithm::SHA256, + HashFormat::Base16), .outputName = "foo", }, }; @@ -74,4 +77,14 @@ INSTANTIATE_TEST_SUITE_P( }(), })); +/** + * We no longer have a notion of "dependent realisations", but we still + * want to parse old realisation files. So make this just be a read test + * (no write direction), accordingly. + */ +TEST_F(RealisationTest, dependent_realisations_from_json) +{ + readJsonTest("with-dependent-realisations", simple); +} + } // namespace nix diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index a47e56317bd..159abeb6082 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -78,19 +78,16 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - drvOutput_2_8, - "drv-output-2.8", - (ServeProto::Version{ - .major = 2, - .minor = 8, - }), + drvOutput, + "drv-output", + defaultVersion, (std::tuple{ { - .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .outputName = "baz", }, DrvOutput{ - .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), .outputName = "quux", }, })) @@ -99,37 +96,54 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - unkeyedRealisation_2_8, - "unkeyed-realisation-2.8", - (ServeProto::Version{ - .major = 2, - .minor = 8, - }), - (UnkeyedRealisation{ - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, + realisation, + "realisation", + defaultVersion, + (std::tuple{ + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, + }, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, })) -VERSIONED_CHARACTERIZATION_TEST( +VERSIONED_READ_CHARACTERIZATION_TEST( ServeProtoTest, - realisation_2_8, - "realisation-2.8", - (ServeProto::Version{ - .major = 2, - .minor = 8, - }), - (Realisation{ - UnkeyedRealisation{ - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, - }, - { - .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, - .outputName = "baz", + realisation_with_deps, + "realisation-with-deps", + defaultVersion, + (std::tuple{ + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, + }, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, }, })) @@ -195,10 +209,7 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -/* We now do a lossy read which does not allow us to faithfully write - back, since we changed the data type. We still however want to test - that this read works, and so for that we have a one-way test. */ -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, buildResult_2_6, "build-result-2.6", @@ -231,72 +242,27 @@ VERSIONED_READ_CHARACTERIZATION_TEST( { "foo", { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - }, - { - "bar", - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - }, - }, - }}, - .timesBuilt = 1, - .startTime = 30, - .stopTime = 50, -#if 0 - // These fields are not yet serialized. - // FIXME Include in next version of protocol or document - // why they are skipped. - .cpuUser = std::chrono::milliseconds(500s), - .cpuSystem = std::chrono::milliseconds(604s), -#endif - }, - }; - t; - })) - -VERSIONED_CHARACTERIZATION_TEST( - ServeProtoTest, - buildResult_2_8, - "build-result-2.8", - (ServeProto::Version{ - .major = 2, - .minor = 8, - }), - ({ - using namespace std::literals::chrono_literals; - std::tuple t{ - BuildResult{.inner{BuildResult::Failure{{ - .status = BuildResult::Failure::OutputRejected, - .msg = HintFmt("no idea why"), - }}}}, - BuildResult{ - .inner{BuildResult::Failure{{ - .status = BuildResult::Failure::NotDeterministic, - .msg = HintFmt("no idea why"), - .isNonDeterministic = true, - }}}, - .timesBuilt = 3, - .startTime = 30, - .stopTime = 50, - }, - BuildResult{ - .inner{BuildResult::Success{ - .status = BuildResult::Success::Built, - .builtOutputs = - { - { - "foo", - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + DrvOutput{ + .drvHash = + Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, }, }, { "bar", { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + DrvOutput{ + .drvHash = + Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, }, }, }, diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index 2bcdd9afad0..aac5822d01f 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -18,17 +18,17 @@ namespace nix { TEST(WorkerProtoVersionNumber, ordering) { using Number = WorkerProto::Version::Number; - EXPECT_LT((Number{.major = 1, .minor = 10}), (Number{.major = 1, .minor = 20})); - EXPECT_GT((Number{.major = 1, .minor = 30}), (Number{.major = 1, .minor = 20})); - EXPECT_EQ((Number{.major = 1, .minor = 10}), (Number{.major = 1, .minor = 10})); - EXPECT_LT((Number{.major = 0, .minor = 255}), (Number{.major = 1, .minor = 0})); + EXPECT_LT((Number{1, 10}), (Number{1, 20})); + EXPECT_GT((Number{1, 30}), (Number{1, 20})); + EXPECT_EQ((Number{1, 10}), (Number{1, 10})); + EXPECT_LT((Number{0, 255}), (Number{1, 0})); } TEST(WorkerProtoVersion, partialOrderingSameFeatures) { using V = WorkerProto::Version; - V v1{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}}; - V v2{.number = {.major = 1, .minor = 30}, .features = {"a", "b"}}; + V v1{.number = {1, 20}, .features = {"a", "b"}}; + V v2{.number = {1, 30}, .features = {"a", "b"}}; EXPECT_TRUE(v1 < v2); EXPECT_TRUE(v2 > v1); @@ -40,8 +40,8 @@ TEST(WorkerProtoVersion, partialOrderingSameFeatures) TEST(WorkerProtoVersion, partialOrderingSubsetFeatures) { using V = WorkerProto::Version; - V fewer{.number = {.major = 1, .minor = 30}, .features = {"a"}}; - V more{.number = {.major = 1, .minor = 30}, .features = {"a", "b"}}; + V fewer{.number = {1, 30}, .features = {"a"}}; + V more{.number = {1, 30}, .features = {"a", "b"}}; // fewer <= more: JUST the features are a subset EXPECT_TRUE(fewer < more); @@ -54,8 +54,8 @@ TEST(WorkerProtoVersion, partialOrderingUnordered) { using V = WorkerProto::Version; // Same number but incomparable features - V v1{.number = {.major = 1, .minor = 20}, .features = {"a", "c"}}; - V v2{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}}; + V v1{.number = {1, 20}, .features = {"a", "c"}}; + V v2{.number = {1, 20}, .features = {"a", "b"}}; EXPECT_FALSE(v1 < v2); EXPECT_FALSE(v1 > v2); @@ -69,8 +69,8 @@ TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures) { using V = WorkerProto::Version; // Higher number but fewer features — unordered - V v1{.number = {.major = 1, .minor = 30}, .features = {"a"}}; - V v2{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}}; + V v1{.number = {1, 30}, .features = {"a"}}; + V v2{.number = {1, 20}, .features = {"a", "b"}}; EXPECT_FALSE(v1 < v2); EXPECT_FALSE(v1 > v2); @@ -80,8 +80,8 @@ TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures) TEST(WorkerProtoVersion, partialOrderingEmptyFeatures) { using V = WorkerProto::Version; - V empty{.number = {.major = 1, .minor = 20}, .features = {}}; - V some{.number = {.major = 1, .minor = 30}, .features = {"a"}}; + V empty{.number = {1, 20}, .features = {}}; + V some{.number = {1, 30}, .features = {"a"}}; // empty features is a subset of everything EXPECT_TRUE(empty < some); @@ -223,67 +223,69 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, drvOutput, - "drv-output-realisation-with-path-not-hash", - (WorkerProto::Version{ - .number = - { - .major = 1, - .minor = 38, - }, - .features = {"realisation-with-path-not-hash"}, - }), + "drv-output", + defaultVersion, (std::tuple{ { - .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .outputName = "baz", }, DrvOutput{ - .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), .outputName = "quux", }, })) VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - unkeyedRealisation_realisation_with_path, - "unkeyed-realisation-realisation-with-path-not-hash", - (WorkerProto::Version{ - .number = + realisation, + "realisation", + defaultVersion, + (std::tuple{ + Realisation{ { - .major = 1, - .minor = 38, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, - .features = {"realisation-with-path-not-hash"}, - }), - (UnkeyedRealisation{ - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, + }, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, })) -VERSIONED_CHARACTERIZATION_TEST( +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, - realisation_realisation_with_path, - "realisation-realisation-with-path-not-hash", - (WorkerProto::Version{ - .number = + realisation_with_deps, + "realisation-with-deps", + defaultVersion, + (std::tuple{ + Realisation{ { - .major = 1, - .minor = 38, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, + }, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", }, - .features = {"realisation-with-path-not-hash"}, - }), - (Realisation{ - UnkeyedRealisation{ - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, - }, - { - .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, - .outputName = "baz", }, })) @@ -316,10 +318,7 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -/* We now do a lossy read which does not allow us to faithfully write - back, since we changed the data type. We still however want to test - that this read works, and so for that we have a one-way test. */ -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_28, "build-result-1.28", @@ -348,13 +347,25 @@ VERSIONED_READ_CHARACTERIZATION_TEST( { "foo", { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + DrvOutput{ + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, }, }, { "bar", { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + DrvOutput{ + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, }, }, }, @@ -363,8 +374,7 @@ VERSIONED_READ_CHARACTERIZATION_TEST( t; })) -// See above note -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_29, "build-result-1.29", @@ -400,13 +410,27 @@ VERSIONED_READ_CHARACTERIZATION_TEST( { "foo", { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + DrvOutput{ + .drvHash = + Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, }, }, { "bar", { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + DrvOutput{ + .drvHash = + Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, }, }, }, @@ -419,8 +443,7 @@ VERSIONED_READ_CHARACTERIZATION_TEST( t; })) -// See above note -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_37, "build-result-1.37", @@ -456,71 +479,27 @@ VERSIONED_READ_CHARACTERIZATION_TEST( { "foo", { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - }, - { - "bar", - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - }, - }, - }}, - .timesBuilt = 1, - .startTime = 30, - .stopTime = 50, - .cpuUser = std::chrono::microseconds(500s), - .cpuSystem = std::chrono::microseconds(604s), - }, - }; - t; - })) - -VERSIONED_CHARACTERIZATION_TEST( - WorkerProtoTest, - buildResult_realisation_with_path, - "build-result-realisation-with-path-not-hash", - (WorkerProto::Version{ - .number = - { - .major = 1, - .minor = 38, - }, - .features = {"realisation-with-path-not-hash"}, - }), - ({ - using namespace std::literals::chrono_literals; - std::tuple t{ - BuildResult{.inner{BuildResult::Failure{{ - .status = BuildResult::Failure::OutputRejected, - .msg = HintFmt("no idea why"), - }}}}, - BuildResult{ - .inner{BuildResult::Failure{{ - .status = BuildResult::Failure::NotDeterministic, - .msg = HintFmt("no idea why"), - .isNonDeterministic = true, - }}}, - .timesBuilt = 3, - .startTime = 30, - .stopTime = 50, - }, - BuildResult{ - .inner{BuildResult::Success{ - .status = BuildResult::Success::Built, - .builtOutputs = - { - { - "foo", - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + DrvOutput{ + .drvHash = + Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, }, }, { "bar", { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + DrvOutput{ + .drvHash = + Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, }, }, }, @@ -941,11 +920,7 @@ TEST_F(WorkerProtoTest, handshake_features) out, in, WorkerProto::Version{ - .number = - { - .major = 1, - .minor = 123, - }, + .number = {.major = 1, .minor = 123}, .features = {"bar", "aap", "mies", "xyzzy"}, }); }); @@ -956,11 +931,7 @@ TEST_F(WorkerProtoTest, handshake_features) out, in, WorkerProto::Version{ - .number = - { - .major = 1, - .minor = 200, - }, + .number = {.major = 1, .minor = 200}, .features = {"foo", "bar", "xyzzy"}, }); diff --git a/src/libstore-tests/worker-substitution.cc b/src/libstore-tests/worker-substitution.cc index 9cc7b56dcb8..3a049b7e721 100644 --- a/src/libstore-tests/worker-substitution.cc +++ b/src/libstore-tests/worker-substitution.cc @@ -198,6 +198,12 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) // Snapshot the destination store before checkpointJson("ca-drv/store-before", dummyStore); + // Compute the hash modulo of the derivation + // For CA floating derivations, the kind is Deferred since outputs aren't known until build + auto hashModulo = hashDerivationModulo(*dummyStore, drv, true); + ASSERT_EQ(hashModulo.kind, DrvHash::Kind::Deferred); + auto drvHash = hashModulo.hashes.at("out"); + // Create the output store object auto outputPath = substituter->addToStore( "test-ca-drv-out", @@ -216,7 +222,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) // Add the realisation (build trace) to the substituter substituter->buildTrace.insert_or_assign( - drvPath, + drvHash, std::map{ { "out", @@ -230,7 +236,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) checkpointJson("ca-drv/substituter", substituter); // The realisation should not exist in the destination store yet - DrvOutput drvOutput{drvPath, "out"}; + DrvOutput drvOutput{drvHash, "out"}; ASSERT_FALSE(dummyStore->queryRealisation(drvOutput)); // Create a worker with our custom substituter @@ -293,6 +299,11 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Write the dependency derivation to the destination store auto depDrvPath = dummyStore->writeDerivation(depDrv); + // Compute the hash modulo for the dependency derivation + auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true); + ASSERT_EQ(depHashModulo.kind, DrvHash::Kind::Deferred); + auto depDrvHash = depHashModulo.hashes.at("out"); + // Create the output store object for the dependency in the substituter auto depOutputPath = substituter->addToStore( "dep-drv-out", @@ -311,7 +322,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Add the realisation for the dependency to the substituter substituter->buildTrace.insert_or_assign( - depDrvPath, + depDrvHash, std::map{ { "out", @@ -342,6 +353,11 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Snapshot the destination store before checkpointJson("issue-11928/store-before", dummyStore); + // Compute the hash modulo for the root derivation + auto rootHashModulo = hashDerivationModulo(*dummyStore, rootDrv, true); + ASSERT_EQ(rootHashModulo.kind, DrvHash::Kind::Deferred); + auto rootDrvHash = rootHashModulo.hashes.at("out"); + // Create the output store object for the root derivation // Note: it does NOT reference the dependency's output auto rootOutputPath = substituter->addToStore( @@ -362,12 +378,12 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) HashAlgorithm::SHA256); // The DrvOutputs for both derivations - DrvOutput depDrvOutput{depDrvPath, "out"}; - DrvOutput rootDrvOutput{rootDrvPath, "out"}; + DrvOutput depDrvOutput{depDrvHash, "out"}; + DrvOutput rootDrvOutput{rootDrvHash, "out"}; // Add the realisation for the root derivation to the substituter substituter->buildTrace.insert_or_assign( - rootDrvPath, + rootDrvHash, std::map{ { "out", diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 48ffd33e475..61480ec1c78 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -514,7 +514,7 @@ StorePath BinaryCacheStore::addToStore( std::string BinaryCacheStore::makeRealisationPath(const DrvOutput & id) { - return realisationsPrefix + "/" + id.drvPath.to_string() + "/" + id.outputName + ".doi"; + return realisationsPrefix + "/" + id.to_string() + ".doi"; } void BinaryCacheStore::queryRealisationUncached( @@ -535,10 +535,7 @@ void BinaryCacheStore::queryRealisationUncached( realisation = std::make_shared(nlohmann::json::parse(*data)); } catch (Error & e) { e.addTrace( - {}, - "while parsing file '%s' as a build trace value for key '%s'", - outputInfoFilePath, - id.to_string()); + {}, "while parsing file '%s' as a realisation for key '%s'", outputInfoFilePath, id.to_string()); throw; } return (*callbackPtr)(std::move(realisation)); @@ -554,10 +551,7 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info) { if (diskCache) diskCache->upsertRealisation(config.getReference().render(/*FIXME withParams=*/false), info); - upsertFile( - makeRealisationPath(info.id), - static_cast(static_cast(info)).dump(), - "application/json"); + upsertFile(makeRealisationPath(info.id), static_cast(info).dump(), "application/json"); } ref BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath) diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 51667557077..da1b6bfd589 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -301,8 +301,9 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths) given this information by the downstream goal, that cannot happen anymore if the downstream goal only cares about one output, but we care about all outputs. */ - for (auto & [outputName, _] : drv->outputs) { - InitialOutput v; + auto outputHashes = staticOutputHashes(worker.evalStore, *drv); + for (auto & [outputName, outputHash] : outputHashes) { + InitialOutput v{.outputHash = outputHash}; /* TODO we might want to also allow randomizing the paths for regular CA derivations, e.g. for sake of checking @@ -1267,7 +1268,7 @@ DerivationBuildingGoal::checkPathValidity(std::map & : PathStatus::Corrupt, }; } - auto drvOutput = DrvOutput{drvPath, i.first}; + auto drvOutput = DrvOutput{info.outputHash, i.first}; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 4a726e29503..4f99928d785 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -37,6 +37,12 @@ DerivationGoal::DerivationGoal( , drvPath(drvPath) , wantedOutput(wantedOutput) , drv{std::make_unique(drv)} + , outputHash{[&] { + auto outputHashes = staticOutputHashes(worker.evalStore, drv); + if (auto * mOutputHash = get(outputHashes, wantedOutput)) + return *mOutputHash; + throw Error("derivation '%s' does not have output '%s'", worker.store.printStorePath(drvPath), wantedOutput); + }()} , buildMode(buildMode) { @@ -96,7 +102,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) them. */ if (worker.settings.useSubstitutes && drvOptions.substitutesAllowed(worker.settings)) { if (!checkResult) { - DrvOutput id{drvPath, wantedOutput}; + DrvOutput id{outputHash, wantedOutput}; auto g = worker.makeDrvOutputSubstitutionGoal(id); waitees.insert(g); co_await await(std::move(waitees)); @@ -180,7 +186,12 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) // No `std::visit` for coroutines yet if (auto * successP = resolvedResult.tryGetSuccess()) { auto & success = *successP; - if (!drv->outputs.contains(wantedOutput)) + auto outputHashes = staticOutputHashes(worker.evalStore, *drv); + auto resolvedHashes = staticOutputHashes(worker.store, drvResolved); + + auto outputHash = get(outputHashes, wantedOutput); + auto resolvedHash = get(resolvedHashes, wantedOutput); + if ((!outputHash) || (!resolvedHash)) throw Error( "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)", worker.store.printStorePath(drvPath), @@ -189,7 +200,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) auto realisation = [&] { auto take1 = get(success.builtOutputs, wantedOutput); if (take1) - return *take1; + return static_cast(*take1); /* The above `get` should work. But stateful tracking of outputs in resolvedResult, this can get out of sync with the @@ -197,7 +208,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) check the store directly if it fails. */ auto take2 = worker.evalStore.queryRealisation( DrvOutput{ - .drvPath = pathResolved, + .drvHash = *resolvedHash, .outputName = wantedOutput, }); if (take2) @@ -213,7 +224,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) Realisation newRealisation{ realisation, { - .drvPath = drvPath, + .drvHash = *outputHash, .outputName = wantedOutput, }}; newRealisation.signatures.clear(); @@ -259,7 +270,16 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) /* In checking mode, the builder will not register any outputs. So we want to make sure the ones that we wanted to check are properly there. */ - success.builtOutputs = {{wantedOutput, assertPathValidity()}}; + success.builtOutputs = {{ + wantedOutput, + { + assertPathValidity(), + { + .drvHash = outputHash, + .outputName = wantedOutput, + }, + }, + }}; } else { /* Otherwise the builder will give us info for out output, but also for other outputs. Filter down to just our output so as @@ -278,7 +298,16 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) if (success.builtOutputs.count(wantedOutput) == 0) { debug( "BUG! wanted output '%s' not in builtOutputs, working around by adding it manually", wantedOutput); - success.builtOutputs = {{wantedOutput, assertPathValidity()}}; + success.builtOutputs = {{ + wantedOutput, + { + assertPathValidity(), + { + .drvHash = outputHash, + .outputName = wantedOutput, + }, + }, + }}; } } } @@ -375,7 +404,7 @@ std::optional> DerivationGoal::checkPa if (drv->type().isImpure()) return std::nullopt; - auto drvOutput = DrvOutput{drvPath, wantedOutput}; + auto drvOutput = DrvOutput{outputHash, wantedOutput}; std::optional mRealisation; @@ -415,7 +444,7 @@ std::optional> DerivationGoal::checkPa Realisation{ *mRealisation, { - .drvPath = drvPath, + .drvHash = outputHash, .outputName = wantedOutput, }, }); @@ -446,7 +475,16 @@ Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, Unke return Goal::doneSuccess( BuildResult::Success{ .status = status, - .builtOutputs = {{wantedOutput, std::move(builtOutput)}}, + .builtOutputs = {{ + wantedOutput, + { + std::move(builtOutput), + DrvOutput{ + .drvHash = outputHash, + .outputName = wantedOutput, + }, + }, + }}, }); } diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 5a5fe06d5e8..71f0ae1e7db 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -10,7 +10,7 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput & id, Worke : Goal(worker, init()) , id(id) { - name = fmt("substitution of '%s'", id.render(worker.store)); + name = fmt("substitution of '%s'", id.to_string()); trace("created"); } @@ -93,8 +93,7 @@ Goal::Co DrvOutputSubstitutionGoal::init() /* None left. Terminate this goal and let someone else deal with it. */ - debug( - "derivation output '%s' is required, but there is no substituter that can provide it", id.render(worker.store)); + debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); if (substituterFailed) { worker.failedSubstitutions++; @@ -109,7 +108,7 @@ Goal::Co DrvOutputSubstitutionGoal::init() std::string DrvOutputSubstitutionGoal::key() { - return "a$" + std::string(id.render(worker.store)); + return "a$" + std::string(id.to_string()); } } // namespace nix diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index 5daeef8c853..c5e4e389799 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -2,16 +2,7 @@ -- Won't be loaded unless the experimental feature `ca-derivations` -- is enabled --- Why the `*V` tables --- --- We are trying to keep different versions of the experiment to have --- completely independent extra schemas from one another. This will --- enable people to switch between versions of the experiment (including --- newer to older) without migrating between them, but at the cost --- of having many abandoned tables lying around. Closer to the end of --- the experiment, we'll provide guidance on how to clean this up. - -create table if not exists BuildTraceV2 ( +create table if not exists Realisations ( id integer primary key autoincrement not null, drvPath text not null, outputName text not null, -- symbolic output id, usually "out" @@ -20,4 +11,31 @@ create table if not exists BuildTraceV2 ( foreign key (outputPath) references ValidPaths(id) on delete cascade ); -create index if not exists IndexBuildTraceV2 on BuildTraceV2(drvPath, outputName); +create index if not exists IndexRealisations on Realisations(drvPath, outputName); + +-- We can end-up in a weird edge-case where a path depends on itself because +-- it’s an output of a CA derivation, that happens to be the same as one of its +-- dependencies. +-- In that case we have a dependency loop (path -> realisation1 -> realisation2 +-- -> path) that we need to break by removing the dependencies between the +-- realisations +create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths + begin + delete from RealisationsRefs where realisationReference in ( + select id from Realisations where outputPath = old.id + ); + end; + +create table if not exists RealisationsRefs ( + referrer integer not null, + realisationReference integer, + foreign key (referrer) references Realisations(id) on delete cascade, + foreign key (realisationReference) references Realisations(id) on delete restrict +); +-- used by deletion trigger +create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); + +-- used by QueryRealisationReferences +create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer); +-- used by cascade deletion when ValidPaths is deleted +create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath); diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index fc88ee88905..a0afe948d64 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -47,6 +47,34 @@ void CommonProto::Serialise::write( conn.to << renderContentAddress(ca); } +Realisation CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) +{ + std::string rawInput = readString(conn.from); + try { + return nlohmann::json::parse(rawInput); + } catch (Error & e) { + e.addTrace({}, "while parsing a realisation object in the remote protocol"); + throw; + } +} + +void CommonProto::Serialise::write( + const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation) +{ + conn.to << static_cast(realisation).dump(); +} + +DrvOutput CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) +{ + return DrvOutput::parse(readString(conn.from)); +} + +void CommonProto::Serialise::write( + const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) +{ + conn.to << drvOutput.to_string(); +} + std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 24f7a1a34b1..155fe2432bb 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -955,8 +955,14 @@ static void performOp( case WorkerProto::Op::RegisterDrvOutput: { logger->startWork(); - auto realisation = WorkerProto::Serialise::read(*store, rconn); - store->registerDrvOutput(realisation); + if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) { + auto outputId = WorkerProto::Serialise::read(*store, rconn); + auto outputPath = StorePath(readString(conn.from)); + store->registerDrvOutput(Realisation{{.outPath = outputPath}, outputId}); + } else { + auto realisation = WorkerProto::Serialise::read(*store, rconn); + store->registerDrvOutput(realisation); + } logger->stopWork(); break; } @@ -964,13 +970,19 @@ static void performOp( case WorkerProto::Op::QueryRealisation: { logger->startWork(); auto outputId = WorkerProto::Serialise::read(*store, rconn); - std::optional info = *store->queryRealisation(outputId); + auto info = store->queryRealisation(outputId); logger->stopWork(); - /* Only return the new format because if we got past - `DrvOutput` serialization, we know that is what we're using. - */ - assert(conn.protoVersion.features.contains(WorkerProto::featureRealisationWithPath)); - WorkerProto::write(*store, wconn, info); + if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) { + std::set outPaths; + if (info) + outPaths.insert(info->outPath); + WorkerProto::write(*store, wconn, outPaths); + } else { + std::set realisations; + if (info) + realisations.insert({*info, outputId}); + WorkerProto::write(*store, wconn, realisations); + } break; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 63cbd2d36af..3fc4191f2aa 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -861,14 +861,13 @@ DrvHashes drvHashes; /* Look up the derivation by value and memoize the `hashDerivationModulo` call. */ -static DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath) +static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath) { - std::optional hash; + std::optional hash; if (drvHashes.cvisit(drvPath, [&hash](const auto & kv) { hash.emplace(kv.second); })) { return *hash; } auto h = hashDerivationModulo(store, store.readInvalidDerivation(drvPath), false); - // Cache it drvHashes.insert_or_assign(drvPath, h); return h; @@ -891,10 +890,12 @@ static DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPa don't leak the provenance of fixed outputs, reducing pointless cache misses as the build itself won't know this. */ -DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) +DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { + auto type = drv.type(); + /* Return a fixed hash for fixed-output derivations. */ - if (drv.type().isFixed()) { + if (type.isFixed()) { std::map outputHashes; for (const auto & i : drv.outputs) { auto & dof = std::get(i.second.raw); @@ -904,66 +905,54 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } - return outputHashes; + return DrvHash{ + .hashes = outputHashes, + .kind = DrvHash::Kind::Regular, + }; } - if (std::visit( - overloaded{ - [](const DerivationType::InputAddressed & ia) { - /* This might be a "pesimistically" deferred output, so we don't - "taint" the kind yet. */ - return false; - }, - [](const DerivationType::ContentAddressed & ca) { - // Already covered - assert(!ca.fixed); - return true; - }, - [](const DerivationType::Impure &) { return true; }}, - drv.type().raw)) { - return DrvHashModulo::DeferredDrv{}; - } + auto kind = std::visit( + overloaded{ + [](const DerivationType::InputAddressed & ia) { + /* This might be a "pesimistically" deferred output, so we don't + "taint" the kind yet. */ + return DrvHash::Kind::Regular; + }, + [](const DerivationType::ContentAddressed & ca) { + return ca.fixed ? DrvHash::Kind::Regular : DrvHash::Kind::Deferred; + }, + [](const DerivationType::Impure &) -> DrvHash::Kind { return DrvHash::Kind::Deferred; }}, + drv.type().raw); - /* For other derivations, replace the inputs paths with recursive - calls to this function. */ DerivedPathMap::ChildNode::Map inputs2; for (auto & [drvPath, node] : drv.inputDrvs.map) { - /* Need to build and resolve dynamic derivations first */ - if (!node.childMap.empty()) { - return DrvHashModulo::DeferredDrv{}; - } - const auto & res = pathDerivationModulo(store, drvPath); - if (std::visit( - overloaded{ - [&](const DrvHashModulo::DeferredDrv &) { return true; }, - // Regular non-CA derivation, replace derivation - [&](const DrvHashModulo::DrvHash & drvHash) { - inputs2.insert_or_assign(drvHash.to_string(HashFormat::Base16, false), node); - return false; - }, - // CA derivation's output hashes - [&](const DrvHashModulo::CaOutputHashes & outputHashes) { - for (auto & outputName : node.value) { - /* Put each one in with a single "out" output.. */ - const auto h = get(outputHashes, outputName); - if (!h) - throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2.insert_or_assign( - h->to_string(HashFormat::Base16, false), - DerivedPathMap::ChildNode{ - .value = {"out"}, - }); - } - return false; - }, - }, - res.raw)) { - return DrvHashModulo::DeferredDrv{}; + if (res.kind == DrvHash::Kind::Deferred) + kind = DrvHash::Kind::Deferred; + for (auto & outputName : node.value) { + const auto h = get(res.hashes, outputName); + if (!h) + throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); + inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName); } } - return hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); + auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); + + std::map outputHashes; + for (const auto & [outputName, _] : drv.outputs) { + outputHashes.insert_or_assign(outputName, hash); + } + + return DrvHash{ + .hashes = outputHashes, + .kind = kind, + }; +} + +std::map staticOutputHashes(Store & store, const Derivation & drv) +{ + return hashDerivationModulo(store, drv, true).hashes; } static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store) @@ -1113,6 +1102,26 @@ void BasicDerivation::applyRewrites(const StringMap & rewrites) } } +static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) +{ + drv.applyRewrites(rewrites); + + auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); + for (auto & [outputName, output] : drv.outputs) { + if (std::holds_alternative(output.raw)) { + auto h = get(hashModulo.hashes, outputName); + if (!h) + throw Error( + "derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", drv.name, outputName); + auto outPath = store.makeOutputPath(outputName, *h, drv.name); + drv.env[outputName] = store.printStorePath(outPath); + output = DerivationOutput::InputAddressed{ + .path = std::move(outPath), + }; + } + } +} + bool Derivation::shouldResolve() const { /* No input drvs means nothing to resolve. */ @@ -1224,13 +1233,9 @@ std::optional Derivation::tryResolve( queryResolutionChain)) return std::nullopt; - resolved.applyRewrites(inputRewrites); - - Derivation resolved2{std::move(resolved)}; - - resolved2.fillInOutputPaths(store); + rewriteDerivation(store, resolved, inputRewrites); - return resolved2; + return resolved; } /** @@ -1257,15 +1262,7 @@ std::optional Derivation::tryResolve( template static void processDerivationOutputPaths(Store & store, auto && drv, std::string_view drvName) { - std::optional hashModulo_; - - auto hashModulo = [&]() -> const auto & { - if (!hashModulo_) { - // somewhat expensive so we do lazily - hashModulo_ = hashDerivationModulo(store, drv, true); - } - return *hashModulo_; - }; + std::optional hashesModulo; for (auto & [outputName, output] : drv.outputs) { auto envHasRightPath = [&](const StorePath & actual, bool isDeferred = false) { @@ -1302,64 +1299,65 @@ static void processDerivationOutputPaths(Store & store, auto && drv, std::string } }; auto hash = [&](const Output & outputVariant) { - std::visit( - overloaded{ - [&](const DrvHashModulo::DrvHash & drvHash) { - auto outPath = store.makeOutputPath(outputName, drvHash, drvName); - - if constexpr (std::is_same_v) { - if (outputVariant.path == outPath) { - envHasRightPath(outPath); - return; // Correct case - } - /* Error case, an explicitly wrong path is - always an error. */ - throw Error( - "derivation has incorrect output '%s', should be '%s'", - store.printStorePath(outputVariant.path), - store.printStorePath(outPath)); - } else if constexpr (std::is_same_v) { - if constexpr (fillIn) { - /* Fill in output path for Deferred outputs */ - output = DerivationOutput::InputAddressed{ - .path = outPath, - }; - envHasRightPath(outPath); - } else { - /* Validation mode: deferred outputs - should have been filled in */ - warn( - "derivation has incorrect deferred output, should be '%s'.\nThis will be an error in future versions of Nix; compatibility of CA derivations will be broken.", - store.printStorePath(outPath)); - } - } else { - /* Will never happen, based on where - `hash` is called. */ - static_assert(false); - } - }, - [&](const DrvHashModulo::CaOutputHashes &) { - /* Shouldn't happen as the original output is - input-addressed (or deferred waiting to be). */ - assert(false); - }, - [&](const DrvHashModulo::DeferredDrv &) { - if constexpr (std::is_same_v) { - /* Error case, an explicitly wrong path is - always an error. */ - throw Error( - "derivation has incorrect output '%s', should be deferred", - store.printStorePath(outputVariant.path)); - } else if constexpr (std::is_same_v) { - /* Correct: Deferred output with Deferred hash kind. */ - } else { - /* Will never happen, based on where - `hash` is called. */ - static_assert(false); - } - }, - }, - hashModulo().raw); + if (!hashesModulo) { + // somewhat expensive so we do lazily + hashesModulo = hashDerivationModulo(store, drv, true); + } + switch (hashesModulo->kind) { + case DrvHash::Kind::Regular: { + auto h = get(hashesModulo->hashes, outputName); + if (!h) + throw Error("derivation produced no hash for output '%s'", outputName); + auto outPath = store.makeOutputPath(outputName, *h, drvName); + + if constexpr (std::is_same_v) { + if (outputVariant.path == outPath) { + return; // Correct case + } + /* Error case, an explicitly wrong path is + always an error. */ + throw Error( + "derivation has incorrect output '%s', should be '%s'", + store.printStorePath(outputVariant.path), + store.printStorePath(outPath)); + } else if constexpr (std::is_same_v) { + if constexpr (fillIn) + /* Fill in output path for Deferred + outputs */ + output = DerivationOutput::InputAddressed{ + .path = outPath, + }; + else + /* Validation mode: deferred outputs + should have been filled in */ + warn( + "derivation has incorrect deferred output, should be '%s'.\nThis will be an error in future versions of Nix; compatibility of CA derivations will be broken.", + store.printStorePath(outPath)); + } else { + /* Will never happen, based on where + `hash` is called. */ + static_assert(false); + } + envHasRightPath(outPath); + break; + } + case DrvHash::Kind::Deferred: + if constexpr (std::is_same_v) { + /* Error case, an explicitly wrong path is + always an error. */ + throw Error( + "derivation has incorrect output '%s', should be deferred", + store.printStorePath(outputVariant.path)); + } else if constexpr (std::is_same_v) { + /* Correct: Deferred output with Deferred + hash kind. */ + } else { + /* Will never happen, based on where + `hash` is called. */ + static_assert(false); + } + break; + } }; std::visit( overloaded{ diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 90b454744e3..c452d38bfc7 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -330,7 +330,7 @@ struct DummyStoreImpl : DummyStore void registerDrvOutput(const Realisation & output) override { - buildTrace.insert_or_visit({output.id.drvPath, {{output.id.outputName, output}}}, [&](auto & kv) { + buildTrace.insert_or_visit({output.id.drvHash, {{output.id.outputName, output}}}, [&](auto & kv) { kv.second.insert_or_assign(output.id.outputName, output); }); } @@ -339,7 +339,7 @@ struct DummyStoreImpl : DummyStore const DrvOutput & drvOutput, Callback> callback) noexcept override { bool visited = false; - buildTrace.cvisit(drvOutput.drvPath, [&](const auto & kv) { + buildTrace.cvisit(drvOutput.drvHash, [&](const auto & kv) { if (auto it = kv.second.find(drvOutput.outputName); it != kv.second.end()) { visited = true; callback(std::make_shared(it->second)); @@ -436,7 +436,11 @@ ref adl_serializer>::from_json(const json & json) for (auto & [k1, v2] : getObject(v)) { UnkeyedRealisation realisation = v2; res->buildTrace.insert_or_visit( - {StorePath{k0}, {{k1, realisation}}}, [&](auto & kv) { kv.second.insert_or_assign(k1, realisation); }); + { + Hash::parseExplicitFormatUnprefixed(k0, HashAlgorithm::SHA256, HashFormat::Base64), + {{k1, realisation}}, + }, + [&](auto & kv) { kv.second.insert_or_assign(k1, realisation); }); } } return res; @@ -469,7 +473,7 @@ void adl_serializer::to_json(json & json, const DummyStore & val) auto obj = json::object(); val.buildTrace.cvisit_all([&](const auto & kv) { auto & [k, v] = kv; - auto & obj2 = obj[k.to_string()] = json::object(); + auto & obj2 = obj[k.to_string(HashFormat::Base64, false)] = json::object(); for (auto & [k2, v2] : kv.second) obj2[k2] = v2; }); diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 2a90691a93b..a41bf515dc6 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -90,18 +90,8 @@ protected: /** * The prefix under which realisation infos will be stored - * - * @note The previous (still experimental, though) hash-keyed - * realisations were under "realisations". "build trace" is a better - * name anyways (issue #11895). This is call "v2" accordingly. - * - * While we're experimenting, we'll freely increase this version - * number. Old build traces will just be "abandoned" at the old URL. - * When we are done experimenting, we'll try lean more on versioning - * the build trace entries themselves than the entire directory, for - * a smoother migration path. */ - constexpr const static std::string realisationsPrefix = "build-trace-v2"; + constexpr const static std::string realisationsPrefix = "realisations"; constexpr const static std::string cacheInfoFile = "nix-cache-info"; @@ -110,7 +100,7 @@ protected: /** * Compute the path to the given realisation * - * It's `${realisationsPrefix}/${drvPath}/${outputName}`. + * It's `${realisationsPrefix}/${drvOutput}.doi`. */ std::string makeRealisationPath(const DrvOutput & id); diff --git a/src/libstore/include/nix/store/build/derivation-building-misc.hh b/src/libstore/include/nix/store/build/derivation-building-misc.hh index 8d6892839c7..2b68fa1782a 100644 --- a/src/libstore/include/nix/store/build/derivation-building-misc.hh +++ b/src/libstore/include/nix/store/build/derivation-building-misc.hh @@ -45,6 +45,7 @@ struct InitialOutputStatus struct InitialOutput { + Hash outputHash; std::optional known; }; diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index c6bd412182f..aaded75511f 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -66,6 +66,8 @@ private: */ std::unique_ptr drv; + const Hash outputHash; + const BuildMode buildMode; /** diff --git a/src/libstore/include/nix/store/common-protocol-impl.hh b/src/libstore/include/nix/store/common-protocol-impl.hh index d0eebe0bc23..cb1020a3c83 100644 --- a/src/libstore/include/nix/store/common-protocol-impl.hh +++ b/src/libstore/include/nix/store/common-protocol-impl.hh @@ -26,13 +26,12 @@ namespace nix { LengthPrefixedProtoHelper::write(store, conn, t); \ } -COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) #define COMMA_ , +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) -COMMON_USE_LENGTH_PREFIX_SERIALISER( - template, std::map) +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::map) #undef COMMA_ /* protocol-specific templates */ diff --git a/src/libstore/include/nix/store/common-protocol.hh b/src/libstore/include/nix/store/common-protocol.hh index 2ef0f795b4e..341d87b4166 100644 --- a/src/libstore/include/nix/store/common-protocol.hh +++ b/src/libstore/include/nix/store/common-protocol.hh @@ -88,9 +88,8 @@ DECLARE_COMMON_SERIALISER(std::set); template DECLARE_COMMON_SERIALISER(std::tuple); -template -DECLARE_COMMON_SERIALISER(std::map); -#undef COMMA_ +template +DECLARE_COMMON_SERIALISER(std::map); /** * These use the empty string for the null case, relying on the fact diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index f63acdf94ba..05e42200449 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -504,40 +504,35 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName); * derivations (fixed-output or not) will have a different hash for each * output. */ -struct DrvHashModulo +struct DrvHash { /** - * Single hash for the derivation - * - * This is for an input-addressed derivation that doesn't - * transitively depend on any floating-CA derivations. + * Map from output names to hashes */ - using DrvHash = Hash; + std::map hashes; - /** - * Known CA drv's output hashes, for fixed-output derivations whose - * output hashes are always known since they are fixed up-front. - */ - using CaOutputHashes = std::map; + enum struct Kind : bool { + /** + * Statically determined derivations. + * This hash will be directly used to compute the output paths + */ + Regular, + + /** + * Floating-output derivations (and their reverse dependencies). + */ + Deferred, + }; /** - * This derivation doesn't yet have known output hashes. - * - * Either because itself is floating CA, or it (transtively) depends - * on a floating CA derivation. + * The kind of derivation this is, simplified for just "derivation hash + * modulo" purposes. */ - using DeferredDrv = std::monostate; - - using Raw = std::variant; - - Raw raw; - - bool operator==(const DrvHashModulo &) const = default; - // auto operator <=> (const DrvHashModulo &) const = default; - - MAKE_WRAPPER_CONSTRUCTOR(DrvHashModulo); + Kind kind; }; +void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; + /** * Returns hashes with the details of fixed-output subderivations * expunged. @@ -562,17 +557,15 @@ struct DrvHashModulo * ATerm, after subderivations have been likewise expunged from that * derivation. */ -DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); +DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); /** - * If a derivation is input addressed and doesn't yet have its input - * addressed (is deferred) try using `hashDerivationModulo`. + * Return a map associating each output to a hash that uniquely identifies its + * derivation (modulo the self-references). * - * Does nothing if not deferred input-addressed, or - * `hashDerivationModulo` indicates it is missing inputs' output paths - * and is not yet ready (and must stay deferred). + * \todo What is the Hash in this map? */ -void resolveInputAddressed(Store & store, Derivation & drv); +std::map staticOutputHashes(Store & store, const Derivation & drv); struct DrvHashFct { @@ -587,7 +580,7 @@ struct DrvHashFct /** * Memoisation of hashDerivationModulo(). */ -typedef boost::concurrent_flat_map DrvHashes; +typedef boost::concurrent_flat_map DrvHashes; // FIXME: global, though at least thread-safe. extern DrvHashes drvHashes; diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index 8fdeeb36251..ac7ab9c680f 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -49,7 +49,7 @@ struct DummyStore : virtual Store * outer map for the derivation, and inner maps for the outputs of a * given derivation. */ - boost::concurrent_flat_map> buildTrace; + boost::concurrent_flat_map> buildTrace; DummyStore(ref config) : Store{*config} diff --git a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh index e1a80e8dc58..035019340f5 100644 --- a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh +++ b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh @@ -56,14 +56,14 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector); #define COMMA_ , template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); +#undef COMMA_ template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); -template -#define LENGTH_PREFIXED_PROTO_HELPER_X std::map +template +#define LENGTH_PREFIXED_PROTO_HELPER_X std::map LENGTH_PREFIXED_PROTO_HELPER(Inner, LENGTH_PREFIXED_PROTO_HELPER_X); -#undef COMMA_ template std::vector @@ -109,11 +109,11 @@ void LengthPrefixedProtoHelper>::write( } } -template -std::map LengthPrefixedProtoHelper>::read( - const StoreDirConfig & store, typename Inner::ReadConn conn) +template +std::map +LengthPrefixedProtoHelper>::read(const StoreDirConfig & store, typename Inner::ReadConn conn) { - std::map resMap; + std::map resMap; auto size = readNum(conn.from); while (size--) { auto k = S::read(store, conn); @@ -123,9 +123,9 @@ std::map LengthPrefixedProtoHelper return resMap; } -template -void LengthPrefixedProtoHelper>::write( - const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) +template +void LengthPrefixedProtoHelper>::write( + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) { conn.to << resMap.size(); for (auto & i : resMap) { diff --git a/src/libstore/include/nix/store/realisation.hh b/src/libstore/include/nix/store/realisation.hh index 16e596e3461..55597acb7c0 100644 --- a/src/libstore/include/nix/store/realisation.hh +++ b/src/libstore/include/nix/store/realisation.hh @@ -18,40 +18,33 @@ struct OutputsSpec; /** * A general `Realisation` key. * - * This is similar to a `DerivedPath::Built`, except it is only a single - * step: `drvPath` is a `StorePath` rather than a `DerivedPath`. + * This is similar to a `DerivedPath::Opaque`, but the derivation is + * identified by its "hash modulo" instead of by its store path. */ struct DrvOutput { /** - * The store path to the derivation + * The hash modulo of the derivation. + * + * Computed from the derivation itself for most types of + * derivations, but computed from the (fixed) content address of the + * output for fixed-output derivations. */ - StorePath drvPath; + Hash drvHash; /** * The name of the output. */ OutputName outputName; - /** - * Skips the store dir on the `drvPath` - */ std::string to_string() const; - /** - * Skips the store dir on the `drvPath` - */ - static DrvOutput from_string(std::string_view); - - /** - * Includes the store dir on `drvPath` - */ - std::string render(const StoreDirConfig & store) const; + std::string strHash() const + { + return drvHash.to_string(HashFormat::Base16, true); + } - /** - * Includes the store dir on `drvPath` - */ - static DrvOutput parse(const StoreDirConfig & store, std::string_view); + static DrvOutput parse(const std::string &); bool operator==(const DrvOutput &) const = default; auto operator<=>(const DrvOutput &) const = default; @@ -71,16 +64,6 @@ struct UnkeyedRealisation size_t checkSignatures(const DrvOutput & key, const PublicKeys & publicKeys) const; - /** - * Just check the `outPath`. Signatures don't matter for this. - * Callers must ensure that the corresponding key is the same for - * most use-cases. - */ - bool isCompatibleWith(const UnkeyedRealisation & other) const - { - return outPath == other.outPath; - } - const StorePath & getPath() const { return outPath; @@ -94,6 +77,8 @@ struct Realisation : UnkeyedRealisation { DrvOutput id; + bool isCompatibleWith(const UnkeyedRealisation & other) const; + bool operator==(const Realisation &) const = default; auto operator<=>(const Realisation &) const = default; }; @@ -104,7 +89,16 @@ struct Realisation : UnkeyedRealisation * Since these are the outputs of a single derivation, we know the * output names are unique so we can use them as the map key. */ -typedef std::map SingleDrvOutputs; +typedef std::map SingleDrvOutputs; + +/** + * Collection type for multiple derivations' outputs' `Realisation`s. + * + * `DrvOutput` is used because in general the derivations are not all + * the same, so we need to identify firstly which derivation, and + * secondly which output of that derivation. + */ +typedef std::map DrvOutputs; struct OpaquePath { @@ -155,17 +149,19 @@ struct RealisedPath class MissingRealisation : public Error { public: - MissingRealisation(const StoreDirConfig & store, DrvOutput & outputId) - : MissingRealisation(store, outputId.drvPath, outputId.outputName) + MissingRealisation(DrvOutput & outputId) + : MissingRealisation(outputId.outputName, outputId.strHash()) { } - MissingRealisation(const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName); - MissingRealisation( - const StoreDirConfig & store, - const SingleDerivedPath & drvPath, - const StorePath & drvPathResolved, - const OutputName & outputName); + MissingRealisation(std::string_view drv, OutputName outputName) + : Error( + "cannot operate on output '%s' of the " + "unbuilt derivation '%s'", + outputName, + drv) + { + } }; } // namespace nix diff --git a/src/libstore/include/nix/store/serve-protocol-impl.hh b/src/libstore/include/nix/store/serve-protocol-impl.hh index 24fc3c9abd8..a9617165a72 100644 --- a/src/libstore/include/nix/store/serve-protocol-impl.hh +++ b/src/libstore/include/nix/store/serve-protocol-impl.hh @@ -34,10 +34,8 @@ SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) #define SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA , SERVE_USE_LENGTH_PREFIX_SERIALISER( - template - , - std::map) + template, + std::map) /** * Use `CommonProto` where possible. diff --git a/src/libstore/include/nix/store/serve-protocol.hh b/src/libstore/include/nix/store/serve-protocol.hh index 94a250b2e3c..dba05a34548 100644 --- a/src/libstore/include/nix/store/serve-protocol.hh +++ b/src/libstore/include/nix/store/serve-protocol.hh @@ -8,18 +8,12 @@ namespace nix { #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb -#define SERVE_PROTOCOL_VERSION (2 << 8 | 8) -#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) -#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) struct StoreDirConfig; struct Source; // items being serialised struct BuildResult; struct UnkeyedValidPathInfo; -struct DrvOutput; -struct UnkeyedRealisation; -struct Realisation; /** * The "serve protocol", used by ssh:// stores. @@ -69,7 +63,7 @@ struct ServeProto static constexpr Version latest = { .major = 2, - .minor = 8, + .minor = 7, }; /** @@ -211,12 +205,6 @@ inline std::ostream & operator<<(std::ostream & s, ServeProto::Command op) template<> DECLARE_SERVE_SERIALISER(BuildResult); template<> -DECLARE_SERVE_SERIALISER(DrvOutput); -template<> -DECLARE_SERVE_SERIALISER(UnkeyedRealisation); -template<> -DECLARE_SERVE_SERIALISER(Realisation); -template<> DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo); template<> DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions); @@ -229,8 +217,8 @@ DECLARE_SERVE_SERIALISER(std::set); template DECLARE_SERVE_SERIALISER(std::tuple); -template -DECLARE_SERVE_SERIALISER(std::map); +template +DECLARE_SERVE_SERIALISER(std::map); #undef COMMA_ } // namespace nix diff --git a/src/libstore/include/nix/store/worker-protocol-impl.hh b/src/libstore/include/nix/store/worker-protocol-impl.hh index 605663d2eeb..26f6b9d44e4 100644 --- a/src/libstore/include/nix/store/worker-protocol-impl.hh +++ b/src/libstore/include/nix/store/worker-protocol-impl.hh @@ -34,10 +34,8 @@ WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) #define WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA , WORKER_USE_LENGTH_PREFIX_SERIALISER( - template - , - std::map) + template, + std::map) /** * Use `CommonProto` where possible. diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index 8e20d3c992c..4098e8fd912 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -12,12 +12,6 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -/* Note: you generally shouldn't change the protocol version. Define a - new `WorkerProto::Feature` instead. */ -#define PROTOCOL_VERSION (1 << 8 | 39) -#define MINIMUM_PROTOCOL_VERSION (1 << 8 | 18) -#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) -#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define STDERR_NEXT 0x6f6c6d67 #define STDERR_READ 0x64617461 // data needed from source #define STDERR_WRITE 0x64617416 // data for sink @@ -36,9 +30,6 @@ struct BuildResult; struct KeyedBuildResult; struct ValidPathInfo; struct UnkeyedValidPathInfo; -struct DrvOutput; -struct UnkeyedRealisation; -struct Realisation; enum BuildMode : uint8_t; enum TrustedFlag : bool; enum class GCAction; @@ -118,12 +109,6 @@ struct WorkerProto static const Version minimum; - /** - * Feature for transmitting `UnkeyedRealisation` and `DrvOutput` - * using drvPath (store path) instead of the old hash-based JSON format. - */ - static constexpr std::string_view featureRealisationWithPath = "realisation-with-path-not-hash"; - /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. @@ -322,14 +307,6 @@ DECLARE_WORKER_SERIALISER(ValidPathInfo); template<> DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); template<> -DECLARE_WORKER_SERIALISER(DrvOutput); -template<> -DECLARE_WORKER_SERIALISER(UnkeyedRealisation); -template<> -DECLARE_WORKER_SERIALISER(Realisation); -template<> -DECLARE_WORKER_SERIALISER(std::optional); -template<> DECLARE_WORKER_SERIALISER(BuildMode); template<> DECLARE_WORKER_SERIALISER(GCAction); @@ -348,8 +325,8 @@ DECLARE_WORKER_SERIALISER(std::set); template DECLARE_WORKER_SERIALISER(std::tuple); -template -DECLARE_WORKER_SERIALISER(std::map); +template +DECLARE_WORKER_SERIALISER(std::map); #undef COMMA_ } // namespace nix diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0d06c747146..cdddcefcc07 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -360,14 +360,14 @@ LocalStore::LocalStore(ref config) state->stmts->RegisterRealisedOutput.create( state->db, R"( - insert into BuildTraceV2 (drvPath, outputName, outputPath, signatures) + insert into Realisations (drvPath, outputName, outputPath, signatures) values (?, ?, (select id from ValidPaths where path = ?), ?) ; )"); state->stmts->UpdateRealisedOutput.create( state->db, R"( - update BuildTraceV2 + update Realisations set signatures = ? where drvPath = ? and @@ -377,16 +377,16 @@ LocalStore::LocalStore(ref config) state->stmts->QueryRealisedOutput.create( state->db, R"( - select BuildTraceV2.id, Output.path, BuildTraceV2.signatures from BuildTraceV2 - inner join ValidPaths as Output on Output.id = BuildTraceV2.outputPath + select Realisations.id, Output.path, Realisations.signatures from Realisations + inner join ValidPaths as Output on Output.id = Realisations.outputPath where drvPath = ? and outputName = ? ; )"); state->stmts->QueryAllRealisedOutputs.create( state->db, R"( - select outputName, Output.path from BuildTraceV2 - inner join ValidPaths as Output on Output.id = BuildTraceV2.outputPath + select outputName, Output.path from Realisations + inner join ValidPaths as Output on Output.id = Realisations.outputPath where drvPath = ? ; )"); @@ -604,7 +604,7 @@ void LocalStore::upgradeDBSchema(State & state) if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) doUpgrade( - "20251016-ca-derivations", + "20220326-ca-derivations", #include "ca-specific-schema.sql.gen.hh" ); } @@ -648,8 +648,8 @@ void LocalStore::registerDrvOutput(const Realisation & info) auto combinedSignatures = oldR->signatures; combinedSignatures.insert(info.signatures.begin(), info.signatures.end()); state->stmts->UpdateRealisedOutput - .use()(concatStringsSep(" ", Signature::toStrings(combinedSignatures)))( - info.id.drvPath.to_string())(info.id.outputName) + .use()(concatStringsSep(" ", Signature::toStrings(combinedSignatures)))(info.id.strHash())( + info.id.outputName) .exec(); } else { throw Error( @@ -663,7 +663,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) } } else { state->stmts->RegisterRealisedOutput - .use()(info.id.drvPath.to_string())(info.id.outputName)(printStorePath(info.outPath))( + .use()(info.id.strHash())(info.id.outputName)(printStorePath(info.outPath))( concatStringsSep(" ", Signature::toStrings(info.signatures))) .exec(); } @@ -1539,7 +1539,7 @@ void LocalStore::addSignatures(const StorePath & storePath, const std::set> LocalStore::queryRealisationCore_(LocalStore::State & state, const DrvOutput & id) { - auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.drvPath.to_string())(id.outputName)); + auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.strHash())(id.outputName)); if (!useQueryRealisedOutput.next()) return std::nullopt; auto realisationDbId = useQueryRealisedOutput.getInt(0); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index f8f94070e48..91879c0554e 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -242,15 +242,16 @@ MissingPaths Store::queryMissing(const std::vector & targets) // If there are unknown output paths, attempt to find if the // paths are known to substituters through a realisation. + auto outputHashes = staticOutputHashes(*this, *drv); knownOutputPaths = true; - for (auto & [outputName, _] : drv->outputs) { + for (auto [outputName, hash] : outputHashes) { if (!bfd.outputs.contains(outputName)) continue; bool found = false; for (auto & sub : getDefaultSubstituters()) { - auto realisation = sub->queryRealisation({drvPath, outputName}); + auto realisation = sub->queryRealisation({hash, outputName}); if (!realisation) continue; found = true; @@ -367,7 +368,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, OutputPathMap outputs; for (auto & [outputName, outputPathOpt] : outputsOpt) { if (!outputPathOpt) - throw MissingRealisation(store, *bfd.drvPath, drvPath, outputName); + throw MissingRealisation(bfd.drvPath->to_string(store), outputName); auto & outputPath = *outputPathOpt; outputs.insert_or_assign(outputName, outputPath); } @@ -391,7 +392,7 @@ StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store bfd.output); auto & optPath = outputPaths.at(bfd.output); if (!optPath) - throw MissingRealisation(store, *bfd.drvPath, drvPath, bfd.output); + throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output); return *optPath; }, }, diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index bbb42a97e56..18932ca1250 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -42,18 +42,12 @@ create table if not exists NARs ( foreign key (cache) references BinaryCaches(id) on delete cascade ); -create table if not exists BuildTrace ( +create table if not exists Realisations ( cache integer not null, - - drvPath text not null, - outputName text not null, - - -- The following are null if the realisation is absent - outputPath text, - sigs text, - + outputId text not null, + content blob, -- Json serialisation of the realisation, or null if the realisation is absent timestamp integer not null, - primary key (cache, drvPath, outputName), + primary key (cache, outputId), foreign key (cache) references BinaryCaches(id) on delete cascade ); @@ -93,7 +87,7 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache NarInfoDiskCacheImpl( const Settings & settings, SQLiteSettings sqliteSettings, - Path dbPath = (getCacheDir() / "binary-cache-v8.sqlite").string()) + Path dbPath = (getCacheDir() / "binary-cache-v7.sqlite").string()) : NarInfoDiskCache{settings} { auto state(_state.lock()); @@ -129,24 +123,24 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache state->insertRealisation.create( state->db, R"( - insert or replace into BuildTrace(cache, drvPath, outputName, outputPath, sigs, timestamp) - values (?, ?, ?, ?, ?, ?) + insert or replace into Realisations(cache, outputId, content, timestamp) + values (?, ?, ?, ?) )"); state->insertMissingRealisation.create( state->db, R"( - insert or replace into BuildTrace(cache, drvPath, outputName, timestamp) - values (?, ?, ?, ?) + insert or replace into Realisations(cache, outputId, timestamp) + values (?, ?, ?) )"); state->queryRealisation.create( state->db, R"( - select outputPath, sigs from BuildTrace - where cache = ? and drvPath = ? and outputName = ? and - ((outputPath is null and timestamp > ?) or - (outputPath is not null and timestamp > ?)) + select content from Realisations + where cache = ? and outputId = ? and + ((content is null and timestamp > ?) or + (content is not null and timestamp > ?)) )"); /* Periodically purge expired entries from the database. */ @@ -303,27 +297,22 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache auto now = time(0); - auto queryRealisation(state->queryRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)( + auto queryRealisation(state->queryRealisation.use()(cache.id)(id.to_string())( now - settings.ttlNegative)(now - settings.ttlPositive)); if (!queryRealisation.next()) - return {oUnknown, nullptr}; + return {oUnknown, 0}; if (queryRealisation.isNull(0)) - return {oInvalid, nullptr}; + return {oInvalid, 0}; try { return { oValid, - std::make_shared( - UnkeyedRealisation{ - .outPath = StorePath{queryRealisation.getStr(0)}, - .signatures = nlohmann::json::parse(queryRealisation.getStr(1)), - }, - id), + std::make_shared(nlohmann::json::parse(queryRealisation.getStr(0))), }; } catch (Error & e) { - e.addTrace({}, "reading build trace key-value from the local disk cache"); + e.addTrace({}, "while parsing the local disk cache"); throw; } }); @@ -369,9 +358,7 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache auto & cache(getCache(*state, uri)); state->insertRealisation - .use()(cache.id)(realisation.id.drvPath.to_string())(realisation.id.outputName)( - realisation.outPath.to_string())(static_cast(realisation.signatures).dump())( - time(0)) + .use()(cache.id)(realisation.id.to_string())(static_cast(realisation).dump())(time(0)) .exec(); }); } @@ -382,7 +369,7 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache auto state(_state.lock()); auto & cache(getCache(*state, uri)); - state->insertMissingRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)(time(0)).exec(); + state->insertMissingRealisation.use()(cache.id)(id.to_string())(time(0)).exec(); }); } }; diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 0fff0bc914a..433d67f26ba 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -8,34 +8,28 @@ namespace nix { MakeError(InvalidDerivationOutputId, Error); -DrvOutput DrvOutput::parse(const StoreDirConfig & store, std::string_view s) +DrvOutput DrvOutput::parse(const std::string & strRep) { - size_t n = s.rfind('^'); - if (n == s.npos) - throw InvalidDerivationOutputId("Invalid derivation output id '%s': missing '^'", s); + size_t n = strRep.find("!"); + if (n == strRep.npos) + throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep); + return DrvOutput{ - .drvPath = store.parseStorePath(s.substr(0, n)), - .outputName = OutputName{s.substr(n + 1)}, + .drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)), + .outputName = strRep.substr(n + 1), }; } -std::string DrvOutput::render(const StoreDirConfig & store) const -{ - return std::string(store.printStorePath(drvPath)) + "^" + outputName; -} - std::string DrvOutput::to_string() const { - return std::string(drvPath.to_string()) + "^" + outputName; + return strHash() + "!" + outputName; } std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const { - auto serialised = static_cast(Realisation{*this, key}); - auto value = serialised.find("value"); - assert(value != serialised.end()); - value->erase("signatures"); - return serialised.dump(); + nlohmann::json serialized = Realisation{*this, key}; + serialized.erase("signatures"); + return serialized.dump(); } void UnkeyedRealisation::sign(const DrvOutput & key, const Signer & signer) @@ -67,20 +61,9 @@ const StorePath & RealisedPath::path() const & return std::visit([](auto & arg) -> auto & { return arg.getPath(); }, raw); } -MissingRealisation::MissingRealisation( - const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName) - : Error("cannot operate on output '%s' of the unbuilt derivation '%s'", outputName, store.printStorePath(drvPath)) -{ -} - -MissingRealisation::MissingRealisation( - const StoreDirConfig & store, - const SingleDerivedPath & drvPath, - const StorePath & drvPathResolved, - const OutputName & outputName) - : MissingRealisation{store, drvPathResolved, outputName} +bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const { - addTrace({}, "looking up realisation for derivation '%s'", drvPath.to_string(store)); + return outPath == other.outPath; } } // namespace nix @@ -91,20 +74,12 @@ using namespace nix; DrvOutput adl_serializer::from_json(const json & json) { - auto obj = getObject(json); - - return { - .drvPath = valueAt(obj, "drvPath"), - .outputName = getString(valueAt(obj, "outputName")), - }; + return DrvOutput::parse(getString(json)); } void adl_serializer::to_json(json & json, const DrvOutput & drvOutput) { - json = { - {"drvPath", drvOutput.drvPath}, - {"outputName", drvOutput.outputName}, - }; + json = drvOutput.to_string(); } UnkeyedRealisation adl_serializer::from_json(const json & json0) @@ -126,25 +101,25 @@ void adl_serializer::to_json(json & json, const UnkeyedReali json = { {"outPath", r.outPath}, {"signatures", r.signatures}, + // back-compat + {"dependentRealisations", json::object()}, }; } -Realisation adl_serializer::from_json(const json & json) +Realisation adl_serializer::from_json(const json & json0) { - auto obj = getObject(json); + auto json = getObject(json0); - return { - static_cast(valueAt(obj, "value")), - static_cast(valueAt(obj, "key")), + return Realisation{ + static_cast(json0), + valueAt(json, "id"), }; } void adl_serializer::to_json(json & json, const Realisation & r) { - json = { - {"key", r.id}, - {"value", static_cast(r)}, - }; + json = static_cast(r); + json["id"] = r.id; } } // namespace nlohmann diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 2e16c999699..f317e906e41 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -503,7 +503,12 @@ void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); conn->to << WorkerProto::Op::RegisterDrvOutput; - WorkerProto::write(*this, *conn, info); + if (conn->protoVersion.number < WorkerProto::Version::Number{1, 31}) { + WorkerProto::write(*this, *conn, info.id); + conn->to << std::string(info.outPath.to_string()); + } else { + WorkerProto::write(*this, *conn, info); + } conn.processStderr(); } @@ -513,10 +518,8 @@ void RemoteStore::queryRealisationUncached( try { auto conn(getConnection()); - if (!conn->protoVersion.features.contains(WorkerProto::featureRealisationWithPath)) { - warn( - "the daemon is missing the '%s' protocol feature, needed to support content-addressing derivations", - WorkerProto::featureRealisationWithPath); + if (conn->protoVersion.number < WorkerProto::Version::Number{1, 27}) { + warn("the daemon is too old to support content-addressing derivations, please upgrade it to 2.4"); return callback(nullptr); } @@ -524,12 +527,21 @@ void RemoteStore::queryRealisationUncached( WorkerProto::write(*this, *conn, id); conn.processStderr(); - callback([&]() -> std::shared_ptr { - auto realisation = WorkerProto::Serialise>::read(*this, *conn); - if (!realisation) - return nullptr; - return std::make_shared(*realisation); - }()); + auto real = [&]() -> std::shared_ptr { + if (conn->protoVersion.number < WorkerProto::Version::Number{1, 31}) { + auto outPaths = WorkerProto::Serialise>::read(*this, *conn); + if (outPaths.empty()) + return nullptr; + return std::make_shared(UnkeyedRealisation{.outPath = *outPaths.begin()}); + } else { + auto realisations = WorkerProto::Serialise>::read(*this, *conn); + if (realisations.empty()) + return nullptr; + return std::make_shared(*realisations.begin()); + } + }(); + + callback(std::shared_ptr(real)); } catch (...) { return callback.rethrow(); } @@ -611,19 +623,30 @@ std::vector RemoteStore::buildPathsWithResults( OutputPathMap outputs; auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath); + auto drv = evalStore->readDerivation(drvPath); + const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive auto built = resolveDerivedPath(*this, bfd, &*evalStore); for (auto & [output, outputPath] : built) { - auto outputId = DrvOutput{drvPath, output}; + auto outputHash = get(outputHashes, output); + if (!outputHash) + throw Error( + "the derivation '%s' doesn't have an output named '%s'", + printStorePath(drvPath), + output); + auto outputId = DrvOutput{*outputHash, output}; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = queryRealisation(outputId); if (!realisation) - throw MissingRealisation(*this, outputId); - success.builtOutputs.emplace(output, *realisation); + throw MissingRealisation(outputId); + success.builtOutputs.emplace(output, Realisation{*realisation, outputId}); } else { success.builtOutputs.emplace( output, - UnkeyedRealisation{ - .outPath = outputPath, + Realisation{ + UnkeyedRealisation{ + .outPath = outputPath, + }, + outputId, }); } } diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 91162015449..15b2a3d049c 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -280,18 +280,9 @@ std::vector RestrictedStore::buildPathsWithResults( for (auto & result : results) { if (auto * successP = result.tryGetSuccess()) { - if (auto * pathBuilt = std::get_if(&result.path)) { - // TODO ugly extra IO - auto drvPath = resolveDerivedPath(*next, *pathBuilt->drvPath); - for (auto & [outputName, output] : successP->builtOutputs) { - newPaths.insert(output.outPath); - newRealisations.insert( - {output, - { - .drvPath = drvPath, - .outputName = outputName, - }}); - } + for (auto & [outputName, output] : successP->builtOutputs) { + newPaths.insert(output.outPath); + newRealisations.insert(output); } } } diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index 258c529d505..ec7b85ed882 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -7,7 +7,6 @@ #include "nix/store/serve-protocol-impl.hh" #include "nix/util/archive.hh" #include "nix/store/path-info.hh" -#include "nix/util/json-utils.hh" #include @@ -29,19 +28,10 @@ BuildResult ServeProto::Serialise::read(const StoreDirConfig & stor if (conn.version >= ServeProto::Version{2, 3}) conn.from >> res.timesBuilt >> isNonDeterministic >> res.startTime >> res.stopTime; - - if (conn.version >= ServeProto::Version{2, 8}) { - success.builtOutputs = ServeProto::Serialise>::read(store, conn); - } else if (conn.version >= ServeProto::Version{2, 6}) { - for (auto & [output, realisation] : ServeProto::Serialise::read(store, conn)) { - size_t n = output.find("!"); - if (n == output.npos) - throw Error("Invalid derivation output id %s", output); - success.builtOutputs.insert_or_assign( - output.substr(n + 1), - UnkeyedRealisation{ - StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}}); - } + if (conn.version >= ServeProto::Version{2, 6}) { + auto builtOutputs = ServeProto::Serialise::read(store, conn); + for (auto && [output, realisation] : builtOutputs) + success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); } res.inner = std::visit( @@ -73,18 +63,15 @@ void ServeProto::Serialise::write( default value for the fields that don't exist in that case. */ auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) { conn.to << errorMsg; - if (conn.version >= ServeProto::Version{2, 3}) conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime; - - if (conn.version >= ServeProto::Version{2, 8}) { - ServeProto::write(store, conn, builtOutputs); - } else if (conn.version >= ServeProto::Version{2, 6}) { - // We no longer support these types of realisations - ServeProto::write(store, conn, StringMap{}); + if (conn.version >= ServeProto::Version{2, 6}) { + DrvOutputs builtOutputsFullKey; + for (auto & [output, realisation] : builtOutputs) + builtOutputsFullKey.insert_or_assign(realisation.id, realisation); + ServeProto::write(store, conn, builtOutputsFullKey); } }; - std::visit( overloaded{ [&](const BuildResult::Failure & failure) { @@ -171,82 +158,4 @@ void ServeProto::Serialise::write( } } -UnkeyedRealisation ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) -{ - if (conn.version < ServeProto::Version{2, 8}) { - throw Error( - "serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations", - conn.version.major, - conn.version.minor); - } - - auto outPath = ServeProto::Serialise::read(store, conn); - auto signatures = ServeProto::Serialise>::read(store, conn); - - return UnkeyedRealisation{ - .outPath = std::move(outPath), - .signatures = std::move(signatures), - }; -} - -void ServeProto::Serialise::write( - const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info) -{ - if (conn.version < ServeProto::Version{2, 8}) { - throw Error( - "serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations", - conn.version.major, - conn.version.minor); - } - ServeProto::write(store, conn, info.outPath); - ServeProto::write(store, conn, info.signatures); -} - -DrvOutput ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) -{ - if (conn.version < ServeProto::Version{2, 8}) { - throw Error( - "serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations", - conn.version.major, - conn.version.minor); - } - - auto drvPath = ServeProto::Serialise::read(store, conn); - auto outputName = ServeProto::Serialise::read(store, conn); - - return DrvOutput{ - .drvPath = std::move(drvPath), - .outputName = std::move(outputName), - }; -} - -void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info) -{ - if (conn.version < ServeProto::Version{2, 8}) { - throw Error( - "serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations", - conn.version.major, - conn.version.minor); - } - ServeProto::write(store, conn, info.drvPath); - ServeProto::write(store, conn, info.outputName); -} - -Realisation ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) -{ - auto id = ServeProto::Serialise::read(store, conn); - auto unkeyed = ServeProto::Serialise::read(store, conn); - - return Realisation{ - std::move(unkeyed), - std::move(id), - }; -} - -void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info) -{ - ServeProto::write(store, conn, info.id); - ServeProto::write(store, conn, static_cast(info)); -} - } // namespace nix diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 83446a1b343..149a0a553ea 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -380,8 +380,9 @@ Store::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore return outputs; auto drv = evalStore.readInvalidDerivation(path); - for (auto & [outputName, _] : drv.outputs) { - auto realisation = queryRealisation(DrvOutput{path, outputName}); + auto drvHashes = staticOutputHashes(*this, drv); + for (auto & [outputName, hash] : drvHashes) { + auto realisation = queryRealisation(DrvOutput{hash, outputName}); if (realisation) { outputs.insert_or_assign(outputName, realisation->outPath); } else { @@ -401,7 +402,7 @@ OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * ev OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) - throw MissingRealisation(*this, path, outName); + throw MissingRealisation(printStorePath(path), outName); result.insert_or_assign(outName, *optOutPath); } return result; diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index d73124fe28e..504063da097 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1958,10 +1958,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() { .outPath = newInfo.path, }, - DrvOutput{ - .drvPath = drvPath, - .outputName = outputName, - }, + DrvOutput{oldinfo->outputHash, outputName}, }; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().isImpure()) { store.signRealisation(thisRealisation); diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 8145b6c4431..49ab4c36b1e 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -8,7 +8,6 @@ #include "nix/store/worker-protocol-impl.hh" #include "nix/util/archive.hh" #include "nix/store/path-info.hh" -#include "nix/util/json-utils.hh" #include #include @@ -21,12 +20,6 @@ const WorkerProto::Version WorkerProto::latest = { .major = 1, .minor = 38, }, - .features = - { - std::string{ - WorkerProto::featureRealisationWithPath, - }, - }, }; const WorkerProto::Version WorkerProto::minimum = { @@ -261,19 +254,10 @@ BuildResult WorkerProto::Serialise::read(const StoreDirConfig & sto res.cpuUser = WorkerProto::Serialise>::read(store, conn); res.cpuSystem = WorkerProto::Serialise>::read(store, conn); } - - if (conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { - success.builtOutputs = WorkerProto::Serialise>::read(store, conn); - } else if (conn.version >= WorkerProto::Version{.number = {1, 28}}) { - for (auto && [output, realisation] : WorkerProto::Serialise::read(store, conn)) { - size_t n = output.find("!"); - if (n == output.npos) - throw Error("Invalid derivation output id %s", output); - success.builtOutputs.insert_or_assign( - output.substr(n + 1), - UnkeyedRealisation{ - StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}}); - } + if (conn.version >= WorkerProto::Version{.number = {1, 28}}) { + auto builtOutputs = WorkerProto::Serialise::read(store, conn); + for (auto && [output, realisation] : builtOutputs) + success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); } res.inner = std::visit( @@ -312,15 +296,13 @@ void WorkerProto::Serialise::write( WorkerProto::write(store, conn, res.cpuUser); WorkerProto::write(store, conn, res.cpuSystem); } - - if (conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { - WorkerProto::write(store, conn, builtOutputs); - } else if (conn.version >= WorkerProto::Version{.number = {1, 28}}) { - // Don't support those types of realisations anymore. - WorkerProto::write(store, conn, StringMap{}); + if (conn.version >= WorkerProto::Version{.number = {1, 28}}) { + DrvOutputs builtOutputsFullKey; + for (auto & [output, realisation] : builtOutputs) + builtOutputsFullKey.insert_or_assign(realisation.id, realisation); + WorkerProto::write(store, conn, builtOutputsFullKey); } }; - std::visit( overloaded{ [&](const BuildResult::Failure & failure) { @@ -413,109 +395,4 @@ void WorkerProto::Serialise::write( } } -UnkeyedRealisation WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) -{ - if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { - throw Error( - "the daemon is missing the '%s' protocol feature, needed to understand build trace", - WorkerProto::featureRealisationWithPath); - } - - auto outPath = WorkerProto::Serialise::read(store, conn); - auto signatures = WorkerProto::Serialise>::read(store, conn); - - return UnkeyedRealisation{ - .outPath = std::move(outPath), - .signatures = std::move(signatures), - }; -} - -void WorkerProto::Serialise::write( - const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info) -{ - if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { - throw Error( - "the daemon is missing the '%s' protocol feature, needed to understand build trace", - WorkerProto::featureRealisationWithPath); - } - WorkerProto::write(store, conn, info.outPath); - WorkerProto::write(store, conn, info.signatures); -} - -std::optional -WorkerProto::Serialise>::read(const StoreDirConfig & store, ReadConn conn) -{ - if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { - // Hack to improve compat - (void) WorkerProto::Serialise::read(store, conn); - return std::nullopt; - } else { - auto temp = readNum(conn.from); - switch (temp) { - case 0: - return std::nullopt; - case 1: - return WorkerProto::Serialise::read(store, conn); - default: - throw Error("Invalid optional build trace from remote"); - } - } -} - -void WorkerProto::Serialise>::write( - const StoreDirConfig & store, WriteConn conn, const std::optional & info) -{ - if (!info) { - conn.to << uint8_t{0}; - } else { - conn.to << uint8_t{1}; - WorkerProto::write(store, conn, *info); - } -} - -DrvOutput WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) -{ - if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { - throw Error( - "the daemon is missing the '%s' protocol feature, needed to support content-addressing derivations", - WorkerProto::featureRealisationWithPath); - } - - auto drvPath = WorkerProto::Serialise::read(store, conn); - auto outputName = WorkerProto::Serialise::read(store, conn); - - return DrvOutput{ - .drvPath = std::move(drvPath), - .outputName = std::move(outputName), - }; -} - -void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info) -{ - if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { - throw Error( - "the daemon is missing the '%s' protocol feature, needed to support content-addressing derivations", - WorkerProto::featureRealisationWithPath); - } - WorkerProto::write(store, conn, info.drvPath); - WorkerProto::write(store, conn, info.outputName); -} - -Realisation WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) -{ - auto id = WorkerProto::Serialise::read(store, conn); - auto unkeyed = WorkerProto::Serialise::read(store, conn); - - return Realisation{ - std::move(unkeyed), - std::move(id), - }; -} - -void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info) -{ - WorkerProto::write(store, conn, info.id); - WorkerProto::write(store, conn, static_cast(info)); -} - } // namespace nix diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index 17d8afdb89e..81933543b92 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -346,11 +346,13 @@ static int main_build_remote(int argc, char ** argv) optResult = std::move(res[0]); } + auto outputHashes = staticOutputHashes(*store, drv); std::set missingRealisations; StorePathSet missingPaths; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { for (auto & outputName : wantedOutputs) { - auto thisOutputId = DrvOutput{*drvPath, outputName}; + auto thisOutputHash = outputHashes.at(outputName); + auto thisOutputId = DrvOutput{thisOutputHash, outputName}; if (!store->queryRealisation(thisOutputId)) { debug("missing output %s", outputName); assert(optResult); @@ -360,7 +362,7 @@ static int main_build_remote(int argc, char ** argv) auto i = success.builtOutputs.find(outputName); assert(i != success.builtOutputs.end()); auto & newRealisation = i->second; - missingRealisations.insert({newRealisation, thisOutputId}); + missingRealisations.insert(newRealisation); missingPaths.insert(newRealisation.outPath); } } diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index b0f70be3a9e..6c4bf2e7051 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -163,13 +163,10 @@ StoreWrapper::queryPathInfo(char * path, int base32) } SV * -StoreWrapper::queryRawRealisation(char * drvPath, char * outputName) +StoreWrapper::queryRawRealisation(char * outputId) PPCODE: try { - auto realisation = THIS->store->queryRealisation(DrvOutput{ - .drvPath = THIS->store->parseStorePath(drvPath), - .outputName = outputName, - }); + auto realisation = THIS->store->queryRealisation(DrvOutput::parse(outputId)); if (realisation) XPUSHs(sv_2mortal(newSVpv(static_cast(*realisation).dump().c_str(), 0))); else diff --git a/tests/functional/ca/common.sh b/tests/functional/ca/common.sh index 10298d9d87d..dc8e650fd68 100644 --- a/tests/functional/ca/common.sh +++ b/tests/functional/ca/common.sh @@ -1,9 +1,6 @@ # shellcheck shell=bash source ../common.sh -# Need backend to support revamped CA -requireDaemonNewerThan "2.34.0pre20251217" - enableFeatures "ca-derivations" TODO_NixOS diff --git a/tests/functional/ca/substitute.sh b/tests/functional/ca/substitute.sh index 022bea5cfce..2f6ebcef5c1 100644 --- a/tests/functional/ca/substitute.sh +++ b/tests/functional/ca/substitute.sh @@ -49,14 +49,14 @@ fi clearStore nix build --file ../simple.nix -L --no-link --post-build-hook "$pushToStore" clearStore -rm -r "$REMOTE_STORE_DIR/build-trace-v2" +rm -r "$REMOTE_STORE_DIR/realisations" nix build --file ../simple.nix -L --no-link --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 # There's no easy way to check whether a realisation is present on the local # store − short of manually querying the db, but the build environment doesn't # have the sqlite binary − so we instead push things again, and check that the # realisations have correctly been pushed to the remote store nix copy --to "$REMOTE_STORE" --file ../simple.nix -if [[ -z "$(ls "$REMOTE_STORE_DIR/build-trace-v2")" ]]; then +if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then echo "Realisations not rebuilt" exit 1 fi @@ -71,5 +71,5 @@ buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 # Try rebuilding, but remove the realisations from the remote cache to force # using the cachecache clearStore -rm -r "$REMOTE_STORE_DIR"/build-trace-v2/* +rm "$REMOTE_STORE_DIR"/realisations/* buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0