diff --git a/CHANGELOG.md b/CHANGELOG.md index 85cca8cfc7..8f0bb31825 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,29 @@ If you want to disable this behavior, you can config it in dfx.json: } } +### feat: configurable custom wasm sections + +It's now possible to define custom wasm metadata sections and their visibility in dfx.json. + +At present, dfx can only add wasm metadata sections to canisters that are in wasm format. It cannot add metadata sections to compressed canisters. Since the frontend canister is now compressed, this means that at present it is not possible to add custom metadata sections to the frontend canister. + +dfx no longer adds `candid:service` metadata to canisters of type `"custom"` by default. If you want dfx to add your canister's candid definition to your custom canister, you can do so like this: + +``` + "my_canister_name": { + "type": "custom", + "candid": "main.did", + "wasm": "main.wasm", + "metadata": [ + { + "name": "candid:service" + } + ] + }, +``` + +This changelog entry doesn't go into all of the details of the possible configuration. For that, please see [concepts/canister-metadata](docs/concepts/canister-metadata.md) and the docs in the JSON schema. + ### fix: Valid canister-based env vars Hyphens are not valid in shell environment variables, but do occur in canister names such as `smiley-dapp`. This poses a problem for vars with names such as `CANISTER_ID_${CANISTER_NAME}`. With this change, hyphens are replaced with underscores in environment variables. The canister id of `smiley-dapp` will be available as `CANISTER_ID_smiley_dapp`. Other environment variables are unaffected. diff --git a/docs/concepts/canister-metadata.md b/docs/concepts/canister-metadata.md new file mode 100644 index 0000000000..955cf23154 --- /dev/null +++ b/docs/concepts/canister-metadata.md @@ -0,0 +1,121 @@ +# Canister Metadata + +## Overview + +Canisters can store custom metadata, which is available from the state tree at `/canister//metadata/`. + +You can configure this metadata in dfx.json, per canister, in the `metadata` array. + +Here is a simple example: + +```json +{ + "canisters": { + "app_backend": { + "main": "src/app_backend/main.mo", + "type": "motoko" + }, + "app_frontend": { + "dependencies": [ + "app_backend" + ], + "frontend": { + "entrypoint": "src/app_frontend/src/index.html" + }, + "source": [ + "src/app_frontend/assets", + "dist/app_frontend/" + ], + "type": "assets", + "metadata": [ + { + "name": "alternative-domains", + "visibility": "public", + "path": "src/app_frontend/metadata/alternative-domains.cbor" + } + ] + } + }, + "version": 1 +} +``` +## Fields + +The JSON schema also documents these fields. + +### name + +A string containing the name of the wasm section. + +### visibility + +A string containing either `private` or `public` (the default). + +Anyone can read the public metadata of a canister. + +Only a controller of the canister can read its private metadata. + +It is not possible to define metadata with the same name with both `private` and `public` visibility, unless they are for different networks. + +### networks + +An array of strings containing the names of the networks that this metadata applies to. + +If this field is absent, it applies to all networks. + +If this field is present as an empty array, it does not apply to any networks. + +If dfx.json contains more than one metadata entry with a given name, dfx will use the first entry that matches the current network and ignore any that follow. + +### path + +A string containing the path of a file containing the wasm section contents. + +## The candid:service metadata + +Dfx automatically adds `candid:service` metadata, with public visibility, for Rust and Motoko canisters. + +You can, however, override this behavior by defining a metadata entry with `"name": "candid:service"`. You can change the visibility or the contents. + +For Motoko canisters, if you specify a `path` for candid:service metadata (replacing the candid:service definition generated by `moc`), dfx will verify that the candid:service definition you provide is a valid subtype of the definition that `moc` generated. + +## A more complex example + +In this example, we change the visibility of the `candid:service` metadata on the ic and staging networks to private, but leave it public for the local network. + +```json +{ + "canisters": { + "app_backend": { + "main": "src/app_backend/main.mo", + "type": "motoko", + "metadata": [ + { + "name": "candid:service", + "networks": [ "ic", "staging" ], + "visibility": "private" + }, + { + "name": "candid:service", + "networks": [ "local" ], + "visibility": "public" + } + ] + }, + "app_frontend": { + "dependencies": [ + "app_backend" + ], + "frontend": { + "entrypoint": "src/app_frontend/src/index.html" + }, + "source": [ + "src/app_frontend/assets", + "dist/app_frontend/" + ], + "type": "assets" + } + }, + "version": 1 +} +``` diff --git a/docs/concepts/index.md b/docs/concepts/index.md new file mode 100644 index 0000000000..89314af246 --- /dev/null +++ b/docs/concepts/index.md @@ -0,0 +1,3 @@ +# DFX Concepts + +- [Canister metadata](./canister-metadata.md) diff --git a/docs/dfx-json-schema.json b/docs/dfx-json-schema.json index 0b51632f62..cbb456b4ec 100644 --- a/docs/dfx-json-schema.json +++ b/docs/dfx-json-schema.json @@ -115,6 +115,50 @@ } } }, + "CanisterMetadataSection": { + "title": "Canister Metadata Configuration", + "description": "Configures a custom metadata section for the canister wasm. dfx uses the first definition of a given name matching the current network, ignoring any of the same name that follow.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "title": "Name", + "description": "The name of the wasm section", + "type": "string" + }, + "networks": { + "title": "Networks", + "description": "Networks this section applies to. If this field is absent, then it applies to all networks. An empty array means this element will not apply to any network.", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "path": { + "title": "Path", + "description": "Path to file containing section contents. For sections with name=`candid:service`, this field is optional, and if not specified, dfx will use the canister's candid definition. If specified for a Motoko canister, the service defined in the specified path must be a valid subtype of the canister's actual candid service definition.", + "type": [ + "string", + "null" + ] + }, + "visibility": { + "title": "Visibility", + "default": "public", + "allOf": [ + { + "$ref": "#/definitions/MetadataVisibility" + } + ] + } + } + }, "ConfigCanistersCanister": { "title": "Canister Configuration", "description": "Configurations for a single canister.", @@ -290,6 +334,15 @@ "null" ] }, + "metadata": { + "title": "Metadata", + "description": "Defines metadata sections to set in the canister .wasm", + "default": [], + "type": "array", + "items": { + "$ref": "#/definitions/CanisterMetadataSection" + } + }, "post_install": { "title": "Post-Install Commands", "description": "One or more commands to run post canister installation.", @@ -689,6 +742,13 @@ } } }, + "MetadataVisibility": { + "type": "string", + "enum": [ + "public", + "private" + ] + }, "NetworkType": { "title": "Network Type", "description": "Type 'ephemeral' is used for networks that are regularly reset. Type 'persistent' is used for networks that last for a long time and where it is preferred that canister IDs get stored in source control.", diff --git a/e2e/assets/metadata/custom/custom_with_default_metadata.did b/e2e/assets/metadata/custom/custom_with_default_metadata.did new file mode 100644 index 0000000000..650072b171 --- /dev/null +++ b/e2e/assets/metadata/custom/custom_with_default_metadata.did @@ -0,0 +1,5 @@ +service : { + // custom_with_default_metadata + getCanisterId: () -> (principal) query; + amInitializer: () -> (bool) query; +} diff --git a/e2e/assets/metadata/custom/custom_with_private_candid_service_metadata.did b/e2e/assets/metadata/custom/custom_with_private_candid_service_metadata.did new file mode 100644 index 0000000000..8f8ac6cd2b --- /dev/null +++ b/e2e/assets/metadata/custom/custom_with_private_candid_service_metadata.did @@ -0,0 +1,5 @@ +service : { + // custom_with_private_candid_service_metadata + getCanisterId: () -> (principal) query; + amInitializer: () -> (bool) query; +} diff --git a/e2e/assets/metadata/custom/custom_with_standard_candid_service_metadata.did b/e2e/assets/metadata/custom/custom_with_standard_candid_service_metadata.did new file mode 100644 index 0000000000..7f00f9da8d --- /dev/null +++ b/e2e/assets/metadata/custom/custom_with_standard_candid_service_metadata.did @@ -0,0 +1,5 @@ +service : { + // custom_with_standard_candid_service_metadata + getCanisterId: () -> (principal) query; + amInitializer: () -> (bool) query; +} diff --git a/e2e/assets/metadata/custom/dfx.json b/e2e/assets/metadata/custom/dfx.json new file mode 100644 index 0000000000..e53f0284fd --- /dev/null +++ b/e2e/assets/metadata/custom/dfx.json @@ -0,0 +1,32 @@ +{ + "version": 1, + "canisters": { + "custom_with_default_metadata": { + "type": "custom", + "candid": "custom_with_default_metadata.did", + "wasm": "main.wasm", + "build": "echo anything" + }, + "custom_with_standard_candid_service_metadata": { + "type": "custom", + "candid": "custom_with_standard_candid_service_metadata.did", + "wasm": "main.wasm", + "metadata": [ + { + "name": "candid:service" + } + ] + }, + "custom_with_private_candid_service_metadata": { + "type": "custom", + "candid": "custom_with_private_candid_service_metadata.did", + "wasm": "main.wasm", + "metadata": [ + { + "name": "candid:service", + "visibility": "private" + } + ] + } + } +} diff --git a/e2e/assets/metadata/motoko/main.mo b/e2e/assets/metadata/motoko/main.mo new file mode 100644 index 0000000000..3883d2e7ec --- /dev/null +++ b/e2e/assets/metadata/motoko/main.mo @@ -0,0 +1,17 @@ +actor { + public query func greet(name : Text) : async Text { + return "Hello, " # name # "!"; + }; + + stable var a : Nat = 0; + public func inc_a() : async Nat { + a += 1; + return a; + }; + + stable var b : Int = 0; + public func inc_b() : async Int { + b += 1; + return b; + }; +}; diff --git a/e2e/assets/metadata/motoko/not_subtype_numbertype.did b/e2e/assets/metadata/motoko/not_subtype_numbertype.did new file mode 100644 index 0000000000..c05f7082a5 --- /dev/null +++ b/e2e/assets/metadata/motoko/not_subtype_numbertype.did @@ -0,0 +1,4 @@ +service : { + greet: (text) -> (text) query; + inc_b: () -> (nat); +} diff --git a/e2e/assets/metadata/motoko/not_subtype_rename.did b/e2e/assets/metadata/motoko/not_subtype_rename.did new file mode 100644 index 0000000000..0846113950 --- /dev/null +++ b/e2e/assets/metadata/motoko/not_subtype_rename.did @@ -0,0 +1,3 @@ +service : { + new_method: (text) -> (text) query; +} diff --git a/e2e/assets/metadata/motoko/patch.bash b/e2e/assets/metadata/motoko/patch.bash new file mode 100644 index 0000000000..da753f124a --- /dev/null +++ b/e2e/assets/metadata/motoko/patch.bash @@ -0,0 +1 @@ +jq '.canisters.e2e_project_backend.main="main.mo"' dfx.json | sponge dfx.json diff --git a/e2e/assets/metadata/motoko/valid_subtype.did b/e2e/assets/metadata/motoko/valid_subtype.did new file mode 100644 index 0000000000..a198260e83 --- /dev/null +++ b/e2e/assets/metadata/motoko/valid_subtype.did @@ -0,0 +1,4 @@ +service : { + greet: (text) -> (text) query; + inc_a: () -> (int); +} diff --git a/e2e/assets/prebuilt_custom_canister/dfx.json b/e2e/assets/prebuilt_custom_canister/dfx.json index c232735bd3..dea4e33388 100644 --- a/e2e/assets/prebuilt_custom_canister/dfx.json +++ b/e2e/assets/prebuilt_custom_canister/dfx.json @@ -23,6 +23,12 @@ "candid": "prebuilt_custom_blank_build.did", "wasm": "main.wasm", "build": [] + }, + "prebuilt_no_metadata_defined": { + "type": "custom", + "candid": "custom_with_build_step.did", + "wasm": "main.wasm", + "build": "echo just a build step" } }, "networks": { diff --git a/e2e/tests-dfx/build.bash b/e2e/tests-dfx/build.bash index fded1c79b8..8f9ecc8dca 100644 --- a/e2e/tests-dfx/build.bash +++ b/e2e/tests-dfx/build.bash @@ -257,27 +257,3 @@ teardown() { assert_command ls .dfx/actuallylocal/canisters/e2e_project_backend/ assert_command ls .dfx/actuallylocal/canisters/e2e_project_backend/e2e_project_backend.wasm } - -@test "does not add candid:service metadata for a custom canister if there are no build steps" { - install_asset prebuilt_custom_canister - install_asset wasm/identity - - dfx_start - dfx deploy - - # this canister has a build step, so dfx sets the candid metadata - dfx canister metadata custom_with_build_step candid:service >from_canister.txt - diff custom_with_build_step.did from_canister.txt - - # this canister doesn't have a build step, so dfx leaves the candid metadata as-is - dfx canister metadata prebuilt_custom_no_build candid:service >from_canister.txt - diff main.did from_canister.txt - - # this canister has a build step, but it is an empty string, so dfx leaves the candid:service metadata as-is - dfx canister metadata prebuilt_custom_blank_build candid:service >from_canister.txt - diff main.did from_canister.txt - - # this canister has a build step, but it is an empty array, so dfx leaves the candid:service metadata as-is - dfx canister metadata prebuilt_custom_empty_build candid:service >from_canister.txt - diff main.did from_canister.txt -} diff --git a/e2e/tests-dfx/metadata.bash b/e2e/tests-dfx/metadata.bash index e7ae2ed6e4..5090dae568 100644 --- a/e2e/tests-dfx/metadata.bash +++ b/e2e/tests-dfx/metadata.bash @@ -4,8 +4,6 @@ load ../utils/_ setup() { standard_setup - - dfx_new } teardown() { @@ -14,6 +12,166 @@ teardown() { standard_teardown } +@test "custom canister metadata rules" { + install_asset metadata/custom + install_asset wasm/identity + + dfx_start + dfx deploy + + echo "leaves existing metadata in a custom canister with no metadata settings" + dfx canister metadata --identity anonymous custom_with_default_metadata candid:service >metadata.txt + diff main.did metadata.txt + + echo "adds candid:service public metadata from candid field if a metadata entry exists" + dfx canister metadata --identity anonymous custom_with_standard_candid_service_metadata candid:service >metadata.txt + diff custom_with_standard_candid_service_metadata.did metadata.txt + + echo "adds candid:service metadata from candid field with private visibility per metadata entry" + assert_command_fail dfx canister metadata --identity anonymous custom_with_private_candid_service_metadata candid:service >metadata.txt + dfx canister metadata custom_with_private_candid_service_metadata candid:service >metadata.txt + diff custom_with_private_candid_service_metadata.did metadata.txt +} + +@test "rust canister metadata rules" { + rustup default stable + rustup target add wasm32-unknown-unknown + + dfx_new_rust + + dfx_start + dfx deploy + + echo "adds public candid:service metadata to a default rust canister" + dfx canister metadata --identity anonymous e2e_project_backend candid:service >metadata.txt + diff src/e2e_project_backend/e2e_project_backend.did metadata.txt + + echo "adds private candid:service metadata if so configured" + jq 'del(.canisters.e2e_project_backend.metadata)' dfx.json | sponge dfx.json + jq '.canisters.e2e_project_backend.metadata[0].name="candid:service"|.canisters.e2e_project_backend.metadata[0].visibility="private"' dfx.json | sponge dfx.json + dfx deploy + assert_command_fail dfx canister metadata --identity anonymous e2e_project_backend candid:service + dfx canister metadata e2e_project_backend candid:service >metadata.txt + diff src/e2e_project_backend/e2e_project_backend.did metadata.txt +} + +@test "motoko canister metadata rules" { + dfx_new + dfx_start + install_asset metadata/motoko + dfx canister create --all + + echo "permits specification of a replacement candid definition, if it is a valid subtype" + jq 'del(.canisters.e2e_project_backend.metadata)' dfx.json | sponge dfx.json + assert_command dfx build + find . -name '*.did' + jq '.canisters.e2e_project_backend.metadata[0].name="candid:service"|.canisters.e2e_project_backend.metadata[0].path="valid_subtype.did"' dfx.json | sponge dfx.json + dfx build + + echo "reports an error if a specified candid:service metadata is not a valid subtype for the canister" + jq 'del(.canisters.e2e_project_backend.metadata)' dfx.json | sponge dfx.json + jq '.canisters.e2e_project_backend.metadata[0].name="candid:service"|.canisters.e2e_project_backend.metadata[0].path="not_subtype_rename.did"' dfx.json | sponge dfx.json + assert_command_fail dfx build + assert_match "Method new_method is only in the expected type" + + echo "reports an error if a specified candid:service metadata is not a valid subtype for the canister" + jq 'del(.canisters.e2e_project_backend.metadata)' dfx.json | sponge dfx.json + jq '.canisters.e2e_project_backend.metadata[0].name="candid:service"|.canisters.e2e_project_backend.metadata[0].path="not_subtype_numbertype.did"' dfx.json | sponge dfx.json + assert_command_fail dfx build + assert_match "int is not a subtype of nat" + + + echo "adds private candid:service metadata if so configured" + jq 'del(.canisters.e2e_project_backend.metadata)' dfx.json | sponge dfx.json + jq '.canisters.e2e_project_backend.metadata[0].name="candid:service"|.canisters.e2e_project_backend.metadata[0].visibility="private"' dfx.json | sponge dfx.json + dfx deploy + assert_command_fail dfx canister metadata --identity anonymous e2e_project_backend candid:service + dfx canister metadata e2e_project_backend candid:service >metadata.txt + diff .dfx/local/canisters/e2e_project_backend/e2e_project_backend.did metadata.txt + + + echo "adds public candid:service metadata to a default motoko canister" + jq 'del(.canisters.e2e_project_backend.metadata)' dfx.json | sponge dfx.json + dfx deploy + dfx canister metadata --identity anonymous e2e_project_backend candid:service >metadata.txt + diff .dfx/local/canisters/e2e_project_backend/e2e_project_backend.did metadata.txt +} + +@test "adds arbitrary metadata to a motoko canister" { + dfx_new + dfx_start + install_asset metadata/motoko + dfx canister create --all + + echo "adds public arbitrary metadata to a default motoko canister" + jq 'del(.canisters.e2e_project_backend.metadata)' dfx.json | sponge dfx.json + jq '.canisters.e2e_project_backend.metadata[0].name="arbitrary"|.canisters.e2e_project_backend.metadata[0].path="arbitrary-metadata.txt"' dfx.json | sponge dfx.json + echo "can be anything" >arbitrary-metadata.txt + dfx deploy + dfx canister metadata --identity anonymous e2e_project_backend arbitrary >from-canister.txt + diff arbitrary-metadata.txt from-canister.txt + + # with private visibility + jq '.canisters.e2e_project_backend.metadata[0].visibility="private"' dfx.json | sponge dfx.json + dfx deploy + assert_command_fail dfx canister metadata --identity anonymous e2e_project_backend arbitrary + dfx canister metadata e2e_project_backend arbitrary >from-canister.txt + diff arbitrary-metadata.txt from-canister.txt +} + +@test "uses the first metadata definition for name and network" { + dfx_new + dfx_start + install_asset metadata/motoko + dfx canister create --all + + jq 'del(.canisters.e2e_project_backend.metadata)' dfx.json | sponge dfx.json + jq '.canisters.e2e_project_backend.metadata[0].name="multiple"|.canisters.e2e_project_backend.metadata[0].path="empty-networks-matches-nothing.txt"|.canisters.e2e_project_backend.metadata[0].networks=[]' dfx.json | sponge dfx.json + jq '.canisters.e2e_project_backend.metadata[1].name="multiple"|.canisters.e2e_project_backend.metadata[1].path="different-network-no-match.txt"|.canisters.e2e_project_backend.metadata[1].networks=["ic"]' dfx.json | sponge dfx.json + jq '.canisters.e2e_project_backend.metadata[2].name="multiple"|.canisters.e2e_project_backend.metadata[2].path="first-match-chosen.txt"' dfx.json | sponge dfx.json + jq '.canisters.e2e_project_backend.metadata[3].name="multiple"|.canisters.e2e_project_backend.metadata[3].path="earlier-match-ignored.txt"' dfx.json | sponge dfx.json + echo "dfx will install this file" >first-match-chosen.txt + dfx deploy + dfx canister metadata --identity anonymous e2e_project_backend multiple >from-canister.txt + diff first-match-chosen.txt from-canister.txt +} + +@test "warns if cannot add metadata to a compressed canister" { + # the frontend canister is compressed + dfx_new_frontend + dfx_start + + jq 'del(.canisters.e2e_project_frontend.metadata)' dfx.json | sponge dfx.json + jq '.canisters.e2e_project_frontend.metadata[0].name="arbitrary"|.canisters.e2e_project_frontend.metadata[0].path="arbitrary-metadata.txt"' dfx.json | sponge dfx.json + echo "can be anything" >arbitrary-metadata.txt + jq . dfx.json + assert_command dfx deploy + assert_match "cannot apply metadata because the canister is not wasm format" +} + +@test "existence of build steps do not control custom canister metadata" { + install_asset prebuilt_custom_canister + install_asset wasm/identity + + dfx_start + dfx deploy + + # this canister has a build step, which doesn't matter: dfx leaves the candid metadata + dfx canister metadata custom_with_build_step candid:service >from_canister.txt + diff main.did from_canister.txt + + # this canister doesn't have a build step, so dfx leaves the candid metadata as-is + dfx canister metadata prebuilt_custom_no_build candid:service >from_canister.txt + diff main.did from_canister.txt + + # this canister has a build step, but it is an empty string, so dfx leaves the candid:service metadata as-is + dfx canister metadata prebuilt_custom_blank_build candid:service >from_canister.txt + diff main.did from_canister.txt + + # this canister has a build step, but it is an empty array, so dfx leaves the candid:service metadata as-is + dfx canister metadata prebuilt_custom_empty_build candid:service >from_canister.txt + diff main.did from_canister.txt +} @test "can read canister metadata from replica" { dfx_new hello diff --git a/src/dfx/src/config/dfinity.rs b/src/dfx/src/config/dfinity.rs index 1e30908633..b0ec398bd8 100644 --- a/src/dfx/src/config/dfinity.rs +++ b/src/dfx/src/config/dfinity.rs @@ -6,6 +6,7 @@ use crate::lib::error::{BuildError, DfxError, DfxResult}; use crate::util::{project_dirs, PossiblyStr, SerdeVec}; use crate::{error_invalid_argument, error_invalid_config, error_invalid_data}; +use crate::config::dfinity::MetadataVisibility::Public; use anyhow::{anyhow, Context}; use byte_unit::Byte; use candid::Principal; @@ -14,7 +15,7 @@ use schemars::JsonSchema; use serde::de::{Error as _, MapAccess, Visitor}; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::Value; -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::default::Default; use std::fmt; use std::net::{IpAddr, Ipv4Addr, SocketAddr, ToSocketAddrs}; @@ -53,6 +54,59 @@ pub struct ConfigCanistersCanisterRemote { pub id: BTreeMap, } +#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum MetadataVisibility { + /// Anyone can query the metadata + Public, + + /// Only the controllers of the canister can query the metadata. + Private, +} + +impl Default for MetadataVisibility { + fn default() -> Self { + Public + } +} + +/// # Canister Metadata Configuration +/// Configures a custom metadata section for the canister wasm. +/// dfx uses the first definition of a given name matching the current network, ignoring any of the same name that follow. +#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] +pub struct CanisterMetadataSection { + /// # Name + /// The name of the wasm section + pub name: String, + + /// # Visibility + #[serde(default)] + pub visibility: MetadataVisibility, + + /// # Networks + /// Networks this section applies to. + /// If this field is absent, then it applies to all networks. + /// An empty array means this element will not apply to any network. + pub networks: Option>, + + /// # Path + /// Path to file containing section contents. + /// For sections with name=`candid:service`, this field is optional, and if not specified, dfx will use + /// the canister's candid definition. + /// If specified for a Motoko canister, the service defined in the specified path must be a valid subtype of the canister's + /// actual candid service definition. + pub path: Option, +} + +impl CanisterMetadataSection { + pub fn applies_to_network(&self, network: &str) -> bool { + self.networks + .as_ref() + .map(|networks| networks.contains(network)) + .unwrap_or(true) + } +} + pub const DEFAULT_SHARED_LOCAL_BIND: &str = "127.0.0.1:4943"; // hex for "IC" pub const DEFAULT_PROJECT_LOCAL_BIND: &str = "127.0.0.1:8000"; pub const DEFAULT_IC_GATEWAY: &str = "https://ic0.app"; @@ -113,6 +167,11 @@ pub struct ConfigCanistersCanister { /// Default is true. #[serde(default = "default_as_true")] pub shrink: bool, + + /// # Metadata + /// Defines metadata sections to set in the canister .wasm + #[serde(default)] + pub metadata: Vec, } #[derive(Clone, Debug, Serialize, JsonSchema)] diff --git a/src/dfx/src/lib/builders/assets.rs b/src/dfx/src/lib/builders/assets.rs index c079e1868e..771b4ff3d8 100644 --- a/src/dfx/src/lib/builders/assets.rs +++ b/src/dfx/src/lib/builders/assets.rs @@ -105,7 +105,6 @@ impl CanisterBuilder for AssetsBuilder { canister_id: info.get_canister_id().expect("Could not find canister ID."), wasm: WasmBuildOutput::File(wasm_path), idl: IdlBuildOutput::File(idl_path), - add_candid_service_metadata: false, }) } diff --git a/src/dfx/src/lib/builders/custom.rs b/src/dfx/src/lib/builders/custom.rs index 96e69c6cb8..7d3e9524e7 100644 --- a/src/dfx/src/lib/builders/custom.rs +++ b/src/dfx/src/lib/builders/custom.rs @@ -7,6 +7,7 @@ use crate::lib::environment::Environment; use crate::lib::error::{BuildError, DfxError, DfxResult}; use crate::lib::models::canister::CanisterPool; +use crate::lib::wasm::file::is_wasm_format; use anyhow::{anyhow, bail, Context}; use bytes::Bytes; use candid::Principal as CanisterId; @@ -18,8 +19,7 @@ use reqwest::{Client, StatusCode}; use slog::info; use slog::Logger; use std::fs; -use std::fs::{create_dir_all, File}; -use std::io::Read; +use std::fs::create_dir_all; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::time::Duration; @@ -122,7 +122,6 @@ impl CanisterBuilder for CustomBuilder { let canister_id = info.get_canister_id().unwrap(); let vars = super::environment_variables(info, &config.network_name, pool, &dependencies); - let mut add_candid_service_metadata = false; for command in build { info!( self.logger, @@ -136,22 +135,13 @@ impl CanisterBuilder for CustomBuilder { .with_context(|| format!("Cannot parse command '{}'.", command))?; // No commands, noop. if !args.is_empty() { - add_candid_service_metadata = true; run_command(args, &vars, info.get_workspace_root()) .with_context(|| format!("Failed to run {}.", command))?; } } - let mut file = - File::open(&wasm).with_context(|| format!("Failed to open {}", wasm.display()))?; - let mut header = [0; 4]; - file.read_exact(&mut header)?; - if header != *b"\0asm" { - add_candid_service_metadata = false; - } - // Custom canister may have WASM gzipped - if info.get_shrink() && header == *b"\0asm" { + if info.get_shrink() && is_wasm_format(&wasm)? { info!(self.logger, "Shrink WASM module size."); super::shrink_wasm(&wasm)?; } @@ -160,7 +150,6 @@ impl CanisterBuilder for CustomBuilder { canister_id, wasm: WasmBuildOutput::File(wasm), idl: IdlBuildOutput::File(candid), - add_candid_service_metadata, }) } diff --git a/src/dfx/src/lib/builders/mod.rs b/src/dfx/src/lib/builders/mod.rs index f955d28c9b..2aac2a5455 100644 --- a/src/dfx/src/lib/builders/mod.rs +++ b/src/dfx/src/lib/builders/mod.rs @@ -43,7 +43,6 @@ pub struct BuildOutput { pub canister_id: CanisterId, pub wasm: WasmBuildOutput, pub idl: IdlBuildOutput, - pub add_candid_service_metadata: bool, } /// A stateless canister builder. This is meant to not keep any state and be passed everything. diff --git a/src/dfx/src/lib/builders/motoko.rs b/src/dfx/src/lib/builders/motoko.rs index 2b250c02f0..b603e49bdf 100644 --- a/src/dfx/src/lib/builders/motoko.rs +++ b/src/dfx/src/lib/builders/motoko.rs @@ -1,5 +1,5 @@ use crate::config::cache::Cache; -use crate::config::dfinity::Profile; +use crate::config::dfinity::{MetadataVisibility, Profile}; use crate::lib::builders::{ BuildConfig, BuildOutput, CanisterBuilder, IdlBuildOutput, WasmBuildOutput, }; @@ -145,6 +145,11 @@ impl CanisterBuilder for MotokoBuilder { None => package_arguments, }; + let candid_service_metadata_visibility = canister_info + .get_metadata(CANDID_SERVICE) + .map(|m| m.visibility) + .unwrap_or(MetadataVisibility::Public); + // Generate wasm let params = MotokoParams { build_target: match profile { @@ -154,6 +159,7 @@ impl CanisterBuilder for MotokoBuilder { suppress_warning: false, input: input_path, package_arguments: &moc_arguments, + candid_service_metadata_visibility, output: output_wasm_path, idl_path: idl_dir_path, idl_map: &id_map, @@ -170,7 +176,6 @@ impl CanisterBuilder for MotokoBuilder { .expect("Could not find canister ID."), wasm: WasmBuildOutput::File(motoko_info.get_output_wasm_path().to_path_buf()), idl: IdlBuildOutput::File(motoko_info.get_output_idl_path().to_path_buf()), - add_candid_service_metadata: false, }) } @@ -224,6 +229,7 @@ struct MotokoParams<'a> { idl_path: &'a Path, idl_map: &'a CanisterIdMap, package_arguments: &'a PackageArguments, + candid_service_metadata_visibility: MetadataVisibility, output: &'a Path, input: &'a Path, // The following fields are control flags for dfx and will not be used by self.to_args() @@ -239,8 +245,10 @@ impl MotokoParams<'_> { BuildTarget::Debug => cmd.args(&["-c", "--debug"]), }; cmd.arg("--idl").arg("--stable-types"); - // TODO add a flag in dfx.json to opt-out public interface - cmd.arg("--public-metadata").arg(CANDID_SERVICE); + if self.candid_service_metadata_visibility == MetadataVisibility::Public { + // moc defaults to private metadata, if this argument is not present. + cmd.arg("--public-metadata").arg(CANDID_SERVICE); + } if !self.idl_map.is_empty() { cmd.arg("--actor-idl").arg(self.idl_path); for (name, canister_id) in self.idl_map.iter() { diff --git a/src/dfx/src/lib/builders/rust.rs b/src/dfx/src/lib/builders/rust.rs index f4f5b0cef6..284b267fd6 100644 --- a/src/dfx/src/lib/builders/rust.rs +++ b/src/dfx/src/lib/builders/rust.rs @@ -105,7 +105,6 @@ impl CanisterBuilder for RustBuilder { canister_id, wasm: WasmBuildOutput::File(rust_info.get_output_wasm_path().to_path_buf()), idl: IdlBuildOutput::File(rust_info.get_output_idl_path().to_path_buf()), - add_candid_service_metadata: true, }) } diff --git a/src/dfx/src/lib/canister_info.rs b/src/dfx/src/lib/canister_info.rs index e18259ce0a..c267be53a1 100644 --- a/src/dfx/src/lib/canister_info.rs +++ b/src/dfx/src/lib/canister_info.rs @@ -1,5 +1,7 @@ #![allow(dead_code)] -use crate::config::dfinity::{CanisterDeclarationsConfig, CanisterTypeProperties, Config}; +use crate::config::dfinity::{ + CanisterDeclarationsConfig, CanisterMetadataSection, CanisterTypeProperties, Config, +}; use crate::lib::canister_info::assets::AssetsCanisterInfo; use crate::lib::canister_info::custom::CustomCanisterInfo; use crate::lib::canister_info::motoko::MotokoCanisterInfo; @@ -7,6 +9,7 @@ use crate::lib::error::DfxResult; use crate::lib::provider::get_network_context; use crate::util; +use crate::lib::metadata::config::CanisterMetadataConfig; use anyhow::{anyhow, Context}; use candid::Principal as CanisterId; use candid::Principal; @@ -49,6 +52,7 @@ pub struct CanisterInfo { post_install: Vec, main: Option, shrink: bool, + metadata: CanisterMetadataConfig, } impl CanisterInfo { @@ -111,6 +115,8 @@ impl CanisterInfo { }; let post_install = canister_config.post_install.clone().into_vec(); + let metadata = + CanisterMetadataConfig::new(&type_specific, &canister_config.metadata, &network_name); let canister_info = CanisterInfo { name: name.to_string(), @@ -127,6 +133,7 @@ impl CanisterInfo { post_install, main: canister_config.main.clone(), shrink: canister_config.shrink, + metadata, }; Ok(canister_info) @@ -254,4 +261,11 @@ impl CanisterInfo { pub fn is_assets(&self) -> bool { matches!(self.type_specific, CanisterTypeProperties::Assets { .. }) } + + pub fn get_metadata(&self, name: &str) -> Option<&CanisterMetadataSection> { + self.metadata.get(name) + } + pub fn metadata(&self) -> &CanisterMetadataConfig { + &self.metadata + } } diff --git a/src/dfx/src/lib/metadata/config.rs b/src/dfx/src/lib/metadata/config.rs new file mode 100644 index 0000000000..dcc6dd3344 --- /dev/null +++ b/src/dfx/src/lib/metadata/config.rs @@ -0,0 +1,47 @@ +use crate::config::dfinity::{CanisterMetadataSection, CanisterTypeProperties}; +use crate::lib::metadata::names::CANDID_SERVICE; + +use crate::config::dfinity::MetadataVisibility::Public; +use std::collections::BTreeMap; + +#[derive(Debug)] +pub struct CanisterMetadataConfig { + pub sections: BTreeMap, +} + +impl CanisterMetadataConfig { + pub fn new( + type_properties: &CanisterTypeProperties, + sections: &Vec, + network: &str, + ) -> Self { + let mut map = BTreeMap::new(); + for section in sections { + if section.applies_to_network(network) && !map.contains_key(§ion.name) { + map.insert(section.name.clone(), section.clone()); + } + } + + let default_candid_service = matches!( + type_properties, + CanisterTypeProperties::Rust { .. } | CanisterTypeProperties::Motoko + ); + if default_candid_service && !map.contains_key(CANDID_SERVICE) { + map.insert( + CANDID_SERVICE.to_string(), + CanisterMetadataSection { + name: CANDID_SERVICE.to_string(), + visibility: Public, + networks: None, + path: None, + }, + ); + } + + CanisterMetadataConfig { sections: map } + } + + pub fn get(&self, name: &str) -> Option<&CanisterMetadataSection> { + self.sections.get(name) + } +} diff --git a/src/dfx/src/lib/metadata/mod.rs b/src/dfx/src/lib/metadata/mod.rs index d2f54b13af..287eefc1f9 100644 --- a/src/dfx/src/lib/metadata/mod.rs +++ b/src/dfx/src/lib/metadata/mod.rs @@ -1 +1,2 @@ +pub mod config; pub mod names; diff --git a/src/dfx/src/lib/models/canister.rs b/src/dfx/src/lib/models/canister.rs index 946a9382a2..c534d707f3 100644 --- a/src/dfx/src/lib/models/canister.rs +++ b/src/dfx/src/lib/models/canister.rs @@ -1,4 +1,4 @@ -use crate::config::dfinity::Config; +use crate::config::dfinity::{Config, MetadataVisibility}; use crate::lib::builders::{ custom_download, BuildConfig, BuildOutput, BuilderPool, CanisterBuilder, IdlBuildOutput, WasmBuildOutput, @@ -6,19 +6,21 @@ use crate::lib::builders::{ use crate::lib::canister_info::CanisterInfo; use crate::lib::environment::Environment; use crate::lib::error::{BuildError, DfxError, DfxResult}; +use crate::lib::metadata::names::CANDID_SERVICE; use crate::lib::models::canister_id_store::CanisterIdStore; +use crate::lib::wasm::file::is_wasm_format; use crate::util::{assets, check_candid_file}; -use crate::lib::wasm::metadata::add_candid_service_metadata; use anyhow::{anyhow, bail, Context}; use candid::Principal as CanisterId; use fn_error_context::context; +use ic_wasm::metadata::{add_metadata, remove_metadata, Kind}; use itertools::Itertools; use petgraph::graph::{DiGraph, NodeIndex}; use rand::{thread_rng, RngCore}; use slog::{error, info, trace, warn, Logger}; use std::cell::RefCell; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use std::convert::TryFrom; use std::io::Read; use std::path::Path; @@ -98,6 +100,82 @@ impl Canister { pub fn generate(&self, pool: &CanisterPool, build_config: &BuildConfig) -> DfxResult { self.builder.generate(pool, &self.info, build_config) } + + #[context("Failed while trying to apply metadata for canister '{}'.", self.info.get_name())] + pub(crate) fn apply_metadata(&self, logger: &Logger) -> DfxResult { + let metadata = self.info.metadata(); + if metadata.sections.is_empty() { + return Ok(()); + } + + let wasm_path = self.info.get_build_wasm_path(); + let idl_path = self.info.get_build_idl_path(); + + if !is_wasm_format(&wasm_path)? { + warn!( + logger, + "Canister '{}': cannot apply metadata because the canister is not wasm format", + self.info.get_name() + ); + return Ok(()); + } + + let mut m = std::fs::read(&wasm_path) + .with_context(|| format!("Failed to read wasm at {}", wasm_path.display()))?; + + for (name, section) in &metadata.sections { + if section.name == CANDID_SERVICE && self.info.is_motoko() { + if let Some(specified_path) = §ion.path { + check_valid_subtype(&idl_path, specified_path)? + } else { + // Motoko compiler handles this + continue; + } + } + + let metadata_path = match section.path.as_ref() { + Some(path) => path, + None if section.name == CANDID_SERVICE => &idl_path, + _ => bail!( + "Metadata section must specify a path. section: {:?}", + §ion + ), + }; + + let data = std::fs::read(&metadata_path) + .with_context(|| format!("Failed to read {}", metadata_path.to_string_lossy()))?; + + let visibility = match section.visibility { + MetadataVisibility::Public => Kind::Public, + MetadataVisibility::Private => Kind::Private, + }; + + // if the metadata already exists in the wasm with a different visibility, + // then we have to remove it + m = remove_metadata(&m, name)?; + + m = add_metadata(&m, visibility, name, data)?; + } + + std::fs::write(&wasm_path, &m) + .with_context(|| format!("Could not write WASM to {:?}", wasm_path)) + } +} + +#[context("{} is not a valid subtype of {}", specified_idl_path.display(), compiled_idl_path.display())] +fn check_valid_subtype(compiled_idl_path: &Path, specified_idl_path: &Path) -> DfxResult { + let (mut env, opt_specified) = + check_candid_file(specified_idl_path).context("Checking specified candid file.")?; + let specified_type = + opt_specified.expect("Specified did file should contain some service interface"); + let (env2, opt_compiled) = + check_candid_file(compiled_idl_path).context("Checking compiled candid file.")?; + let compiled_type = + opt_compiled.expect("Compiled did file should contain some service interface"); + let mut gamma = HashSet::new(); + let specified_type = env.merge_type(env2, specified_type); + candid::types::subtype::subtype(&mut gamma, &env, &compiled_type, &specified_type)?; + Ok(()) } /// A canister pool is a list of canisters. @@ -372,9 +450,8 @@ impl CanisterPool { ) })?; } - if build_output.add_candid_service_metadata { - add_candid_service_metadata(&wasm_file_path, &idl_file_path)?; - } + + canister.apply_metadata(self.get_logger())?; let canister_id = canister.canister_id(); diff --git a/src/dfx/src/lib/wasm/file.rs b/src/dfx/src/lib/wasm/file.rs new file mode 100644 index 0000000000..89519d4e84 --- /dev/null +++ b/src/dfx/src/lib/wasm/file.rs @@ -0,0 +1,13 @@ +use crate::lib::error::DfxResult; +use anyhow::Context; +use std::fs::File; +use std::io::Read; +use std::path::Path; + +pub fn is_wasm_format(path: &Path) -> DfxResult { + let mut file = + File::open(&path).with_context(|| format!("Failed to open {}", path.display()))?; + let mut header = [0; 4]; + file.read_exact(&mut header)?; + Ok(header == *b"\0asm") +} diff --git a/src/dfx/src/lib/wasm/metadata.rs b/src/dfx/src/lib/wasm/metadata.rs deleted file mode 100644 index c24c5caa52..0000000000 --- a/src/dfx/src/lib/wasm/metadata.rs +++ /dev/null @@ -1,19 +0,0 @@ -use crate::lib::error::DfxResult; -use crate::lib::metadata::names::CANDID_SERVICE; - -use anyhow::Context; -use fn_error_context::context; -use ic_wasm::metadata::{add_metadata, Kind}; -use std::path::Path; - -#[context("Failed to add candid service metadata from {} to {}.", idl_path.to_string_lossy(), wasm_path.to_string_lossy())] -pub fn add_candid_service_metadata(wasm_path: &Path, idl_path: &Path) -> DfxResult { - let wasm = std::fs::read(wasm_path).context("Could not read the WASM module.")?; - let idl = std::fs::read(&idl_path) - .with_context(|| format!("Failed to read {}", idl_path.to_string_lossy()))?; - let processed_wasm = add_metadata(&wasm, Kind::Public, CANDID_SERVICE, idl) - .context("Could not add metadata to the WASM module.")?; - std::fs::write(wasm_path, &processed_wasm) - .with_context(|| format!("Could not write WASM to {:?}", wasm_path))?; - Ok(()) -} diff --git a/src/dfx/src/lib/wasm/mod.rs b/src/dfx/src/lib/wasm/mod.rs index de5643fda5..2e172cd0fb 100644 --- a/src/dfx/src/lib/wasm/mod.rs +++ b/src/dfx/src/lib/wasm/mod.rs @@ -1 +1 @@ -pub mod metadata; +pub mod file;