datastore: metrics datastore configuration optional and off by default

config/init.go

@@ -141,6 +141,15 @@ func DefaultDatastoreConfig() Datastore {
 }
 
 func pebbleSpec() map[string]interface{} {
+	return map[string]interface{}{
+		"type":               "pebbleds",
+		"prefix":             "pebble.datastore",
+		"path":               "pebbleds",
+		"formatMajorVersion": int(pebble.FormatNewest),
+	}
+}
+
+func pebbleSpecMeasure() map[string]interface{} {
 	return map[string]interface{}{
 		"type":   "measure",
 		"prefix": "pebble.datastore",
@@ -153,6 +162,16 @@ func pebbleSpec() map[string]interface{} {
 }
 
 func badgerSpec() map[string]interface{} {
+	return map[string]interface{}{
+		"type":       "badgerds",
+		"prefix":     "badger.datastore",
+		"path":       "badgerds",
+		"syncWrites": false,
+		"truncate":   true,
+	}
+}
+
+func badgerSpecMeasure() map[string]interface{} {
 	return map[string]interface{}{
 		"type":   "measure",
 		"prefix": "badger.datastore",
@@ -166,6 +185,29 @@ func badgerSpec() map[string]interface{} {
 }
 
 func flatfsSpec() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "mount",
+		"mounts": []interface{}{
+			map[string]interface{}{
+				"mountpoint": "/blocks",
+				"type":       "flatfs",
+				"prefix":     "flatfs.datastore",
+				"path":       "blocks",
+				"sync":       false,
+				"shardFunc":  "/repo/flatfs/shard/v1/next-to-last/2",
+			},
+			map[string]interface{}{
+				"mountpoint":  "/",
+				"type":        "levelds",
+				"prefix":      "leveldb.datastore",
+				"path":        "datastore",
+				"compression": "none",
+			},
+		},
+	}
+}
+
+func flatfsSpecMeasure() map[string]interface{} {
 	return map[string]interface{}{
 		"type": "mount",
 		"mounts": []interface{}{

config/profile.go

@@ -150,6 +150,20 @@ NOTE: This profile may only be applied when first initializing node at IPFS_PATH
 			return nil
 		},
 	},
+	"flatfs-measure": {
+		Description: `Configures the node to use the flatfs datastore with metrics tracking wrapper.
+Additional '*_datastore_*' metrics will be exposed on /debug/metrics/prometheus
+
+NOTE: This profile may only be applied when first initializing node at IPFS_PATH
+      via 'ipfs init --profile flatfs-measure'
+`,
+
+		InitOnly: true,
+		Transform: func(c *Config) error {
+			c.Datastore.Spec = flatfsSpecMeasure()
+			return nil
+		},
+	},
 	"pebbleds": {
 		Description: `Configures the node to use the pebble high-performance datastore.
 
@@ -176,6 +190,20 @@ NOTE: This profile may only be applied when first initializing node at IPFS_PATH
 			return nil
 		},
 	},
+	"pebbleds-measure": {
+		Description: `Configures the node to use the pebble datastore with metrics tracking wrapper.
+Additional '*_datastore_*' metrics will be exposed on /debug/metrics/prometheus
+
+NOTE: This profile may only be applied when first initializing node at IPFS_PATH
+      via 'ipfs init --profile pebbleds-measure'
+`,
+
+		InitOnly: true,
+		Transform: func(c *Config) error {
+			c.Datastore.Spec = pebbleSpecMeasure()
+			return nil
+		},
+	},
 	"badgerds": {
 		Description: `Configures the node to use the legacy badgerv1 datastore.
 
@@ -205,6 +233,20 @@ NOTE: This profile may only be applied when first initializing node at IPFS_PATH
 			return nil
 		},
 	},
+	"badgerds-measure": {
+		Description: `Configures the node to use the legacy badgerv1 datastore with metrics wrapper.
+Additional '*_datastore_*' metrics will be exposed on /debug/metrics/prometheus
+
+NOTE: This profile may only be applied when first initializing node at IPFS_PATH
+      via 'ipfs init --profile badgerds-measure'
+`,
+
+		InitOnly: true,
+		Transform: func(c *Config) error {
+			c.Datastore.Spec = badgerSpecMeasure()
+			return nil
+		},
+	},
 	"lowpower": {
 		Description: `Reduces daemon overhead on the system. May affect node
 functionality - performance of content discovery and data

docs/changelogs/v0.35.md

@@ -16,7 +16,8 @@ This release  was brought to you by the [Shipyard](http://ipshipyard.com/) team.
   - [Enhanced DAG-Shaping Controls for `ipfs add`](#enhanced-dag-shaping-controls-for-ipfs-add)
     - [New `ipfs add` Options](#new-ipfs-add-options)
     - [Persistent `Import.*` Configuration](#persistent-import-configuration)
-    - [Updated Configuration Profiles](#updated-configuration-profiles)
+    - [Updated `Import` Profiles](#updated-import-profiles)
+  - [`Datastore` Metrics Now Opt-In](#datastore-metrics-now-opt-in)
   - [Optimized, dedicated queue for providing fresh CIDs](#optimized-dedicated-queue-for-providing-fresh-cids)
     - [Deprecated `ipfs stats provider`](#deprecated-ipfs-stats-provider)
   - [Pebble Database Format Config](#pebble-database-format-config)
@@ -74,7 +75,7 @@ You can set default values for these options using the following configuration s
 - [`Import.UnixFSHAMTDirectoryMaxFanout`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfshamtdirectorymaxfanout)
 - [`Import.UnixFSHAMTDirectorySizeThreshold`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfshamtdirectorysizethreshold)
 
-##### Updated Configuration Profiles
+##### Updated `Import` Profiles
 
 The release updated configuration [profiles](https://github.com/ipfs/kubo/blob/master/docs/config.md#profiles) to incorporate these new `Import.*` settings:
 - Updated Profile: `test-cid-v1` now includes current defaults as explicit `Import.UnixFSFileMaxLinks=174`, `Import.UnixFSDirectoryMaxLinks=0`, `Import.UnixFSHAMTDirectoryMaxFanout=256` and `Import.UnixFSHAMTDirectorySizeThreshold=256KiB`
@@ -84,6 +85,15 @@ The release updated configuration [profiles](https://github.com/ipfs/kubo/blob/m
 > [!TIP]
 > Apply one of CIDv1 test [profiles](https://github.com/ipfs/kubo/blob/master/docs/config.md#profiles) with `ipfs config profile apply test-cid-v1[-wide]`.
 
+#### `Datastore` Metrics Now Opt-In
+
+To reduce overhead in the default configuration, datastore metrics are no longer enabled by default when initializing a Kubo repository with `ipfs init`.
+Metrics prefixed with `<dsname>_datastore` (e.g., `flatfs_datastore_...`, `leveldb_datastore_...`) are not exposed unless explicitly enabled. For a complete list of affected default metrics, refer to [`prometheus_metrics_added_by_measure_profile`](https://github.com/ipfs/kubo/blob/master/test/sharness/t0119-prometheus-data/prometheus_metrics_added_by_measure_profile).
+
+Convenience opt-in [profiles](https://github.com/ipfs/kubo/blob/master/docs/config.md#profiles) can be enabled at initialization time with `ipfs init --profile`: `flatfs-measure`, `pebbleds-measure`, `badgerds-measure`
+
+It is also possible to manually add the `measure` wrapper. See examples in [`Datastore.Spec`](https://github.com/ipfs/kubo/blob/master/docs/config.md#datastorespec) documentation.
+
 #### Optimized, dedicated queue for providing fresh CIDs
 
 From `kubo` [`v0.33.0`](https://github.com/ipfs/kubo/releases/tag/v0.33.0),

docs/config.md

@@ -203,8 +203,11 @@ config file at runtime.
     - [`local-discovery` profile](#local-discovery-profile)
     - [`default-networking` profile](#default-networking-profile)
     - [`flatfs` profile](#flatfs-profile)
+    - [`flatfs-measure` profile](#flatfs-measure-profile)
     - [`pebbleds` profile](#pebbleds-profile)
+    - [`pebbleds-measure` profile](#pebbleds-measure-profile)
     - [`badgerds` profile](#badgerds-profile)
+    - [`badgerds-measure` profile](#badgerds-measure-profile)
     - [`lowpower` profile](#lowpower-profile)
     - [`announce-off` profile](#announce-off-profile)
     - [`announce-on` profile](#announce-on-profile)
@@ -735,6 +738,30 @@ datastores to provide extra functionality (eg metrics, logging, or caching).
 
 Default:
 ```
+{
+  "mounts": [
+  {
+    "mountpoint": "/blocks",
+    "path": "blocks",
+    "prefix": "flatfs.datastore",
+    "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
+    "sync": false,
+    "type": "flatfs"
+  },
+  {
+    "compression": "none",
+    "mountpoint": "/",
+    "path": "datastore",
+    "prefix": "leveldb.datastore",
+    "type": "levelds"
+  }
+  ],
+  "type": "mount"
+}
+```
+
+With `flatfs-measure` profile:
+```
 {
   "mounts": [
   {
@@ -2773,9 +2800,13 @@ You should use this datastore if:
 > [!NOTE]
 > See caveats and configuration options at [`datastores.md#flatfs`](datastores.md#flatfs)
 
+### `flatfs-measure` profile
+
+Configures the node to use the flatfs datastore with metrics. This is the same as [`flatfs` profile](#flatfs-profile) with the addition of the `measure` datastore wrapper.
+
 ### `pebbleds` profile
 
-Configures the node to use the **EXPERIMENTAL** pebble high-performance datastore.
+Configures the node to use the pebble high-performance datastore.
 
 Pebble is a LevelDB/RocksDB inspired key-value store focused on performance and internal usage by CockroachDB.
 You should use this datastore if:
@@ -2793,6 +2824,10 @@ You should use this datastore if:
 > [!NOTE]
 > See other caveats and configuration options at [`datastores.md#pebbleds`](datastores.md#pebbleds)
 
+### `pebbleds-measure` profile
+
+Configures the node to use the pebble datastore with metrics. This is the same as [`pebbleds` profile](#pebble-profile) with the addition of the `measure` datastore wrapper.
+
 ### `badgerds` profile
 
 Configures the node to use the **legacy** badgerv1 datastore.
@@ -2818,6 +2853,10 @@ Also, be aware that:
 > [!NOTE]
 > See other caveats and configuration options at [`datastores.md#pebbleds`](datastores.md#pebbleds)
 
+### `badgerds-measure` profile
+
+Configures the node to use the **legacy** badgerv1 datastore with metrics. This is the same as [`badgerds` profile](#badger-profile) with the addition of the `measure` datastore wrapper.
+
 ### `lowpower` profile
 
 Reduces daemon overhead on the system by disabling optional swarm services.

test/sharness/lib/test-lib.sh

@@ -214,6 +214,27 @@ test_init_ipfs() {
 
 }
 
+test_init_ipfs_measure() {
+  args=("$@")
+
+  # we set the Addresses.API config variable.
+  # the cli client knows to use it, so only need to set.
+  # todo: in the future, use env?
+
+  test_expect_success "ipfs init succeeds" '
+    export IPFS_PATH="$(pwd)/.ipfs" &&
+    ipfs init "${args[@]}" --profile=test,flatfs-measure > /dev/null
+  '
+
+  test_expect_success "prepare config -- mounting" '
+    mkdir mountdir ipfs ipns &&
+    test_config_set Mounts.IPFS "$(pwd)/ipfs" &&
+    test_config_set Mounts.IPNS "$(pwd)/ipns" ||
+    test_fsh cat "\"$IPFS_PATH/config\""
+  '
+
+}
+
 test_wait_for_file() {
   loops=$1
   delay=$2

test/sharness/t0060-daemon.sh

@@ -8,7 +8,7 @@ test_description="Test daemon command"
 
 . lib/test-lib.sh
 
-test_expect_success "create badger config" '
+test_expect_success "create pebble config" '
   ipfs init --profile=pebbleds,test > /dev/null &&
   cp "$IPFS_PATH/config" init-config
 '
@@ -21,7 +21,7 @@ test_launch_ipfs_daemon --init --init-config="$(pwd)/init-config" --init-profile
 test_kill_ipfs_daemon
 
 test_expect_success "daemon initialization with existing config works" '
-  ipfs config "Datastore.Spec.child.path" >actual &&
+  ipfs config "Datastore.Spec.path" >actual &&
   test $(cat actual) = "pebbleds" &&
   ipfs config Addresses > orig_addrs
 '