chore(deps, internal docs): Add usage method to VRL functions

Cargo.toml

@@ -149,6 +149,7 @@ chrono-tz = { version = "0.10.4", default-features = false, features = ["serde"]
 clap = { version = "4.5.53", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] }
 clap_complete = "4.5.65"
 colored = { version = "3.0.0", default-features = false }
+const-str = { version = "1.0.0", default-features = false }
 crossbeam-utils = { version = "0.8.21", default-features = false }
 darling = { version = "0.20.11", default-features = false, features = ["suggestions"] }
 dashmap = { version = "6.1.0", default-features = false }

LICENSE-3rdparty.csv

@@ -184,6 +184,7 @@ concurrent-queue,https://github.com/smol-rs/concurrent-queue,Apache-2.0 OR MIT,"
 const-oid,https://github.com/RustCrypto/formats/tree/master/const-oid,Apache-2.0 OR MIT,RustCrypto Developers
 const-random,https://github.com/tkaitchuck/constrandom,MIT OR Apache-2.0,Tom Kaitchuck <Tom.Kaitchuck@gmail.com>
 const-random-macro,https://github.com/tkaitchuck/constrandom,MIT OR Apache-2.0,Tom Kaitchuck <Tom.Kaitchuck@gmail.com>
+const-str,https://github.com/Nugine/const-str,MIT,Nugine <nugine@foxmail.com>
 convert_case,https://github.com/rutrum/convert-case,MIT,David Purdum <purdum41@gmail.com>
 convert_case,https://github.com/rutrum/convert-case,MIT,rutrum <dave@rutrum.net>
 cookie,https://github.com/SergioBenitez/cookie-rs,MIT OR Apache-2.0,"Sergio Benitez <sb@sergio.bz>, Alex Crichton <alex@alexcrichton.com>"
@@ -278,7 +279,6 @@ executor-trait,https://github.com/amqp-rs/executor-trait,Apache-2.0 OR MIT,Marc-
 exitcode,https://github.com/benwilber/exitcode,Apache-2.0,Ben Wilber <benwilber@gmail.com>
 fakedata_generator,https://github.com/kevingimbel/fakedata_generator,MIT,Kevin Gimbel <hallo@kevingimbel.com>
 fallible-iterator,https://github.com/sfackler/rust-fallible-iterator,MIT OR Apache-2.0,Steven Fackler <sfackler@gmail.com>
-fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien <raph@google.com>, Robin Stocker <robin@nibor.org>"
 fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien <raph@google.com>, Robin Stocker <robin@nibor.org>, Keith Hall <keith.hall@available.systems>"
 fastrand,https://github.com/smol-rs/fastrand,Apache-2.0 OR MIT,Stjepan Glavina <stjepang@gmail.com>
 ff,https://github.com/zkcrypto/ff,MIT OR Apache-2.0,"Sean Bowe <ewillbefull@gmail.com>, Jack Grigg <thestr4d@gmail.com>"

lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs

@@ -13,6 +13,10 @@ impl Function for ParseDnstap {
         "parse_dnstap"
     }
 
+    fn usage(&self) -> &'static str {
+        "Parses the `value` as base64 encoded DNSTAP data."
+    }
+
     fn parameters(&self) -> &'static [Parameter] {
         &[
             Parameter {

lib/enrichment/Cargo.toml

@@ -8,5 +8,7 @@ publish = false
 [dependencies]
 arc-swap.workspace = true
 chrono.workspace = true
+const-str.workspace = true
 dyn-clone = { version = "1.0.20", default-features = false }
+indoc.workspace = true
 vrl.workspace = true

lib/enrichment/src/find_enrichment_table_records.rs

@@ -51,6 +51,13 @@ impl Function for FindEnrichmentTableRecords {
         "find_enrichment_table_records"
     }
 
+    fn usage(&self) -> &'static str {
+        const_str::concat!(
+            "Searches an [enrichment table](/docs/reference/glossary/#enrichment-tables) for rows that match the provided condition.\n\n",
+            super::ENRICHMENT_TABLE_EXPLAINER
+        )
+    }
+
     fn parameters(&self) -> &'static [Parameter] {
         &[
             Parameter {

lib/enrichment/src/get_enrichment_table_record.rs

@@ -48,6 +48,14 @@ impl Function for GetEnrichmentTableRecord {
         "get_enrichment_table_record"
     }
 
+    fn usage(&self) -> &'static str {
+        const USAGE: &str = const_str::concat!(
+            "Searches an [enrichment table](/docs/reference/glossary/#enrichment-tables) for a row that matches the provided condition. A single row must be matched. If no rows are found or more than one row is found, an error is returned.\n\n",
+            super::ENRICHMENT_TABLE_EXPLAINER
+        );
+        USAGE
+    }
+
     fn parameters(&self) -> &'static [Parameter] {
         &[
             Parameter {

lib/enrichment/src/lib.rs

@@ -9,6 +9,7 @@ mod test_util;
 mod vrl_util;
 
 use dyn_clone::DynClone;
+use indoc::indoc;
 pub use tables::{TableRegistry, TableSearch};
 use vrl::{
     compiler::Function,
@@ -97,3 +98,59 @@ pub fn vrl_functions() -> Vec<Box<dyn Function>> {
         Box::new(find_enrichment_table_records::FindEnrichmentTableRecords) as _,
     ]
 }
+
+pub(crate) const ENRICHMENT_TABLE_EXPLAINER: &str = indoc! {r#"
+    For `file` enrichment tables, this condition needs to be a VRL object in which
+    the key-value pairs indicate a field to search mapped to a value to search in that field.
+    This function returns the rows that match the provided condition(s). _All_ fields need to
+    match for rows to be returned; if any fields do not match, then no rows are returned.
+
+    There are currently three forms of search criteria:
+
+    1. **Exact match search**. The given field must match the value exactly. Case sensitivity
+       can be specified using the `case_sensitive` argument. An exact match search can use an
+       index directly into the dataset, which should make this search fairly "cheap" from a
+       performance perspective.
+
+    2. **Wildcard match search**. The given fields specified by the exact match search may also
+        be matched exactly to the value provided to the `wildcard` parameter.
+        A wildcard match search can also use an index directly into the dataset.
+
+    3. **Date range search**. The given field must be greater than or equal to the `from` date
+       and/or less than or equal to the `to` date. A date range search involves
+       sequentially scanning through the rows that have been located using any exact match
+       criteria. This can be an expensive operation if there are many rows returned by any exact
+       match criteria. Therefore, use date ranges as the _only_ criteria when the enrichment
+       data set is very small.
+
+    For `geoip` and `mmdb` enrichment tables, this condition needs to be a VRL object with a single key-value pair
+    whose value needs to be a valid IP address. Example: `{"ip": .ip }`. If a return field is expected
+    and without a value, `null` is used. This table can return the following fields:
+
+    * ISP databases:
+        * `autonomous_system_number`
+        * `autonomous_system_organization`
+        * `isp`
+        * `organization`
+
+    * City databases:
+        * `city_name`
+        * `continent_code`
+        * `country_code`
+        * `country_name`
+        * `region_code`
+        * `region_name`
+        * `metro_code`
+        * `latitude`
+        * `longitude`
+        * `postal_code`
+        * `timezone`
+
+    * Connection-Type databases:
+        * `connection_type`
+
+    To use this function, you need to update your configuration to
+    include an
+    [`enrichment_tables`](/docs/reference/configuration/global-options/#enrichment_tables)
+    parameter.
+"#};

lib/vector-vrl-metrics/Cargo.toml

@@ -8,6 +8,7 @@ license = "MPL-2.0"
 
 [dependencies]
 arc-swap.workspace = true
+const-str.workspace = true
 vrl.workspace = true
 vector-core = { path = "../vector-core", default-features = false, features = ["vrl"] }
 vector-common = { path = "../vector-common", default-features = false }

lib/vector-vrl-metrics/src/aggregate_vector_metrics.rs

@@ -47,6 +47,13 @@ impl Function for AggregateVectorMetrics {
         "aggregate_vector_metrics"
     }
 
+    fn usage(&self) -> &'static str {
+        const_str::concat!(
+            "Aggregates internal Vector metrics, using one of 4 aggregation functions, filtering by name and optionally by tags. Returns the aggregated value. Only includes counter and gauge metrics.\n\n",
+            crate::VECTOR_METRICS_EXPLAINER
+        )
+    }
+
     fn parameters(&self) -> &'static [Parameter] {
         &[
             Parameter {

lib/vector-vrl-metrics/src/find_vector_metrics.rs

@@ -30,6 +30,13 @@ impl Function for FindVectorMetrics {
         "find_vector_metrics"
     }
 
+    fn usage(&self) -> &'static str {
+        const_str::concat!(
+            "Searches internal Vector metrics by name and optionally by tags. Returns all matching metrics.\n\n",
+            crate::VECTOR_METRICS_EXPLAINER
+        )
+    }
+
     fn parameters(&self) -> &'static [Parameter] {
         &[
             Parameter {

lib/vector-vrl-metrics/src/get_vector_metric.rs

@@ -27,6 +27,13 @@ impl Function for GetVectorMetric {
         "get_vector_metric"
     }
 
+    fn usage(&self) -> &'static str {
+        const_str::concat!(
+            "Searches internal Vector metrics by name and optionally by tags. Returns the first matching metric.\n\n",
+            crate::VECTOR_METRICS_EXPLAINER
+        )
+    }
+
     fn parameters(&self) -> &'static [Parameter] {
         &[
             Parameter {

lib/vector-vrl-metrics/src/lib.rs

@@ -8,6 +8,13 @@ mod find_vector_metrics;
 mod get_vector_metric;
 pub use common::MetricsStorage;
 
+pub(crate) const VECTOR_METRICS_EXPLAINER: &str = "\
+Internal Vector metrics functions work with a snapshot of the metrics. The interval at which \
+the snapshot is updated is controlled through the \
+`metrics_storage_refresh_period` (/docs/reference/configuration/global-options/#metrics_storage_refresh_period) \
+global option. Higher values can reduce performance impact of that process, but may cause \
+stale metrics data in the snapshot.";
+
 pub fn all() -> Vec<Box<dyn Function>> {
     vec![
         Box::new(get_vector_metric::GetVectorMetric) as _,

lib/vector-vrl/functions/Cargo.toml

@@ -7,6 +7,7 @@ publish = false
 license = "MPL-2.0"
 
 [dependencies]
+indoc.workspace = true
 vrl.workspace = true
 enrichment = { path = "../../enrichment" }
 dnstap-parser = { path = "../../dnstap-parser", optional = true }

lib/vector-vrl/functions/src/get_secret.rs

@@ -17,6 +17,10 @@ impl Function for GetSecret {
         "get_secret"
     }
 
+    fn usage(&self) -> &'static str {
+        "Returns the value of the given secret from an event."
+    }
+
     fn parameters(&self) -> &'static [Parameter] {
         &[Parameter {
             keyword: "key",

lib/vector-vrl/functions/src/remove_secret.rs

@@ -14,6 +14,10 @@ impl Function for RemoveSecret {
         "remove_secret"
     }
 
+    fn usage(&self) -> &'static str {
+        "Removes a secret from an event."
+    }
+
     fn parameters(&self) -> &'static [Parameter] {
         &[Parameter {
             keyword: "key",

lib/vector-vrl/functions/src/set_secret.rs

@@ -21,6 +21,10 @@ impl Function for SetSecret {
         "set_secret"
     }
 
+    fn usage(&self) -> &'static str {
+        "Sets the given secret in the event."
+    }
+
     fn parameters(&self) -> &'static [Parameter] {
         &[
             Parameter {