From 41655945ac0f0025c6b7f734c90c507f7c7dea6b Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Tue, 2 May 2023 16:44:51 -0400 Subject: [PATCH 01/23] save --- Cargo.lock | 139 ++++++++---------- Cargo.toml | 16 +- lib/codecs/Cargo.toml | 4 +- lib/enrichment/Cargo.toml | 3 +- .../src/find_enrichment_table_records.rs | 10 +- .../src/get_enrichment_table_record.rs | 16 +- lib/enrichment/src/lib.rs | 5 +- lib/enrichment/src/tables.rs | 2 +- lib/enrichment/src/vrl_util.rs | 10 +- lib/opentelemetry-proto/Cargo.toml | 4 +- lib/prometheus-parser/Cargo.toml | 2 +- lib/vector-common/Cargo.toml | 6 +- lib/vector-config/Cargo.toml | 4 +- lib/vector-core/Cargo.toml | 10 +- lib/vector-core/src/event/error.rs | 30 ---- lib/vector-core/src/event/metric/mod.rs | 2 +- lib/vector-core/src/event/metric/tags.rs | 3 +- lib/vector-core/src/event/mod.rs | 1 - lib/vector-core/src/event/vrl_target.rs | 5 +- lib/vector-core/src/vrl.rs | 8 +- lib/vector-lookup/Cargo.toml | 5 +- lib/vector-lookup/src/lib.rs | 7 +- lib/vector-lookup/src/lookup_v2/mod.rs | 2 +- lib/vector-vrl/cli/Cargo.toml | 4 +- lib/vector-vrl/functions/Cargo.toml | 4 +- lib/vector-vrl/functions/src/get_secret.rs | 4 +- lib/vector-vrl/functions/src/lib.rs | 7 +- lib/vector-vrl/functions/src/remove_secret.rs | 4 +- lib/vector-vrl/functions/src/set_secret.rs | 6 +- .../functions/src/set_semantic_meaning.rs | 9 +- lib/vector-vrl/tests/Cargo.toml | 10 +- lib/vector-vrl/tests/src/main.rs | 3 +- lib/vector-vrl/web-playground/Cargo.toml | 6 +- lib/vector-vrl/web-playground/src/lib.rs | 7 +- lib/vrl/value/Cargo.toml | 2 +- src/conditions/vrl.rs | 12 +- src/sinks/aws_cloudwatch_logs/service.rs | 1 - src/sources/aws_kinesis_firehose/handlers.rs | 2 +- src/transforms/metric_to_log.rs | 3 +- src/transforms/remap.rs | 19 +-- 40 files changed, 158 insertions(+), 239 deletions(-) delete mode 100644 lib/vector-core/src/event/error.rs diff --git a/Cargo.lock b/Cargo.lock index 880981e0ea850..5e404d8b4182e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1999,7 +1999,7 @@ dependencies = [ "indoc", "memchr", "once_cell", - "ordered-float 3.6.0", + "ordered-float 3.7.0", "prost", "regex", "serde", @@ -2583,7 +2583,7 @@ checksum = "8d7439c3735f405729d52c3fbbe4de140eaf938a1fe47d227c27f8254d4302a5" [[package]] name = "datadog-filter" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.2.0#258cc611080c1bdb5ed6897e8a79b76ab2038fcb" +source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "datadog-search-syntax", "dyn-clone", @@ -2593,18 +2593,18 @@ dependencies = [ [[package]] name = "datadog-grok" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.2.0#258cc611080c1bdb5ed6897e8a79b76ab2038fcb" +source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "bytes 1.4.0", "chrono", "chrono-tz", "lalrpop", "lalrpop-util", - "lookup", "nom", "once_cell", "onig", - "ordered-float 3.6.0", + "ordered-float 3.7.0", + "path", "peeking_take_while", "regex", "serde_json", @@ -2617,7 +2617,7 @@ dependencies = [ [[package]] name = "datadog-search-syntax" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.2.0#258cc611080c1bdb5ed6897e8a79b76ab2038fcb" +source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "itertools", "once_cell", @@ -2929,7 +2929,6 @@ dependencies = [ "arc-swap", "chrono", "dyn-clone", - "value", "vector-common", "vrl", ] @@ -4127,17 +4126,6 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f551f8c3a39f68f986517db0d1759de85881894fdc7db798bd2a9df9cb04b7fc" -[[package]] -name = "inherent" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a036328c11e86e024522cb1e9b78ba9df3e316995e004e98854a18e4a326d2e1" -dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", - "syn 1.0.109", -] - [[package]] name = "inotify" version = "0.9.6" @@ -4541,7 +4529,6 @@ dependencies = [ "itertools", "lalrpop-util", "petgraph", - "pico-args", "regex", "regex-syntax 0.6.29", "string_cache", @@ -4555,9 +4542,6 @@ name = "lalrpop-util" version = "0.19.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5c1f7869c94d214466c5fd432dfed12c379fd87786768d36455892d46b18edd" -dependencies = [ - "regex", -] [[package]] name = "lapin" @@ -4721,21 +4705,6 @@ dependencies = [ "snap", ] -[[package]] -name = "lookup" -version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.2.0#258cc611080c1bdb5ed6897e8a79b76ab2038fcb" -dependencies = [ - "inherent", - "lalrpop", - "lalrpop-util", - "once_cell", - "quickcheck", - "regex", - "serde", - "snafu", -] - [[package]] name = "lru" version = "0.10.0" @@ -5665,7 +5634,7 @@ dependencies = [ "bytes 1.4.0", "chrono", "hex", - "ordered-float 3.6.0", + "ordered-float 3.7.0", "prost", "prost-build", "tonic 0.9.2", @@ -5686,9 +5655,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "3.6.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13a384337e997e6860ffbaa83708b2ef329fd8c54cb67a5f64d421e0f943254f" +checksum = "2fc2dbde8f8a79f2102cc474ceb0ad68e3b80b85289ea62389b60e66777e4213" dependencies = [ "num-traits", ] @@ -5784,6 +5753,17 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +[[package]] +name = "path" +version = "0.1.0" +source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" +dependencies = [ + "once_cell", + "regex", + "serde", + "snafu", +] + [[package]] name = "pbkdf2" version = "0.11.0" @@ -5924,12 +5904,6 @@ dependencies = [ "siphasher", ] -[[package]] -name = "pico-args" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8bcd96cb740d03149cbad5518db9fd87126a10ab519c011893b1754134c468" - [[package]] name = "pin-project" version = "1.0.12" @@ -9086,14 +9060,15 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.2.0#258cc611080c1bdb5ed6897e8a79b76ab2038fcb" +source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "async-graphql", "bytes 1.4.0", "chrono", - "lookup", "mlua", - "ordered-float 3.6.0", + "once_cell", + "ordered-float 3.7.0", + "path", "quickcheck", "regex", "serde", @@ -9261,7 +9236,7 @@ dependencies = [ "openssl-probe", "openssl-src", "opentelemetry-proto", - "ordered-float 3.6.0", + "ordered-float 3.7.0", "paste", "percent-encoding", "pin-project", @@ -9426,7 +9401,7 @@ dependencies = [ "indexmap", "metrics", "nom", - "ordered-float 3.6.0", + "ordered-float 3.7.0", "paste", "pin-project", "quickcheck", @@ -9537,7 +9512,7 @@ dependencies = [ "noisy_float", "once_cell", "openssl", - "ordered-float 3.6.0", + "ordered-float 3.7.0", "parking_lot", "pin-project", "proptest", @@ -9589,10 +9564,10 @@ dependencies = [ name = "vector-lookup" version = "0.1.0" dependencies = [ - "lookup", "serde", "vector-config", "vector-config-macros", + "vrl", ] [[package]] @@ -9609,8 +9584,6 @@ dependencies = [ name = "vector-vrl-functions" version = "0.1.0" dependencies = [ - "lookup", - "value", "vrl", ] @@ -9624,7 +9597,7 @@ dependencies = [ "clap 4.1.14", "enrichment", "glob", - "lookup", + "path", "prettydiff", "regex", "serde", @@ -9669,37 +9642,39 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vrl" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.2.0#258cc611080c1bdb5ed6897e8a79b76ab2038fcb" +source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ - "bytes 1.4.0", - "indoc", - "lookup", - "ordered-float 3.6.0", + "datadog-filter", + "datadog-grok", + "datadog-search-syntax", + "path", "value", "vrl-compiler", "vrl-core", "vrl-diagnostic", "vrl-parser", + "vrl-stdlib", ] [[package]] name = "vrl-cli" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.2.0#258cc611080c1bdb5ed6897e8a79b76ab2038fcb" +source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "clap 4.1.14", "exitcode", "indoc", - "lookup", "once_cell", + "path", "prettytable-rs", "regex", "rustyline", "serde_json", "thiserror", "value", - "vrl", + "vrl-compiler", "vrl-core", + "vrl-diagnostic", "vrl-stdlib", "webbrowser", ] @@ -9707,7 +9682,7 @@ dependencies = [ [[package]] name = "vrl-compiler" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.2.0#258cc611080c1bdb5ed6897e8a79b76ab2038fcb" +source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "anymap", "bytes 1.4.0", @@ -9715,9 +9690,9 @@ dependencies = [ "dyn-clone", "getrandom 0.2.9", "lalrpop-util", - "lookup", - "ordered-float 3.6.0", + "ordered-float 3.7.0", "paste", + "path", "regex", "serde", "thiserror", @@ -9730,15 +9705,15 @@ dependencies = [ [[package]] name = "vrl-core" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.2.0#258cc611080c1bdb5ed6897e8a79b76ab2038fcb" +source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "bytes 1.4.0", "chrono", "chrono-tz", "derivative", - "lookup", "nom", - "ordered-float 3.6.0", + "ordered-float 3.7.0", + "path", "serde", "serde_json", "snafu", @@ -9750,7 +9725,7 @@ dependencies = [ [[package]] name = "vrl-diagnostic" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.2.0#258cc611080c1bdb5ed6897e8a79b76ab2038fcb" +source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "codespan-reporting", "termcolor", @@ -9759,13 +9734,13 @@ dependencies = [ [[package]] name = "vrl-parser" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.2.0#258cc611080c1bdb5ed6897e8a79b76ab2038fcb" +source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "lalrpop", "lalrpop-util", - "lookup", - "ordered-float 3.6.0", + "ordered-float 3.7.0", "paste", + "path", "thiserror", "vrl-diagnostic", ] @@ -9773,7 +9748,7 @@ dependencies = [ [[package]] name = "vrl-stdlib" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.2.0#258cc611080c1bdb5ed6897e8a79b76ab2038fcb" +source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "aes", "base16", @@ -9797,11 +9772,13 @@ dependencies = [ "hmac", "hostname", "indexmap", - "lookup", + "indoc", "md-5", "nom", "ofb", "once_cell", + "ordered-float 3.7.0", + "path", "percent-encoding", "quoted_printable", "rand 0.8.5", @@ -9822,7 +9799,7 @@ dependencies = [ "utf8-width", "uuid", "value", - "vrl", + "vrl-compiler", "vrl-core", "vrl-diagnostic", "woothee", @@ -9832,14 +9809,14 @@ dependencies = [ [[package]] name = "vrl-tests" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.2.0#258cc611080c1bdb5ed6897e8a79b76ab2038fcb" +source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "ansi_term", "chrono", "chrono-tz", "clap 4.1.14", "glob", - "lookup", + "path", "prettydiff", "regex", "serde", @@ -9847,7 +9824,9 @@ dependencies = [ "tikv-jemallocator", "tracing-subscriber", "value", - "vrl", + "vrl-compiler", + "vrl-core", + "vrl-diagnostic", "vrl-stdlib", ] diff --git a/Cargo.toml b/Cargo.toml index 29eb3df403472..622cfd981df12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,7 +126,7 @@ portpicker = { path = "lib/portpicker" } prometheus-parser = { path = "lib/prometheus-parser", optional = true } opentelemetry-proto = { path = "lib/opentelemetry-proto", optional = true } tracing-limit = { path = "lib/tracing-limit" } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } +value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } vector-api-client = { path = "lib/vector-api-client", optional = true } vector-buffers = { path = "lib/vector-buffers", default-features = false } vector-common = { path = "lib/vector-common" } @@ -135,7 +135,7 @@ vector-config-common = { path = "lib/vector-config-common" } vector-config-macros = { path = "lib/vector-config-macros" } vector-core = { path = "lib/vector-core", default-features = false, features = ["vrl"] } vector-vrl-functions = { path = "lib/vector-vrl/functions" } -vrl-cli = { package = "vrl-cli", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", optional = true } +vrl-cli = { package = "vrl-cli", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", optional = true } loki-logproto = { path = "lib/loki-logproto", optional = true } # Tokio / Futures @@ -223,14 +223,14 @@ number_prefix = { version = "0.4.0", default-features = false, features = ["std" tui = { version = "0.19.0", optional = true, default-features = false, features = ["crossterm"] } # Datadog Pipelines -datadog-filter = { package = "datadog-filter", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } -datadog-search-syntax = { package = "datadog-search-syntax", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } +datadog-filter = { package = "datadog-filter", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +datadog-search-syntax = { package = "datadog-search-syntax", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } hex = { version = "0.4.3", default-features = false, optional = true } sha2 = { version = "0.10.6", default-features = false, optional = true } # VRL Lang -vrl = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } -vrl-stdlib = { package = "vrl-stdlib", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } +vrl = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +vrl-stdlib = { package = "vrl-stdlib", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } # External libs arc-swap = { version = "1.6", default-features = false, optional = true } @@ -287,7 +287,7 @@ notify = { version = "5.1.0", default-features = false, features = ["macos_fseve once_cell = { version = "1.17", default-features = false } openssl = { version = "0.10.50", default-features = false, features = ["vendored"] } openssl-probe = { version = "0.1.5", default-features = false } -ordered-float = { version = "3.6.0", default-features = false } +ordered-float = { version = "3.7.0", default-features = false } paste = "1.0.12" percent-encoding = { version = "2.2.0", default-features = false } pin-project = { version = "1.0.12", default-features = false } @@ -360,7 +360,7 @@ test-generator = "0.3.1" tokio-test = "0.4.2" tokio = { version = "1.28.0", features = ["test-util"] } tower-test = "0.4.0" -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", features = ["test"] } +value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", features = ["test"] } vector-core = { path = "lib/vector-core", default-features = false, features = ["vrl", "test"] } wiremock = "0.5.18" zstd = { version = "0.12.3", default-features = false } diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 34284c827bffc..4e9cdc2240465 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -15,7 +15,7 @@ dyn-clone = { version = "1", default-features = false } lookup = { package = "vector-lookup", path = "../vector-lookup", default-features = false } memchr = { version = "2", default-features = false } once_cell = { version = "1.17", default-features = false } -ordered-float = { version = "3.6.0", default-features = false } +ordered-float = { version = "3.7.0", default-features = false } prost = { version = "0.11.8", default-features = false, features = ["std"] } regex = { version = "1.8.1", default-features = false, features = ["std", "perf"] } serde = { version = "1", default-features = false, features = ["derive"] } @@ -25,7 +25,7 @@ snafu = { version = "0.7.4", default-features = false, features = ["futures"] } syslog_loose = { version = "0.18", default-features = false, optional = true } tokio-util = { version = "0.7", default-features = false, features = ["codec"] } tracing = { version = "0.1", default-features = false } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", default-features = false } +value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", default-features = false } vector-common = { path = "../vector-common", default-features = false } vector-config = { path = "../vector-config", default-features = false } vector-config-common = { path = "../vector-config-common", default-features = false } diff --git a/lib/enrichment/Cargo.toml b/lib/enrichment/Cargo.toml index 7735b3a23cf7a..471ffaea26009 100644 --- a/lib/enrichment/Cargo.toml +++ b/lib/enrichment/Cargo.toml @@ -10,5 +10,4 @@ arc-swap = { version = "1.6.0", default-features = false } dyn-clone = { version = "1.0.11", default-features = false } chrono = { version = "0.4.19", default-features = false } vector-common = { path = "../vector-common", default-features = false, features = [ "btreemap", "conversion", "serde" ] } -vrl = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", default-features = false } +vrl = { git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } diff --git a/lib/enrichment/src/find_enrichment_table_records.rs b/lib/enrichment/src/find_enrichment_table_records.rs index 34c5373763657..254bc983970d7 100644 --- a/lib/enrichment/src/find_enrichment_table_records.rs +++ b/lib/enrichment/src/find_enrichment_table_records.rs @@ -1,8 +1,5 @@ use std::collections::BTreeMap; - -use ::value::Value; -use vrl::prelude::*; -use vrl::state::TypeState; +use vrl::stdlib::prelude::*; use crate::{ vrl_util::{self, add_index, evaluate_condition}, @@ -164,7 +161,7 @@ impl FunctionExpression for FindEnrichmentTableRecordsFn { let value = value.resolve(ctx)?; evaluate_condition(key, value) }) - .collect::>>()?; + .collect::>>()?; let select = self .select @@ -196,7 +193,8 @@ impl FunctionExpression for FindEnrichmentTableRecordsFn { mod tests { use ::value::Secrets; use vector_common::TimeZone; - use vrl::TargetValue; + use vrl::compiler::TargetValue; + use vrl::value::Secrets; use super::*; use crate::test_util::get_table_registry; diff --git a/lib/enrichment/src/get_enrichment_table_record.rs b/lib/enrichment/src/get_enrichment_table_record.rs index 4a059ab4c28cc..0ad8299469231 100644 --- a/lib/enrichment/src/get_enrichment_table_record.rs +++ b/lib/enrichment/src/get_enrichment_table_record.rs @@ -1,14 +1,6 @@ use std::collections::BTreeMap; - -use value::{kind::Collection, Kind, Value}; -use vrl::prelude::FunctionExpression; -use vrl::state::TypeState; -use vrl::{ - function::{ArgumentList, Compiled, Example, FunctionCompileContext, Parameter}, - prelude::{expression, DiagnosticMessage, Resolved, Result, TypeDef, VrlValueConvert}, - value::kind, - Context, Expression, Function, -}; +use vrl::compiler::value::Error; +use vrl::stdlib::prelude::*; use crate::{ vrl_util::{self, add_index, evaluate_condition}, @@ -29,7 +21,7 @@ fn get_enrichment_table_record( .iter() .map(|value| Ok(value.try_bytes_utf8_lossy()?.to_string())) .collect::, _>>(), - value => Err(vrl::value::Error::Expected { + value => Err(Error::Expected { got: value.kind(), expected: Kind::array(Collection::any()), }), @@ -162,7 +154,7 @@ impl FunctionExpression for GetEnrichmentTableRecordFn { let value = value.resolve(ctx)?; evaluate_condition(key, value) }) - .collect::>>()?; + .collect::>>()?; let select = self .select diff --git a/lib/enrichment/src/lib.rs b/lib/enrichment/src/lib.rs index 73213c1ea9550..625c4780900a5 100644 --- a/lib/enrichment/src/lib.rs +++ b/lib/enrichment/src/lib.rs @@ -11,7 +11,8 @@ use std::collections::BTreeMap; use dyn_clone::DynClone; pub use tables::{TableRegistry, TableSearch}; -use value::Value; +use vrl::compiler::Function; +use vrl::value::Value; #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub struct IndexHandle(pub usize); @@ -77,7 +78,7 @@ pub trait Table: DynClone { dyn_clone::clone_trait_object!(Table); -pub fn vrl_functions() -> Vec> { +pub fn vrl_functions() -> Vec> { vec![ Box::new(get_enrichment_table_record::GetEnrichmentTableRecord) as _, Box::new(find_enrichment_table_records::FindEnrichmentTableRecords) as _, diff --git a/lib/enrichment/src/tables.rs b/lib/enrichment/src/tables.rs index 2bfef6969bc88..44dbd237a268a 100644 --- a/lib/enrichment/src/tables.rs +++ b/lib/enrichment/src/tables.rs @@ -35,7 +35,7 @@ use std::{ }; use arc_swap::ArcSwap; -use value::Value; +use vrl::value::Value; use super::{Condition, IndexHandle, Table}; use crate::Case; diff --git a/lib/enrichment/src/vrl_util.rs b/lib/enrichment/src/vrl_util.rs index d26198b979460..3f73094406b32 100644 --- a/lib/enrichment/src/vrl_util.rs +++ b/lib/enrichment/src/vrl_util.rs @@ -1,13 +1,9 @@ //! Utilities shared between both VRL functions. use std::collections::BTreeMap; -use ::value::Value; -use vrl::{ - diagnostic::{Label, Span}, - prelude::*, -}; - use crate::{Case, Condition, IndexHandle, TableRegistry}; +use vrl::diagnostic::{Label, Span}; +use vrl::stdlib::prelude::*; #[derive(Debug)] pub enum Error { @@ -42,7 +38,7 @@ impl DiagnosticMessage for Error { } /// Evaluates the condition object to search the enrichment tables with. -pub(crate) fn evaluate_condition(key: &str, value: Value) -> Result { +pub(crate) fn evaluate_condition(key: &str, value: Value) -> ExpressionResult { Ok(match value { Value::Object(map) if map.contains_key("from") && map.contains_key("to") => { Condition::BetweenDates { diff --git a/lib/opentelemetry-proto/Cargo.toml b/lib/opentelemetry-proto/Cargo.toml index 40221e0ac1881..6f0e87d9b2088 100644 --- a/lib/opentelemetry-proto/Cargo.toml +++ b/lib/opentelemetry-proto/Cargo.toml @@ -14,8 +14,8 @@ bytes = { version = "1.4.0", default-features = false, features = ["serde"] } chrono = { version = "0.4.19", default-features = false, features = ["serde"] } hex = { version = "0.4.3", default-features = false, features = ["std"] } lookup = { package = "vector-lookup", path = "../vector-lookup", default-features = false } -ordered-float = { version = "3.6.0", default-features = false } +ordered-float = { version = "3.7.0", default-features = false } prost = { version = "0.11", default-features = false, features = ["std"] } tonic = { version = "0.9", default-features = false, features = ["codegen", "gzip", "prost", "tls", "tls-roots", "transport"] } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } +value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } vector-core = { path = "../vector-core", default-features = false } diff --git a/lib/prometheus-parser/Cargo.toml b/lib/prometheus-parser/Cargo.toml index b9f414372238d..9ff0f381f71f2 100644 --- a/lib/prometheus-parser/Cargo.toml +++ b/lib/prometheus-parser/Cargo.toml @@ -16,7 +16,7 @@ prost = "0.11" prost-types = "0.11" snafu = { version = "0.7" } vector-common = { path = "../vector-common", features = ["btreemap"] } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } +value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } [build-dependencies] prost-build = "0.11" diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 57119279f8b24..de3ffa6135136 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -51,7 +51,7 @@ futures = { version = "0.3.28", default-features = false, features = ["std"] } indexmap = { version = "~1.9.3", default-features = false } metrics = "0.20.1" nom = { version = "7", optional = true } -ordered-float = { version = "3.6.0", default-features = false } +ordered-float = { version = "3.7.0", default-features = false } paste = "1.0.12" pin-project = { version = "1.0.12", default-features = false } ryu = { version = "1", default-features = false } @@ -62,8 +62,8 @@ snafu = { version = "0.7", optional = true } stream-cancel = { version = "0.8.1", default-features = false } tokio = { version = "1.28.0", default-features = false, features = ["macros", "time"] } tracing = { version = "0.1.34", default-features = false } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", features = ["json", "arbitrary"] } -vrl-core = { package = "vrl-core", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", default-features = false } +value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", features = ["json", "arbitrary"] } +vrl-core = { package = "vrl-core", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", default-features = false } vector-config = { path = "../vector-config" } vector-config-common = { path = "../vector-config-common" } vector-config-macros = { path = "../vector-config-macros" } diff --git a/lib/vector-config/Cargo.toml b/lib/vector-config/Cargo.toml index ce5411e7082a7..6d8f8d86c12bf 100644 --- a/lib/vector-config/Cargo.toml +++ b/lib/vector-config/Cargo.toml @@ -26,8 +26,8 @@ snafu = { version = "0.7.4", default-features = false } toml = { version = "0.7.3", default-features = false } tracing = { version = "0.1.34", default-features = false } url = { version = "2.3.1", default-features = false, features = ["serde"] } -vrl-compiler = { package = "vrl-compiler", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } -vrl-core = { package = "vrl-core", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", default-features = false, features = ["serde"] } +vrl-compiler = { package = "vrl-compiler", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +vrl-core = { package = "vrl-core", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", default-features = false, features = ["serde"] } vector-config-common = { path = "../vector-config-common" } vector-config-macros = { path = "../vector-config-macros" } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 72177e7b5e75c..3fb4a03888a1a 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -23,7 +23,7 @@ headers = { version = "0.3.8", default-features = false } http = { version = "0.2.9", default-features = false } hyper-proxy = { version = "0.9.1", default-features = false, features = ["openssl-tls"] } indexmap = { version = "~1.9.3", default-features = false, features = ["serde"] } -lookup = { package = "vector-lookup", path = "../vector-lookup", features = ["arbitrary"] } +lookup = { package = "vector-lookup", path = "../vector-lookup" } metrics = "0.20.1" metrics-tracing-context = { version = "0.12.0", default-features = false } metrics-util = { version = "0.14.0", default-features = false, features = ["registry"] } @@ -31,7 +31,7 @@ mlua = { version = "0.8.8", default-features = false, features = ["lua54", "send no-proxy = { version = "0.3.2", default-features = false, features = ["serialize"] } once_cell = { version = "1.17", default-features = false } openssl = { version = "0.10.50", default-features = false, features = ["vendored"] } -ordered-float = { version = "3.6.0", default-features = false } +ordered-float = { version = "3.7.0", default-features = false } parking_lot = { version = "0.12.1", default-features = false } pin-project = { version = "1.0.12", default-features = false } proptest = { version = "1.1", optional = true } @@ -60,7 +60,7 @@ tracing-subscriber = { version = "0.3.16", default-features = false, features = typetag = { version = "0.2.7", default-features = false } twox-hash = { version = "1.6.3", default-features = false } url = { version = "2", default-features = false } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", default-features = false, features = ["lua", "toml", "json", "api"] } +value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", default-features = false, features = ["lua", "toml", "json", "api"] } vector-buffers = { path = "../vector-buffers", default-features = false } vector-common = { path = "../vector-common" } vector-config = { path = "../vector-config" } @@ -68,7 +68,7 @@ vector-config-common = { path = "../vector-config-common" } vector-config-macros = { path = "../vector-config-macros" } # Rename to "vrl" once we use a release with stable `-Z namespaced-features`: # https://doc.rust-lang.org/cargo/reference/unstable.html#namespaced-features -vrl-lib = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } +vrl-lib = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } [target.'cfg(target_os = "macos")'.dependencies] security-framework = "2.8.2" @@ -97,7 +97,7 @@ rand = "0.8.5" rand_distr = "0.4.3" tracing-subscriber = { version = "0.3.16", default-features = false, features = ["env-filter", "fmt", "ansi", "registry"] } vector-common = { path = "../vector-common", default-features = false, features = ["test"] } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", default-features = false, features = ["lua", "toml", "json", "api", "arbitrary", "test"] } +value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", default-features = false, features = ["lua", "toml", "json", "api", "arbitrary", "test"] } [features] api = ["dep:async-graphql", "value/api"] diff --git a/lib/vector-core/src/event/error.rs b/lib/vector-core/src/event/error.rs deleted file mode 100644 index 0f0a7ac245306..0000000000000 --- a/lib/vector-core/src/event/error.rs +++ /dev/null @@ -1,30 +0,0 @@ -use lookup::LookupBuf; -use snafu::Snafu; - -use crate::event::Value; - -#[derive(Debug, Snafu)] -pub enum EventError { - #[snafu(display( - "Cannot insert value nested inside primitive located at {}. {} was the original target.", - primitive_at, - original_target - ))] - PrimitiveDescent { - primitive_at: LookupBuf, - original_target: LookupBuf, - original_value: Option, - }, - #[snafu(display("Lookup Error: {}", source))] - LookupError { source: lookup::LookupError }, - #[snafu(display("Empty coalesce subsegment found."))] - EmptyCoalesceSubSegment, - #[snafu(display("Cannot remove self."))] - RemovingSelf, -} - -impl From for EventError { - fn from(v: lookup::LookupError) -> Self { - Self::LookupError { source: v } - } -} diff --git a/lib/vector-core/src/event/metric/mod.rs b/lib/vector-core/src/event/metric/mod.rs index 1347a1429a247..6f63a61febee6 100644 --- a/lib/vector-core/src/event/metric/mod.rs +++ b/lib/vector-core/src/event/metric/mod.rs @@ -10,7 +10,7 @@ use chrono::{DateTime, Utc}; use vector_common::EventDataEq; use vector_config::configurable_component; #[cfg(feature = "vrl")] -use vrl_lib::prelude::VrlValueConvert; +use vrl_lib::compiler::value::VrlValueConvert; use crate::{ event::{ diff --git a/lib/vector-core/src/event/metric/tags.rs b/lib/vector-core/src/event/metric/tags.rs index 9a2070b0f4d30..8d3394a2f3b05 100644 --- a/lib/vector-core/src/event/metric/tags.rs +++ b/lib/vector-core/src/event/metric/tags.rs @@ -11,7 +11,6 @@ use indexmap::IndexSet; use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize, Serializer}; use vector_common::byte_size_of::ByteSizeOf; use vector_config::{configurable_component, Configurable}; -use vrl_lib::prelude::fmt::Formatter; /// A single tag value, either a bare tag or a value. #[derive(Clone, Configurable, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] @@ -117,7 +116,7 @@ impl Default for TagValueSet { } impl Display for TagValueSet { - fn fmt(&self, f: &mut Formatter<'_>) -> vrl_lib::prelude::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { for (i, value) in self.iter().enumerate() { if i != 0 { write!(f, ", ")?; diff --git a/lib/vector-core/src/event/mod.rs b/lib/vector-core/src/event/mod.rs index 06840a046eb9f..03947095037f3 100644 --- a/lib/vector-core/src/event/mod.rs +++ b/lib/vector-core/src/event/mod.rs @@ -25,7 +25,6 @@ pub use vrl_target::{TargetEvents, VrlTarget}; pub mod array; pub mod discriminant; -pub mod error; mod estimated_json_encoded_size_of; mod log_event; #[cfg(feature = "lua")] diff --git a/lib/vector-core/src/event/vrl_target.rs b/lib/vector-core/src/event/vrl_target.rs index 5eca15ac21a5a..b31216b202766 100644 --- a/lib/vector-core/src/event/vrl_target.rs +++ b/lib/vector-core/src/event/vrl_target.rs @@ -4,7 +4,8 @@ use std::{collections::BTreeMap, convert::TryFrom, marker::PhantomData}; use lookup::lookup_v2::OwnedSegment; use lookup::{OwnedTargetPath, OwnedValuePath, PathPrefix}; use snafu::Snafu; -use vrl_lib::{prelude::VrlValueConvert, ProgramInfo, SecretTarget}; +use vrl_lib::compiler::value::VrlValueConvert; +use vrl_lib::compiler::{ProgramInfo, SecretTarget, Target}; use super::{Event, EventMetadata, LogEvent, Metric, MetricKind, TraceEvent, Value}; use crate::config::log_schema; @@ -198,7 +199,7 @@ fn set_metric_tag_values(name: String, value: &Value, metric: &mut Metric, multi } } -impl vrl_lib::Target for VrlTarget { +impl Target for VrlTarget { fn target_insert( &mut self, target_path: &OwnedTargetPath, diff --git a/lib/vector-core/src/vrl.rs b/lib/vector-core/src/vrl.rs index d8b6533245ff6..1f6424a399346 100644 --- a/lib/vector-core/src/vrl.rs +++ b/lib/vector-core/src/vrl.rs @@ -1,6 +1,8 @@ use lookup::{owned_value_path, OwnedTargetPath}; -use vrl_lib::state::TypeState; -use vrl_lib::{diagnostic::DiagnosticList, CompilationResult, CompileConfig, Function}; +use vrl_lib::compiler::{ + compile_with_state, CompilationResult, CompileConfig, Function, TypeState, +}; +use vrl_lib::diagnostic::DiagnosticList; /// Compiles a VRL program /// Vector metadata is set to read-only to prevent it from being mutated @@ -20,5 +22,5 @@ pub fn compile_vrl( // prevents users from potentially breaking behavior relying on it. config.set_read_only_path(OwnedTargetPath::metadata(owned_value_path!("vector")), true); - vrl_lib::compile_with_state(source, fns, state, config) + compile_with_state(source, fns, state, config) } diff --git a/lib/vector-lookup/Cargo.toml b/lib/vector-lookup/Cargo.toml index e3ceaed8cf1c5..302c81c3da6e7 100644 --- a/lib/vector-lookup/Cargo.toml +++ b/lib/vector-lookup/Cargo.toml @@ -10,7 +10,4 @@ license = "MPL-2.0" serde = { version = "1.0.159", default-features = false, features = ["derive", "alloc"] } vector-config = { path = "../vector-config" } vector-config-macros = { path = "../vector-config-macros" } -lookup = { package = "lookup", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", default-features = false } - -[features] -arbitrary = ["lookup/arbitrary"] +vrl = { git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", default-features = false, features = ["path"] } diff --git a/lib/vector-lookup/src/lib.rs b/lib/vector-lookup/src/lib.rs index 43965828feb2b..7058cfc6e899f 100644 --- a/lib/vector-lookup/src/lib.rs +++ b/lib/vector-lookup/src/lib.rs @@ -1,10 +1,7 @@ #![deny(warnings)] -pub use lookup_v2::PathPrefix; - -pub use lookup::{ - event_path, metadata_path, owned_value_path, path, Field, FieldBuf, Look, LookSegment, Lookup, - LookupBuf, LookupError, OwnedTargetPath, OwnedValuePath, Segment, SegmentBuf, +pub use vrl::path::{ + event_path, metadata_path, owned_value_path, path, OwnedTargetPath, OwnedValuePath, PathPrefix, }; pub mod lookup_v2; diff --git a/lib/vector-lookup/src/lookup_v2/mod.rs b/lib/vector-lookup/src/lookup_v2/mod.rs index d6df060185262..b8c61d83fc137 100644 --- a/lib/vector-lookup/src/lookup_v2/mod.rs +++ b/lib/vector-lookup/src/lookup_v2/mod.rs @@ -3,7 +3,7 @@ mod optional_path; pub use optional_path::{OptionalTargetPath, OptionalValuePath}; use vector_config_macros::configurable_component; -pub use lookup::lookup_v2::{ +pub use vrl::path::{ parse_target_path, parse_value_path, BorrowedSegment, OwnedSegment, OwnedTargetPath, OwnedValuePath, PathConcat, PathParseError, PathPrefix, TargetPath, ValuePath, }; diff --git a/lib/vector-vrl/cli/Cargo.toml b/lib/vector-vrl/cli/Cargo.toml index 9c6391517ae35..9d871755f0e78 100644 --- a/lib/vector-vrl/cli/Cargo.toml +++ b/lib/vector-vrl/cli/Cargo.toml @@ -7,7 +7,7 @@ publish = false license = "MPL-2.0" [dependencies] -vrl-cli = { package = "vrl-cli", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } +vrl-cli = { package = "vrl-cli", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } clap = { version = "4.1.14", features = ["derive"] } -vrl-stdlib = { package = "vrl-stdlib", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } +vrl-stdlib = { package = "vrl-stdlib", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } vector-vrl-functions = { path = "../functions" } diff --git a/lib/vector-vrl/functions/Cargo.toml b/lib/vector-vrl/functions/Cargo.toml index 3c565b386a7f4..cdd91b3bfd6c9 100644 --- a/lib/vector-vrl/functions/Cargo.toml +++ b/lib/vector-vrl/functions/Cargo.toml @@ -7,6 +7,4 @@ publish = false license = "MPL-2.0" [dependencies] -lookup = { package = "lookup", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } -vrl = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", default-features = false, features = [] } +vrl = { git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } diff --git a/lib/vector-vrl/functions/src/get_secret.rs b/lib/vector-vrl/functions/src/get_secret.rs index 6446bb168079c..1ef9b8b89f0b9 100644 --- a/lib/vector-vrl/functions/src/get_secret.rs +++ b/lib/vector-vrl/functions/src/get_secret.rs @@ -1,6 +1,4 @@ -use ::value::Value; -use vrl::prelude::*; -use vrl::state::TypeState; +use vrl::stdlib::prelude::*; fn get_secret(ctx: &mut Context, key: Value) -> std::result::Result { let key_bytes = key.as_bytes().expect("argument must be a string"); diff --git a/lib/vector-vrl/functions/src/lib.rs b/lib/vector-vrl/functions/src/lib.rs index 0a5eebca362c8..9d608f3d15706 100644 --- a/lib/vector-vrl/functions/src/lib.rs +++ b/lib/vector-vrl/functions/src/lib.rs @@ -1,12 +1,13 @@ #![deny(warnings)] +use vrl::compiler::Function; +use vrl::path::OwnedTargetPath; + pub mod get_secret; pub mod remove_secret; pub mod set_secret; pub mod set_semantic_meaning; -use lookup::OwnedTargetPath; - #[allow(clippy::large_enum_variant)] #[derive(Clone, Debug)] pub enum MetadataKey { @@ -16,7 +17,7 @@ pub enum MetadataKey { pub const LEGACY_METADATA_KEYS: [&str; 2] = ["datadog_api_key", "splunk_hec_token"]; -pub fn all() -> Vec> { +pub fn all() -> Vec> { vec![ Box::new(set_semantic_meaning::SetSemanticMeaning) as _, Box::new(get_secret::GetSecret) as _, diff --git a/lib/vector-vrl/functions/src/remove_secret.rs b/lib/vector-vrl/functions/src/remove_secret.rs index 1a3554e746ff5..71f81439734c5 100644 --- a/lib/vector-vrl/functions/src/remove_secret.rs +++ b/lib/vector-vrl/functions/src/remove_secret.rs @@ -1,6 +1,4 @@ -use ::value::Value; -use vrl::prelude::*; -use vrl::state::TypeState; +use vrl::stdlib::prelude::*; fn remove_secret(ctx: &mut Context, key: Value) -> std::result::Result { let key_bytes = key.as_bytes().expect("argument must be a string"); diff --git a/lib/vector-vrl/functions/src/set_secret.rs b/lib/vector-vrl/functions/src/set_secret.rs index 45036d797603d..4254043ff65c8 100644 --- a/lib/vector-vrl/functions/src/set_secret.rs +++ b/lib/vector-vrl/functions/src/set_secret.rs @@ -1,6 +1,6 @@ -use ::value::Value; -use vrl::prelude::*; -use vrl::state::TypeState; +use vrl::stdlib::prelude::*; +// use vrl::value::kind: +use vrl::compiler::TypeDef; fn set_secret( ctx: &mut Context, diff --git a/lib/vector-vrl/functions/src/set_semantic_meaning.rs b/lib/vector-vrl/functions/src/set_semantic_meaning.rs index 9785e38274dad..861b13d990c96 100644 --- a/lib/vector-vrl/functions/src/set_semantic_meaning.rs +++ b/lib/vector-vrl/functions/src/set_semantic_meaning.rs @@ -1,9 +1,8 @@ +use std::collections::BTreeMap; use std::ops::{Deref, DerefMut}; - -use ::value::Value; -use lookup::OwnedValuePath; -use vrl::state::TypeState; -use vrl::{diagnostic::Label, prelude::*}; +use vrl::diagnostic::Label; +use vrl::path::OwnedValuePath; +use vrl::stdlib::prelude::*; #[derive(Debug, Default, Clone)] pub struct MeaningList(pub BTreeMap); diff --git a/lib/vector-vrl/tests/Cargo.toml b/lib/vector-vrl/tests/Cargo.toml index 5e7b683d4c42d..1f157a9aae016 100644 --- a/lib/vector-vrl/tests/Cargo.toml +++ b/lib/vector-vrl/tests/Cargo.toml @@ -7,11 +7,11 @@ publish = false [dependencies] enrichment = { path = "../../enrichment" } -lookup = { package = "lookup", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } -stdlib = { package = "vrl-stdlib", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } -vrl = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } -vrl-tests = { package = "vrl-tests", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" } +lookup = { package = "path", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +stdlib = { package = "vrl-stdlib", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +vrl = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +vrl-tests = { package = "vrl-tests", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } vector-vrl-functions = { path = "../../vector-vrl/functions" } ansi_term = "0.12" diff --git a/lib/vector-vrl/tests/src/main.rs b/lib/vector-vrl/tests/src/main.rs index 4b9a3e23b8984..e4c4dfe4b9583 100644 --- a/lib/vector-vrl/tests/src/main.rs +++ b/lib/vector-vrl/tests/src/main.rs @@ -9,7 +9,8 @@ use vrl_tests::{get_tests_from_functions, run_tests, Test, TestConfig}; use chrono_tz::Tz; use clap::Parser; use glob::glob; -use vrl::{CompileConfig, TimeZone, VrlRuntime}; +use vrl::compiler::{CompileConfig, VrlRuntime}; +use vrl::core::TimeZone; #[cfg(not(target_env = "msvc"))] #[global_allocator] diff --git a/lib/vector-vrl/web-playground/Cargo.toml b/lib/vector-vrl/web-playground/Cargo.toml index 49cab59700529..0c4ac0b2a4045 100644 --- a/lib/vector-vrl/web-playground/Cargo.toml +++ b/lib/vector-vrl/web-playground/Cargo.toml @@ -10,9 +10,9 @@ crate-type = ["cdylib"] [dependencies] wasm-bindgen = "0.2" -vrl = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", default-features = false} -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", default-features = false} -stdlib = { package = "vrl-stdlib", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0"} +vrl = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", default-features = false} +value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", default-features = false} +stdlib = { package = "vrl-stdlib", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3"} serde = { version = "1.0", features = ["derive"] } serde-wasm-bindgen = "0.5" gloo-utils = { version = "0.1", features = ["serde"] } diff --git a/lib/vector-vrl/web-playground/src/lib.rs b/lib/vector-vrl/web-playground/src/lib.rs index a3c0a067c5e2e..4f1fbcecc212b 100644 --- a/lib/vector-vrl/web-playground/src/lib.rs +++ b/lib/vector-vrl/web-playground/src/lib.rs @@ -1,11 +1,12 @@ use ::value::Value; use gloo_utils::format::JsValueSerdeExt; use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; use value::Secrets; +use vrl::compiler::runtime::{Runtime, Terminate}; +use vrl::compiler::{CompileConfig, TargetValue, TypeState}; use vrl::diagnostic::DiagnosticList; -use vrl::state::TypeState; -use vrl::{diagnostic::Formatter, prelude::BTreeMap, CompileConfig, Runtime}; -use vrl::{TargetValue, Terminate, TimeZone}; +use vrl::diagnostic::Formatter; use wasm_bindgen::prelude::*; #[derive(Serialize, Deserialize)] diff --git a/lib/vrl/value/Cargo.toml b/lib/vrl/value/Cargo.toml index a675ff4fcba41..d9e51dbe25ba5 100644 --- a/lib/vrl/value/Cargo.toml +++ b/lib/vrl/value/Cargo.toml @@ -11,7 +11,7 @@ publish = false bytes = { version = "1.4.0", default-features = false, features = ["serde"] } chrono = { version = "0.4.19", default-features = false, features = ["serde", "std"] } lookup = { path = "../lookup", default-features = false } -ordered-float = { version = "3.6.0", default-features = false } +ordered-float = { version = "3.7.0", default-features = false } regex = { version = "1.7.2", default-features = false, features = ["std", "perf"]} snafu = { version = "0.7.4", default-features = false } tracing = { version = "0.1.34", default-features = false, features = ["attributes"] } diff --git a/src/conditions/vrl.rs b/src/conditions/vrl.rs index 6105e459cb058..4d44478ee1c4c 100644 --- a/src/conditions/vrl.rs +++ b/src/conditions/vrl.rs @@ -2,7 +2,9 @@ use value::Value; use vector_common::TimeZone; use vector_config::configurable_component; use vector_core::compile_vrl; -use vrl::{diagnostic::Formatter, CompilationResult, CompileConfig, Program, Runtime, VrlRuntime}; +use vrl::compiler::runtime::{Runtime, RuntimeResult, Terminate}; +use vrl::compiler::{CompilationResult, CompileConfig, Program, TypeState, VrlRuntime}; +use vrl::diagnostic::Formatter; use crate::event::TargetEvents; use crate::{ @@ -44,7 +46,7 @@ impl ConditionalConfig for VrlConfig { .chain(vector_vrl_functions::all()) .collect::>(); - let state = vrl::state::TypeState::default(); + let state = TypeState::default(); let mut config = CompileConfig::default(); config.set_custom(enrichment_tables.clone()); @@ -81,7 +83,7 @@ pub struct Vrl { } impl Vrl { - fn run(&self, event: Event) -> (Event, vrl::RuntimeResult) { + fn run(&self, event: Event) -> (Event, RuntimeResult) { let mut target = VrlTarget::new(event, self.program.info(), false); // TODO: use timezone from remap config let timezone = TimeZone::default(); @@ -117,7 +119,7 @@ impl Conditional for Vrl { let (event, result) = self.run(event); let value_result = result.map_err(|err| match err { - vrl::Terminate::Abort(err) => { + Terminate::Abort(err) => { let err = Formatter::new( &self.source, vrl::diagnostic::Diagnostic::from( @@ -128,7 +130,7 @@ impl Conditional for Vrl { .to_string(); format!("source execution aborted: {}", err) } - vrl::Terminate::Error(err) => { + Terminate::Error(err) => { let err = Formatter::new( &self.source, vrl::diagnostic::Diagnostic::from( diff --git a/src/sinks/aws_cloudwatch_logs/service.rs b/src/sinks/aws_cloudwatch_logs/service.rs index b22f54ce29d3f..505de36f77e8d 100644 --- a/src/sinks/aws_cloudwatch_logs/service.rs +++ b/src/sinks/aws_cloudwatch_logs/service.rs @@ -24,7 +24,6 @@ use tower::{ }; use vector_common::request_metadata::MetaDescriptive; use vector_core::{internal_event::CountByteSize, stream::DriverResponse}; -use vrl::prelude::fmt::Debug; use crate::{ event::EventStatus, diff --git a/src/sources/aws_kinesis_firehose/handlers.rs b/src/sources/aws_kinesis_firehose/handlers.rs index a454dd5140033..054f91af58a72 100644 --- a/src/sources/aws_kinesis_firehose/handlers.rs +++ b/src/sources/aws_kinesis_firehose/handlers.rs @@ -20,7 +20,7 @@ use vector_core::{ event::BatchNotifier, EstimatedJsonEncodedSizeOf, }; -use vrl::SecretTarget; +use vrl::compiler::SecretTarget; use warp::reject; use super::{ diff --git a/src/transforms/metric_to_log.rs b/src/transforms/metric_to_log.rs index 959d6e6f31140..34b412c325506 100644 --- a/src/transforms/metric_to_log.rs +++ b/src/transforms/metric_to_log.rs @@ -3,13 +3,12 @@ use codecs::MetricTagValues; use lookup::lookup_v2::parse_value_path; use lookup::{event_path, owned_value_path, path, PathPrefix}; use serde_json::Value; -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use value::kind::Collection; use value::Kind; use vector_common::TimeZone; use vector_config::configurable_component; use vector_core::config::LogNamespace; -use vrl::prelude::BTreeMap; use crate::config::OutputId; use crate::{ diff --git a/src/transforms/remap.rs b/src/transforms/remap.rs index 1872f4e49d9db..3661969d379b0 100644 --- a/src/transforms/remap.rs +++ b/src/transforms/remap.rs @@ -20,12 +20,10 @@ use vector_core::compile_vrl; use vector_core::config::LogNamespace; use vector_core::schema::Definition; use vector_vrl_functions::set_semantic_meaning::MeaningList; -use vrl::prelude::state::TypeState; -use vrl::{ - diagnostic::{Formatter, Note}, - prelude::{DiagnosticMessage, ExpressionError}, - CompileConfig, Program, Runtime, Terminate, VrlRuntime, -}; +use vrl::compiler::runtime::{Runtime, Terminate}; +use vrl::compiler::state::ExternalEnv; +use vrl::compiler::{CompileConfig, ExpressionError, Function, Program, TypeState, VrlRuntime}; +use vrl::diagnostic::{DiagnosticMessage, Formatter, Note}; use crate::config::OutputId; use crate::{ @@ -143,12 +141,7 @@ impl RemapConfig { &self, enrichment_tables: enrichment::TableRegistry, merged_schema_definition: schema::Definition, - ) -> Result<( - vrl::Program, - String, - Vec>, - CompileConfig, - )> { + ) -> Result<(Program, String, Vec>, CompileConfig)> { let source = match (&self.source, &self.file) { (Some(source), None) => source.to_owned(), (None, Some(path)) => { @@ -170,7 +163,7 @@ impl RemapConfig { let state = TypeState { local: Default::default(), - external: vrl::state::ExternalEnv::new_with_kind( + external: ExternalEnv::new_with_kind( merged_schema_definition.event_kind().clone(), merged_schema_definition.metadata_kind().clone(), ), From da13fb3f2b8cad9fe582ed10bd5d42443f591f88 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Tue, 2 May 2023 20:23:32 -0400 Subject: [PATCH 02/23] save --- Cargo.lock | 13 --------- Cargo.toml | 14 ++++----- lib/codecs/Cargo.toml | 2 +- lib/enrichment/Cargo.toml | 2 +- .../src/find_enrichment_table_records.rs | 4 +-- .../src/get_enrichment_table_record.rs | 11 +++---- lib/enrichment/src/tables.rs | 3 +- lib/enrichment/src/test_util.rs | 2 +- lib/opentelemetry-proto/Cargo.toml | 2 +- lib/prometheus-parser/Cargo.toml | 2 +- lib/vector-common/Cargo.toml | 4 +-- lib/vector-config/Cargo.toml | 4 +-- lib/vector-core/Cargo.toml | 6 ++-- lib/vector-core/src/event/log_event.rs | 2 +- lib/vector-core/src/event/vrl_target.rs | 18 +++++------- lib/vector-lookup/Cargo.toml | 2 +- lib/vector-vrl/cli/Cargo.toml | 4 +-- lib/vector-vrl/functions/Cargo.toml | 2 +- lib/vector-vrl/tests/Cargo.toml | 10 +++---- lib/vector-vrl/web-playground/Cargo.toml | 6 ++-- lib/vector-vrl/web-playground/src/lib.rs | 5 ++-- src/config/enterprise.rs | 9 ++++-- src/sources/opentelemetry/tests.rs | 25 ++++++++-------- src/sources/socket/mod.rs | 29 ++++++++++--------- 24 files changed, 85 insertions(+), 96 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5e404d8b4182e..c1d5c3ad2f9fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2583,7 +2583,6 @@ checksum = "8d7439c3735f405729d52c3fbbe4de140eaf938a1fe47d227c27f8254d4302a5" [[package]] name = "datadog-filter" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "datadog-search-syntax", "dyn-clone", @@ -2593,7 +2592,6 @@ dependencies = [ [[package]] name = "datadog-grok" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "bytes 1.4.0", "chrono", @@ -2617,7 +2615,6 @@ dependencies = [ [[package]] name = "datadog-search-syntax" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "itertools", "once_cell", @@ -5756,7 +5753,6 @@ checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" [[package]] name = "path" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "once_cell", "regex", @@ -9060,7 +9056,6 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "async-graphql", "bytes 1.4.0", @@ -9642,7 +9637,6 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vrl" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "datadog-filter", "datadog-grok", @@ -9659,7 +9653,6 @@ dependencies = [ [[package]] name = "vrl-cli" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "clap 4.1.14", "exitcode", @@ -9682,7 +9675,6 @@ dependencies = [ [[package]] name = "vrl-compiler" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "anymap", "bytes 1.4.0", @@ -9705,7 +9697,6 @@ dependencies = [ [[package]] name = "vrl-core" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "bytes 1.4.0", "chrono", @@ -9725,7 +9716,6 @@ dependencies = [ [[package]] name = "vrl-diagnostic" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "codespan-reporting", "termcolor", @@ -9734,7 +9724,6 @@ dependencies = [ [[package]] name = "vrl-parser" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "lalrpop", "lalrpop-util", @@ -9748,7 +9737,6 @@ dependencies = [ [[package]] name = "vrl-stdlib" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "aes", "base16", @@ -9809,7 +9797,6 @@ dependencies = [ [[package]] name = "vrl-tests" version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=8d27cd2cdd3#8d27cd2cdd3ba291bdd6fe1271d87ec1631a181e" dependencies = [ "ansi_term", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 622cfd981df12..9a5db1552ef3a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,7 +126,7 @@ portpicker = { path = "lib/portpicker" } prometheus-parser = { path = "lib/prometheus-parser", optional = true } opentelemetry-proto = { path = "lib/opentelemetry-proto", optional = true } tracing-limit = { path = "lib/tracing-limit" } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value" } vector-api-client = { path = "lib/vector-api-client", optional = true } vector-buffers = { path = "lib/vector-buffers", default-features = false } vector-common = { path = "lib/vector-common" } @@ -135,7 +135,7 @@ vector-config-common = { path = "lib/vector-config-common" } vector-config-macros = { path = "lib/vector-config-macros" } vector-core = { path = "lib/vector-core", default-features = false, features = ["vrl"] } vector-vrl-functions = { path = "lib/vector-vrl/functions" } -vrl-cli = { package = "vrl-cli", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", optional = true } +vrl-cli = { package = "vrl-cli", path = "/home/nathan_fox/repos/vrl/lib/cli", optional = true } loki-logproto = { path = "lib/loki-logproto", optional = true } # Tokio / Futures @@ -223,14 +223,14 @@ number_prefix = { version = "0.4.0", default-features = false, features = ["std" tui = { version = "0.19.0", optional = true, default-features = false, features = ["crossterm"] } # Datadog Pipelines -datadog-filter = { package = "datadog-filter", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } -datadog-search-syntax = { package = "datadog-search-syntax", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +datadog-filter = { package = "datadog-filter", path = "/home/nathan_fox/repos/vrl/lib/datadog/filter" } +datadog-search-syntax = { package = "datadog-search-syntax", path = "/home/nathan_fox/repos/vrl/lib/datadog/search-syntax" } hex = { version = "0.4.3", default-features = false, optional = true } sha2 = { version = "0.10.6", default-features = false, optional = true } # VRL Lang -vrl = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } -vrl-stdlib = { package = "vrl-stdlib", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +vrl = { package = "vrl", path = "/home/nathan_fox/repos/vrl" } +vrl-stdlib = { package = "vrl-stdlib", path = "/home/nathan_fox/repos/vrl/lib/stdlib" } # External libs arc-swap = { version = "1.6", default-features = false, optional = true } @@ -360,7 +360,7 @@ test-generator = "0.3.1" tokio-test = "0.4.2" tokio = { version = "1.28.0", features = ["test-util"] } tower-test = "0.4.0" -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", features = ["test"] } +value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value", features = ["test"] } vector-core = { path = "lib/vector-core", default-features = false, features = ["vrl", "test"] } wiremock = "0.5.18" zstd = { version = "0.12.3", default-features = false } diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 4e9cdc2240465..cc23cf17f0c0a 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -25,7 +25,7 @@ snafu = { version = "0.7.4", default-features = false, features = ["futures"] } syslog_loose = { version = "0.18", default-features = false, optional = true } tokio-util = { version = "0.7", default-features = false, features = ["codec"] } tracing = { version = "0.1", default-features = false } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", default-features = false } +value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value", default-features = false } vector-common = { path = "../vector-common", default-features = false } vector-config = { path = "../vector-config", default-features = false } vector-config-common = { path = "../vector-config-common", default-features = false } diff --git a/lib/enrichment/Cargo.toml b/lib/enrichment/Cargo.toml index 471ffaea26009..4b28631c10b41 100644 --- a/lib/enrichment/Cargo.toml +++ b/lib/enrichment/Cargo.toml @@ -10,4 +10,4 @@ arc-swap = { version = "1.6.0", default-features = false } dyn-clone = { version = "1.0.11", default-features = false } chrono = { version = "0.4.19", default-features = false } vector-common = { path = "../vector-common", default-features = false, features = [ "btreemap", "conversion", "serde" ] } -vrl = { git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +vrl = { path = "/home/nathan_fox/repos/vrl" } diff --git a/lib/enrichment/src/find_enrichment_table_records.rs b/lib/enrichment/src/find_enrichment_table_records.rs index 254bc983970d7..9e46582a95d57 100644 --- a/lib/enrichment/src/find_enrichment_table_records.rs +++ b/lib/enrichment/src/find_enrichment_table_records.rs @@ -191,8 +191,8 @@ impl FunctionExpression for FindEnrichmentTableRecordsFn { #[cfg(test)] mod tests { - use ::value::Secrets; use vector_common::TimeZone; + use vrl::compiler::state::RuntimeState; use vrl::compiler::TargetValue; use vrl::value::Secrets; @@ -221,7 +221,7 @@ mod tests { metadata: value!({}), secrets: Secrets::new(), }; - let mut runtime_state = vrl::state::Runtime::default(); + let mut runtime_state = RuntimeState::default(); let mut ctx = Context::new(&mut target, &mut runtime_state, &tz); registry.finish_load(); diff --git a/lib/enrichment/src/get_enrichment_table_record.rs b/lib/enrichment/src/get_enrichment_table_record.rs index 0ad8299469231..da590ae5749f6 100644 --- a/lib/enrichment/src/get_enrichment_table_record.rs +++ b/lib/enrichment/src/get_enrichment_table_record.rs @@ -184,9 +184,10 @@ impl FunctionExpression for GetEnrichmentTableRecordFn { #[cfg(test)] mod tests { - use value::Secrets; use vector_common::TimeZone; - use vrl::TargetValue; + use vrl::compiler::state::RuntimeState; + use vrl::compiler::TargetValue; + use vrl::value::Secrets; use super::*; use crate::test_util::get_table_registry; @@ -210,16 +211,16 @@ mod tests { let object: Value = BTreeMap::new().into(); let mut target = TargetValue { value: object, - metadata: vrl::value!({}), + metadata: value!({}), secrets: Secrets::new(), }; - let mut runtime_state = vrl::state::Runtime::default(); + let mut runtime_state = RuntimeState::default(); let mut ctx = Context::new(&mut target, &mut runtime_state, &tz); registry.finish_load(); let got = func.resolve(&mut ctx); - assert_eq!(Ok(vrl::value! ({ "field": "result" })), got); + assert_eq!(Ok(value! ({ "field": "result" })), got); } } diff --git a/lib/enrichment/src/tables.rs b/lib/enrichment/src/tables.rs index 44dbd237a268a..fb43c3a7d0205 100644 --- a/lib/enrichment/src/tables.rs +++ b/lib/enrichment/src/tables.rs @@ -272,10 +272,9 @@ fn fmt_enrichment_table( #[cfg(test)] mod tests { - use value::Value; - use super::*; use crate::test_util::DummyEnrichmentTable; + use vrl::value::Value; #[test] fn tables_loaded() { diff --git a/lib/enrichment/src/test_util.rs b/lib/enrichment/src/test_util.rs index 52708a824900a..cf634f2102696 100644 --- a/lib/enrichment/src/test_util.rs +++ b/lib/enrichment/src/test_util.rs @@ -3,7 +3,7 @@ use std::{ sync::{Arc, Mutex}, }; -use value::Value; +use vrl::value::Value; use crate::{Case, Condition, IndexHandle, Table, TableRegistry}; diff --git a/lib/opentelemetry-proto/Cargo.toml b/lib/opentelemetry-proto/Cargo.toml index 6f0e87d9b2088..23425b11c5547 100644 --- a/lib/opentelemetry-proto/Cargo.toml +++ b/lib/opentelemetry-proto/Cargo.toml @@ -17,5 +17,5 @@ lookup = { package = "vector-lookup", path = "../vector-lookup", default-feature ordered-float = { version = "3.7.0", default-features = false } prost = { version = "0.11", default-features = false, features = ["std"] } tonic = { version = "0.9", default-features = false, features = ["codegen", "gzip", "prost", "tls", "tls-roots", "transport"] } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value" } vector-core = { path = "../vector-core", default-features = false } diff --git a/lib/prometheus-parser/Cargo.toml b/lib/prometheus-parser/Cargo.toml index 9ff0f381f71f2..d47915c488768 100644 --- a/lib/prometheus-parser/Cargo.toml +++ b/lib/prometheus-parser/Cargo.toml @@ -16,7 +16,7 @@ prost = "0.11" prost-types = "0.11" snafu = { version = "0.7" } vector-common = { path = "../vector-common", features = ["btreemap"] } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value" } [build-dependencies] prost-build = "0.11" diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index de3ffa6135136..beab1a5706cec 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -62,8 +62,8 @@ snafu = { version = "0.7", optional = true } stream-cancel = { version = "0.8.1", default-features = false } tokio = { version = "1.28.0", default-features = false, features = ["macros", "time"] } tracing = { version = "0.1.34", default-features = false } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", features = ["json", "arbitrary"] } -vrl-core = { package = "vrl-core", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", default-features = false } +value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value", features = ["json", "arbitrary"] } +vrl-core = { package = "vrl-core", path = "/home/nathan_fox/repos/vrl/lib/core", default-features = false } vector-config = { path = "../vector-config" } vector-config-common = { path = "../vector-config-common" } vector-config-macros = { path = "../vector-config-macros" } diff --git a/lib/vector-config/Cargo.toml b/lib/vector-config/Cargo.toml index 6d8f8d86c12bf..120f5d0b49a63 100644 --- a/lib/vector-config/Cargo.toml +++ b/lib/vector-config/Cargo.toml @@ -26,8 +26,8 @@ snafu = { version = "0.7.4", default-features = false } toml = { version = "0.7.3", default-features = false } tracing = { version = "0.1.34", default-features = false } url = { version = "2.3.1", default-features = false, features = ["serde"] } -vrl-compiler = { package = "vrl-compiler", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } -vrl-core = { package = "vrl-core", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", default-features = false, features = ["serde"] } +vrl-compiler = { package = "vrl-compiler", path = "/home/nathan_fox/repos/vrl/lib/compiler" } +vrl-core = { package = "vrl-core", path = "/home/nathan_fox/repos/vrl/lib/core", default-features = false, features = ["serde"] } vector-config-common = { path = "../vector-config-common" } vector-config-macros = { path = "../vector-config-macros" } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 3fb4a03888a1a..6d7554db94dc6 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -60,7 +60,7 @@ tracing-subscriber = { version = "0.3.16", default-features = false, features = typetag = { version = "0.2.7", default-features = false } twox-hash = { version = "1.6.3", default-features = false } url = { version = "2", default-features = false } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", default-features = false, features = ["lua", "toml", "json", "api"] } +value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value", default-features = false, features = ["lua", "toml", "json", "api"] } vector-buffers = { path = "../vector-buffers", default-features = false } vector-common = { path = "../vector-common" } vector-config = { path = "../vector-config" } @@ -68,7 +68,7 @@ vector-config-common = { path = "../vector-config-common" } vector-config-macros = { path = "../vector-config-macros" } # Rename to "vrl" once we use a release with stable `-Z namespaced-features`: # https://doc.rust-lang.org/cargo/reference/unstable.html#namespaced-features -vrl-lib = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +vrl-lib = { package = "vrl", path = "/home/nathan_fox/repos/vrl" } [target.'cfg(target_os = "macos")'.dependencies] security-framework = "2.8.2" @@ -97,7 +97,7 @@ rand = "0.8.5" rand_distr = "0.4.3" tracing-subscriber = { version = "0.3.16", default-features = false, features = ["env-filter", "fmt", "ansi", "registry"] } vector-common = { path = "../vector-common", default-features = false, features = ["test"] } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", default-features = false, features = ["lua", "toml", "json", "api", "arbitrary", "test"] } +value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value", default-features = false, features = ["lua", "toml", "json", "api", "arbitrary", "test"] } [features] api = ["dep:async-graphql", "value/api"] diff --git a/lib/vector-core/src/event/log_event.rs b/lib/vector-core/src/event/log_event.rs index ff2fa485ca49b..e775a3235fb9e 100644 --- a/lib/vector-core/src/event/log_event.rs +++ b/lib/vector-core/src/event/log_event.rs @@ -694,7 +694,7 @@ mod test { use super::*; use crate::test_util::open_fixture; use lookup::event_path; - use vrl_lib::value; + use vrl_lib::compiler::value; // The following two tests assert that renaming a key has no effect if the // keys are equivalent, whether the key exists in the log or not. diff --git a/lib/vector-core/src/event/vrl_target.rs b/lib/vector-core/src/event/vrl_target.rs index b31216b202766..d320d068f472c 100644 --- a/lib/vector-core/src/event/vrl_target.rs +++ b/lib/vector-core/src/event/vrl_target.rs @@ -592,7 +592,6 @@ mod test { use lookup::owned_value_path; use similar_asserts::assert_eq; use value::btreemap; - use vrl_lib::Target; use super::super::MetricValue; use super::*; @@ -647,7 +646,7 @@ mod test { let path = OwnedTargetPath::event(path); assert_eq!( - vrl_lib::Target::target_get(&target, &path).map(Option::<&Value>::cloned), + Target::target_get(&target, &path).map(Option::<&Value>::cloned), expect ); } @@ -755,11 +754,11 @@ mod test { let path = OwnedTargetPath::event(path); assert_eq!( - vrl_lib::Target::target_insert(&mut target, &path, value.clone()), + Target::target_insert(&mut target, &path, value.clone()), result ); assert_eq!( - vrl_lib::Target::target_get(&target, &path).map(Option::<&Value>::cloned), + Target::target_get(&target, &path).map(Option::<&Value>::cloned), Ok(Some(value)) ); assert_eq!( @@ -850,16 +849,14 @@ mod test { }; let mut target = VrlTarget::new(Event::Log(LogEvent::from(object)), &info, false); let path = OwnedTargetPath::event(path); - let removed = vrl_lib::Target::target_get(&target, &path) - .unwrap() - .cloned(); + let removed = Target::target_get(&target, &path).unwrap().cloned(); assert_eq!( - vrl_lib::Target::target_remove(&mut target, &path, compact), + Target::target_remove(&mut target, &path, compact), Ok(removed) ); assert_eq!( - vrl_lib::Target::target_get(&target, &OwnedTargetPath::event_root()) + Target::target_get(&target, &OwnedTargetPath::event_root()) .map(Option::<&Value>::cloned), Ok(expect) ); @@ -914,8 +911,7 @@ mod test { false, ); - ::vrl_lib::Target::target_insert(&mut target, &OwnedTargetPath::event_root(), value) - .unwrap(); + Target::target_insert(&mut target, &OwnedTargetPath::event_root(), value).unwrap(); assert_eq!( match target.into_events() { diff --git a/lib/vector-lookup/Cargo.toml b/lib/vector-lookup/Cargo.toml index 302c81c3da6e7..61d3068eb67b3 100644 --- a/lib/vector-lookup/Cargo.toml +++ b/lib/vector-lookup/Cargo.toml @@ -10,4 +10,4 @@ license = "MPL-2.0" serde = { version = "1.0.159", default-features = false, features = ["derive", "alloc"] } vector-config = { path = "../vector-config" } vector-config-macros = { path = "../vector-config-macros" } -vrl = { git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", default-features = false, features = ["path"] } +vrl = { path = "/home/nathan_fox/repos/vrl", default-features = false, features = ["path"] } diff --git a/lib/vector-vrl/cli/Cargo.toml b/lib/vector-vrl/cli/Cargo.toml index 9d871755f0e78..bda7a6a42f7b6 100644 --- a/lib/vector-vrl/cli/Cargo.toml +++ b/lib/vector-vrl/cli/Cargo.toml @@ -7,7 +7,7 @@ publish = false license = "MPL-2.0" [dependencies] -vrl-cli = { package = "vrl-cli", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +vrl-cli = { package = "vrl-cli", path = "/home/nathan_fox/repos/vrl/lib/cli" } clap = { version = "4.1.14", features = ["derive"] } -vrl-stdlib = { package = "vrl-stdlib", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +vrl-stdlib = { package = "vrl-stdlib", path = "/home/nathan_fox/repos/vrl/lib/stdlib" } vector-vrl-functions = { path = "../functions" } diff --git a/lib/vector-vrl/functions/Cargo.toml b/lib/vector-vrl/functions/Cargo.toml index cdd91b3bfd6c9..84e324f9533c3 100644 --- a/lib/vector-vrl/functions/Cargo.toml +++ b/lib/vector-vrl/functions/Cargo.toml @@ -7,4 +7,4 @@ publish = false license = "MPL-2.0" [dependencies] -vrl = { git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +vrl = { path = "/home/nathan_fox/repos/vrl" } diff --git a/lib/vector-vrl/tests/Cargo.toml b/lib/vector-vrl/tests/Cargo.toml index 1f157a9aae016..2726d3e22cef8 100644 --- a/lib/vector-vrl/tests/Cargo.toml +++ b/lib/vector-vrl/tests/Cargo.toml @@ -7,11 +7,11 @@ publish = false [dependencies] enrichment = { path = "../../enrichment" } -lookup = { package = "path", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } -stdlib = { package = "vrl-stdlib", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } -vrl = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } -vrl-tests = { package = "vrl-tests", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3" } +lookup = { package = "path", path = "/home/nathan_fox/repos/vrl/lib/path" } +stdlib = { package = "vrl-stdlib", path = "/home/nathan_fox/repos/vrl/lib/stdlib" } +vrl = { package = "vrl", path = "/home/nathan_fox/repos/vrl" } +value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value" } +vrl-tests = { package = "vrl-tests", path = "/home/nathan_fox/repos/vrl/lib/tests" } vector-vrl-functions = { path = "../../vector-vrl/functions" } ansi_term = "0.12" diff --git a/lib/vector-vrl/web-playground/Cargo.toml b/lib/vector-vrl/web-playground/Cargo.toml index 0c4ac0b2a4045..3804b3cdaa3c9 100644 --- a/lib/vector-vrl/web-playground/Cargo.toml +++ b/lib/vector-vrl/web-playground/Cargo.toml @@ -10,9 +10,9 @@ crate-type = ["cdylib"] [dependencies] wasm-bindgen = "0.2" -vrl = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", default-features = false} -value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3", default-features = false} -stdlib = { package = "vrl-stdlib", git = "https://github.com/vectordotdev/vrl", rev = "8d27cd2cdd3"} +vrl = { package = "vrl", path = "/home/nathan_fox/repos/vrl", default-features = false} +value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value", default-features = false} +stdlib = { package = "vrl-stdlib", path = "/home/nathan_fox/repos/vrl/lib/stdlib"} serde = { version = "1.0", features = ["derive"] } serde-wasm-bindgen = "0.5" gloo-utils = { version = "0.1", features = ["serde"] } diff --git a/lib/vector-vrl/web-playground/src/lib.rs b/lib/vector-vrl/web-playground/src/lib.rs index 4f1fbcecc212b..403ea5bf1a6c7 100644 --- a/lib/vector-vrl/web-playground/src/lib.rs +++ b/lib/vector-vrl/web-playground/src/lib.rs @@ -4,7 +4,8 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use value::Secrets; use vrl::compiler::runtime::{Runtime, Terminate}; -use vrl::compiler::{CompileConfig, TargetValue, TypeState}; +use vrl::compiler::{compile_with_state, CompileConfig, TargetValue, TypeState}; +use vrl::core::TimeZone; use vrl::diagnostic::DiagnosticList; use vrl::diagnostic::Formatter; use wasm_bindgen::prelude::*; @@ -88,7 +89,7 @@ fn compile(mut input: Input) -> Result { secrets: Secrets::new(), }; - let program = match vrl::compile_with_state(&input.program, &functions, &state, config) { + let program = match compile_with_state(&input.program, &functions, &state, config) { Ok(program) => program, Err(diagnostics) => return Err(VrlDiagnosticResult::new(&input.program, diagnostics)), }; diff --git a/src/config/enterprise.rs b/src/config/enterprise.rs index ba70fca806934..28598c1a7e8cf 100644 --- a/src/config/enterprise.rs +++ b/src/config/enterprise.rs @@ -833,8 +833,11 @@ mod test { use indexmap::IndexMap; use tokio::time::sleep; use value::btreemap; + use value::kind::Collection; use value::Kind; use vector_core::config::proxy::ProxyConfig; + use vrl::compiler::state::ExternalEnv; + use vrl::compiler::{compile, compile_with_external, CompileConfig}; use vrl::prelude::Collection; use vrl::CompileConfig; use wiremock::{matchers, Mock, MockServer, ResponseTemplate}; @@ -1003,13 +1006,13 @@ mod test { // We need to set up some state here to inform the VRL compiler that // .tags is an object and merge() is thus a safe operation (mimicking // the environment this code will actually run in). - let state = vrl::state::ExternalEnv::new_with_kind( + let state = ExternalEnv::new_with_kind( Kind::object(btreemap! { "tags" => Kind::object(BTreeMap::new()), }), Kind::object(Collection::empty()), ); - assert!(vrl::compile_with_external( + assert!(compile_with_external( vrl.as_str(), vrl_stdlib::all().as_ref(), &state, @@ -1031,6 +1034,6 @@ mod test { vrl, r#". = merge(., {"pull_request":"1234","replica":"abcd","variant":"baseline"}, deep: true)"# ); - assert!(vrl::compile(vrl.as_str(), vrl_stdlib::all().as_ref()).is_ok()); + assert!(compile(vrl.as_str(), vrl_stdlib::all().as_ref()).is_ok()); } } diff --git a/src/sources/opentelemetry/tests.rs b/src/sources/opentelemetry/tests.rs index 689fa92a6c593..1093db34d08e3 100644 --- a/src/sources/opentelemetry/tests.rs +++ b/src/sources/opentelemetry/tests.rs @@ -12,6 +12,7 @@ use similar_asserts::assert_eq; use std::collections::BTreeMap; use tonic::Request; use vector_core::config::LogNamespace; +use vrl::compiler::value; use crate::{ config::{SourceConfig, SourceContext}, @@ -110,12 +111,12 @@ async fn receive_grpc_logs_vector_namespace() { let event = output.pop().unwrap(); schema_definitions.unwrap().assert_valid_for_event(&event); - assert_eq!(event.as_log().get(".").unwrap(), &vrl::value!("log body")); + assert_eq!(event.as_log().get(".").unwrap(), &value!("log body")); let meta = event.as_log().metadata().value(); assert_eq!( meta.get(path!("vector", "source_type")).unwrap(), - &vrl::value!(OpentelemetryConfig::NAME) + &value!(OpentelemetryConfig::NAME) ); assert!(meta .get(path!("vector", "ingest_timestamp")) @@ -123,45 +124,45 @@ async fn receive_grpc_logs_vector_namespace() { .is_timestamp()); assert_eq!( meta.get(path!("opentelemetry", "resources")).unwrap(), - &vrl::value!({res_key: "res_val"}) + &value!({res_key: "res_val"}) ); assert_eq!( meta.get(path!("opentelemetry", "attributes")).unwrap(), - &vrl::value!({attr_key: "attr_val"}) + &value!({attr_key: "attr_val"}) ); assert_eq!( meta.get(path!("opentelemetry", "trace_id")).unwrap(), - &vrl::value!("4ac52aadf321c2e531db005df08792f5") + &value!("4ac52aadf321c2e531db005df08792f5") ); assert_eq!( meta.get(path!("opentelemetry", "span_id")).unwrap(), - &vrl::value!("0b9e4bda2a55530d") + &value!("0b9e4bda2a55530d") ); assert_eq!( meta.get(path!("opentelemetry", "severity_text")).unwrap(), - &vrl::value!("info") + &value!("info") ); assert_eq!( meta.get(path!("opentelemetry", "severity_number")).unwrap(), - &vrl::value!(9) + &value!(9) ); assert_eq!( meta.get(path!("opentelemetry", "flags")).unwrap(), - &vrl::value!(4) + &value!(4) ); assert_eq!( meta.get(path!("opentelemetry", "observed_timestamp")) .unwrap(), - &vrl::value!(Utc.timestamp_nanos(2)) + &value!(Utc.timestamp_nanos(2)) ); assert_eq!( meta.get(path!("opentelemetry", "timestamp")).unwrap(), - &vrl::value!(Utc.timestamp_nanos(1)) + &value!(Utc.timestamp_nanos(1)) ); assert_eq!( meta.get(path!("opentelemetry", "dropped_attributes_count")) .unwrap(), - &vrl::value!(3) + &value!(3) ); }) .await; diff --git a/src/sources/socket/mod.rs b/src/sources/socket/mod.rs index 6ef2537916657..9b73579516ecf 100644 --- a/src/sources/socket/mod.rs +++ b/src/sources/socket/mod.rs @@ -343,8 +343,9 @@ mod test { task::JoinHandle, time::{timeout, Duration, Instant}, }; - use value::btreemap; use vector_core::event::EventContainer; + use vrl::compiler::value; + use vrl::compiler::value::btreemap; #[cfg(unix)] use { super::{unix::UnixConfig, Mode}, @@ -435,15 +436,15 @@ mod test { assert_eq!(log.value(), &"test".into()); assert_eq!( event_meta.get(path!("vector", "source_type")).unwrap(), - &vrl::value!(SocketConfig::NAME) + &value!(SocketConfig::NAME) ); assert_eq!( event_meta.get(path!(SocketConfig::NAME, "host")).unwrap(), - &vrl::value!(addr.ip().to_string()) + &value!(addr.ip().to_string()) ); assert_eq!( event_meta.get(path!(SocketConfig::NAME, "port")).unwrap(), - &vrl::value!(addr.port()) + &value!(addr.port()) ); }) .await; @@ -651,7 +652,7 @@ mod test { assert_eq!(log.value(), &"one line".into()); - let tls_meta: BTreeMap = btreemap!( + let tls_meta: BTreeMap = btreemap!( "subject" => "CN=localhost,OU=Vector,O=Datadog,L=New York,ST=New York,C=US" ); @@ -659,7 +660,7 @@ mod test { event_meta .get(path!(SocketConfig::NAME, "tls_client_metadata")) .unwrap(), - &vrl::value!(tls_meta.clone()) + &value!(tls_meta.clone()) ); let event = rx.next().await.unwrap(); @@ -672,7 +673,7 @@ mod test { event_meta .get(path!(SocketConfig::NAME, "tls_client_metadata")) .unwrap(), - &vrl::value!(tls_meta.clone()) + &value!(tls_meta.clone()) ); }) .await; @@ -1094,15 +1095,15 @@ mod test { assert_eq!(log.value(), &"test".into()); assert_eq!( event_meta.get(path!("vector", "source_type")).unwrap(), - &vrl::value!(SocketConfig::NAME) + &value!(SocketConfig::NAME) ); assert_eq!( event_meta.get(path!(SocketConfig::NAME, "host")).unwrap(), - &vrl::value!(from.ip().to_string()) + &value!(from.ip().to_string()) ); assert_eq!( event_meta.get(path!(SocketConfig::NAME, "port")).unwrap(), - &vrl::value!(from.port()) + &value!(from.port()) ); }) .await; @@ -1392,12 +1393,12 @@ mod test { assert_eq!( event_meta.get(path!("vector", "source_type")).unwrap(), - &vrl::value!(SocketConfig::NAME) + &value!(SocketConfig::NAME) ); assert_eq!( event_meta.get(path!(SocketConfig::NAME, "host")).unwrap(), - &vrl::value!(UNNAMED_SOCKET_HOST) + &value!(UNNAMED_SOCKET_HOST) ); }) .await; @@ -1523,11 +1524,11 @@ mod test { assert_eq!(1, events.len()); assert_eq!( event_meta.get(path!("vector", "source_type")).unwrap(), - &vrl::value!(SocketConfig::NAME) + &value!(SocketConfig::NAME) ); assert_eq!( event_meta.get(path!(SocketConfig::NAME, "host")).unwrap(), - &vrl::value!(UNNAMED_SOCKET_HOST) + &value!(UNNAMED_SOCKET_HOST) ); }) .await; From 0fbf4bac9b1fef64c424f2aa0806afb53c79eb51 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Tue, 2 May 2023 20:30:24 -0400 Subject: [PATCH 03/23] save --- src/config/enterprise.rs | 2 -- src/sinks/influxdb/logs.rs | 3 ++- src/sources/aws_kinesis_firehose/mod.rs | 11 ++++++----- src/sources/datadog_agent/tests.rs | 1 - src/sources/socket/mod.rs | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/config/enterprise.rs b/src/config/enterprise.rs index 28598c1a7e8cf..e9f7e24224c66 100644 --- a/src/config/enterprise.rs +++ b/src/config/enterprise.rs @@ -838,8 +838,6 @@ mod test { use vector_core::config::proxy::ProxyConfig; use vrl::compiler::state::ExternalEnv; use vrl::compiler::{compile, compile_with_external, CompileConfig}; - use vrl::prelude::Collection; - use vrl::CompileConfig; use wiremock::{matchers, Mock, MockServer, ResponseTemplate}; use super::{ diff --git a/src/sinks/influxdb/logs.rs b/src/sinks/influxdb/logs.rs index 93e61aed8c7dd..c2cbc70ed2ad4 100644 --- a/src/sinks/influxdb/logs.rs +++ b/src/sinks/influxdb/logs.rs @@ -883,6 +883,7 @@ mod integration_tests { use std::sync::Arc; use vector_core::config::{LegacyKey, LogNamespace}; use vector_core::event::{BatchNotifier, BatchStatus, Event, LogEvent}; + use vrl::compiler::value; use super::*; use crate::{ @@ -939,7 +940,7 @@ mod integration_tests { event2.insert("source_type", "file"); let mut namespaced_log = - LogEvent::from(vrl::value!("namespaced message")).with_batch_notifier(&batch); + LogEvent::from(value!("namespaced message")).with_batch_notifier(&batch); LogNamespace::Vector.insert_source_metadata( "file", &mut namespaced_log, diff --git a/src/sources/aws_kinesis_firehose/mod.rs b/src/sources/aws_kinesis_firehose/mod.rs index 198f14104d308..c14fdc3ce5003 100644 --- a/src/sources/aws_kinesis_firehose/mod.rs +++ b/src/sources/aws_kinesis_firehose/mod.rs @@ -255,6 +255,7 @@ mod tests { use similar_asserts::assert_eq; use tokio::time::{sleep, Duration}; use vector_common::assert_event_data_eq; + use vrl::compiler::value; use super::*; use crate::{ @@ -609,12 +610,12 @@ mod tests { let meta = log.metadata(); // event data, currently assumes default bytes deserializer - assert_eq!(log.value(), &vrl::value!(Bytes::from(expected.to_owned()))); + assert_eq!(log.value(), &value!(Bytes::from(expected.to_owned()))); // vector metadata assert_eq!( meta.value().get(path!("vector", "source_type")).unwrap(), - &vrl::value!("aws_kinesis_firehose") + &value!("aws_kinesis_firehose") ); assert!(meta .value() @@ -627,19 +628,19 @@ mod tests { meta.value() .get(path!("aws_kinesis_firehose", "request_id")) .unwrap(), - &vrl::value!(REQUEST_ID) + &value!(REQUEST_ID) ); assert_eq!( meta.value() .get(path!("aws_kinesis_firehose", "source_arn")) .unwrap(), - &vrl::value!(SOURCE_ARN) + &value!(SOURCE_ARN) ); assert_eq!( meta.value() .get(path!("aws_kinesis_firehose", "timestamp")) .unwrap(), - &vrl::value!(timestamp.trunc_subsecs(3)) + &value!(timestamp.trunc_subsecs(3)) ); } diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index d1d629c119e01..4c4a420ed2dc5 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -25,7 +25,6 @@ use vector_core::{ event::{metric::TagValue, MetricTags}, metric_tags, }; -use vrl::prelude::Collection; use crate::schema::Definition; use crate::{ diff --git a/src/sources/socket/mod.rs b/src/sources/socket/mod.rs index 9b73579516ecf..f1b4ced6d4cc7 100644 --- a/src/sources/socket/mod.rs +++ b/src/sources/socket/mod.rs @@ -345,7 +345,7 @@ mod test { }; use vector_core::event::EventContainer; use vrl::compiler::value; - use vrl::compiler::value::btreemap; + #[cfg(unix)] use { super::{unix::UnixConfig, Mode}, From 5313866169aab6bdc800a14001de97dea4b9cdc1 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Tue, 2 May 2023 23:51:39 -0400 Subject: [PATCH 04/23] save --- Cargo.toml | 14 ++++---- lib/codecs/Cargo.toml | 2 +- lib/enrichment/Cargo.toml | 2 +- lib/opentelemetry-proto/Cargo.toml | 2 +- lib/prometheus-parser/Cargo.toml | 2 +- lib/vector-common/Cargo.toml | 4 +-- lib/vector-config/Cargo.toml | 4 +-- lib/vector-core/Cargo.toml | 6 ++-- lib/vector-lookup/Cargo.toml | 2 +- lib/vector-vrl/cli/Cargo.toml | 4 +-- lib/vector-vrl/functions/Cargo.toml | 2 +- lib/vector-vrl/tests/Cargo.toml | 10 +++--- lib/vector-vrl/web-playground/Cargo.toml | 6 ++-- src/sources/datadog_agent/tests.rs | 1 + src/sources/docker_logs/tests.rs | 32 +++++++++---------- src/sources/exec/mod.rs | 29 +++++++++-------- src/sources/file.rs | 13 ++++---- .../file_descriptors/file_descriptor.rs | 7 ++-- src/sources/file_descriptors/stdin.rs | 7 ++-- src/sources/kafka.rs | 11 ++++--- src/sources/kubernetes_logs/parser/cri.rs | 15 +++++---- src/sources/kubernetes_logs/parser/docker.rs | 13 ++++---- src/sources/kubernetes_logs/parser/mod.rs | 5 +-- .../kubernetes_logs/parser/test_util.rs | 5 +-- src/sources/redis/mod.rs | 3 +- src/sources/socket/mod.rs | 3 +- 26 files changed, 108 insertions(+), 96 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9a5db1552ef3a..e399636b973bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,7 +126,7 @@ portpicker = { path = "lib/portpicker" } prometheus-parser = { path = "lib/prometheus-parser", optional = true } opentelemetry-proto = { path = "lib/opentelemetry-proto", optional = true } tracing-limit = { path = "lib/tracing-limit" } -value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value" } +value = { package = "value", path = "/home/nathan/repos/vrl/lib/value" } vector-api-client = { path = "lib/vector-api-client", optional = true } vector-buffers = { path = "lib/vector-buffers", default-features = false } vector-common = { path = "lib/vector-common" } @@ -135,7 +135,7 @@ vector-config-common = { path = "lib/vector-config-common" } vector-config-macros = { path = "lib/vector-config-macros" } vector-core = { path = "lib/vector-core", default-features = false, features = ["vrl"] } vector-vrl-functions = { path = "lib/vector-vrl/functions" } -vrl-cli = { package = "vrl-cli", path = "/home/nathan_fox/repos/vrl/lib/cli", optional = true } +vrl-cli = { package = "vrl-cli", path = "/home/nathan/repos/vrl/lib/cli", optional = true } loki-logproto = { path = "lib/loki-logproto", optional = true } # Tokio / Futures @@ -223,14 +223,14 @@ number_prefix = { version = "0.4.0", default-features = false, features = ["std" tui = { version = "0.19.0", optional = true, default-features = false, features = ["crossterm"] } # Datadog Pipelines -datadog-filter = { package = "datadog-filter", path = "/home/nathan_fox/repos/vrl/lib/datadog/filter" } -datadog-search-syntax = { package = "datadog-search-syntax", path = "/home/nathan_fox/repos/vrl/lib/datadog/search-syntax" } +datadog-filter = { package = "datadog-filter", path = "/home/nathan/repos/vrl/lib/datadog/filter" } +datadog-search-syntax = { package = "datadog-search-syntax", path = "/home/nathan/repos/vrl/lib/datadog/search-syntax" } hex = { version = "0.4.3", default-features = false, optional = true } sha2 = { version = "0.10.6", default-features = false, optional = true } # VRL Lang -vrl = { package = "vrl", path = "/home/nathan_fox/repos/vrl" } -vrl-stdlib = { package = "vrl-stdlib", path = "/home/nathan_fox/repos/vrl/lib/stdlib" } +vrl = { package = "vrl", path = "/home/nathan/repos/vrl" } +vrl-stdlib = { package = "vrl-stdlib", path = "/home/nathan/repos/vrl/lib/stdlib" } # External libs arc-swap = { version = "1.6", default-features = false, optional = true } @@ -360,7 +360,7 @@ test-generator = "0.3.1" tokio-test = "0.4.2" tokio = { version = "1.28.0", features = ["test-util"] } tower-test = "0.4.0" -value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value", features = ["test"] } +value = { package = "value", path = "/home/nathan/repos/vrl/lib/value", features = ["test"] } vector-core = { path = "lib/vector-core", default-features = false, features = ["vrl", "test"] } wiremock = "0.5.18" zstd = { version = "0.12.3", default-features = false } diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index cc23cf17f0c0a..3e3ef5c9ac6a9 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -25,7 +25,7 @@ snafu = { version = "0.7.4", default-features = false, features = ["futures"] } syslog_loose = { version = "0.18", default-features = false, optional = true } tokio-util = { version = "0.7", default-features = false, features = ["codec"] } tracing = { version = "0.1", default-features = false } -value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value", default-features = false } +value = { package = "value", path = "/home/nathan/repos/vrl/lib/value", default-features = false } vector-common = { path = "../vector-common", default-features = false } vector-config = { path = "../vector-config", default-features = false } vector-config-common = { path = "../vector-config-common", default-features = false } diff --git a/lib/enrichment/Cargo.toml b/lib/enrichment/Cargo.toml index 4b28631c10b41..19fd4b2706895 100644 --- a/lib/enrichment/Cargo.toml +++ b/lib/enrichment/Cargo.toml @@ -10,4 +10,4 @@ arc-swap = { version = "1.6.0", default-features = false } dyn-clone = { version = "1.0.11", default-features = false } chrono = { version = "0.4.19", default-features = false } vector-common = { path = "../vector-common", default-features = false, features = [ "btreemap", "conversion", "serde" ] } -vrl = { path = "/home/nathan_fox/repos/vrl" } +vrl = { path = "/home/nathan/repos/vrl" } diff --git a/lib/opentelemetry-proto/Cargo.toml b/lib/opentelemetry-proto/Cargo.toml index 23425b11c5547..02e5cc28cfb1f 100644 --- a/lib/opentelemetry-proto/Cargo.toml +++ b/lib/opentelemetry-proto/Cargo.toml @@ -17,5 +17,5 @@ lookup = { package = "vector-lookup", path = "../vector-lookup", default-feature ordered-float = { version = "3.7.0", default-features = false } prost = { version = "0.11", default-features = false, features = ["std"] } tonic = { version = "0.9", default-features = false, features = ["codegen", "gzip", "prost", "tls", "tls-roots", "transport"] } -value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value" } +value = { package = "value", path = "/home/nathan/repos/vrl/lib/value" } vector-core = { path = "../vector-core", default-features = false } diff --git a/lib/prometheus-parser/Cargo.toml b/lib/prometheus-parser/Cargo.toml index d47915c488768..a38b1c1b0db03 100644 --- a/lib/prometheus-parser/Cargo.toml +++ b/lib/prometheus-parser/Cargo.toml @@ -16,7 +16,7 @@ prost = "0.11" prost-types = "0.11" snafu = { version = "0.7" } vector-common = { path = "../vector-common", features = ["btreemap"] } -value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value" } +value = { package = "value", path = "/home/nathan/repos/vrl/lib/value" } [build-dependencies] prost-build = "0.11" diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index beab1a5706cec..8c488d9baf097 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -62,8 +62,8 @@ snafu = { version = "0.7", optional = true } stream-cancel = { version = "0.8.1", default-features = false } tokio = { version = "1.28.0", default-features = false, features = ["macros", "time"] } tracing = { version = "0.1.34", default-features = false } -value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value", features = ["json", "arbitrary"] } -vrl-core = { package = "vrl-core", path = "/home/nathan_fox/repos/vrl/lib/core", default-features = false } +value = { package = "value", path = "/home/nathan/repos/vrl/lib/value", features = ["json", "arbitrary"] } +vrl-core = { package = "vrl-core", path = "/home/nathan/repos/vrl/lib/core", default-features = false } vector-config = { path = "../vector-config" } vector-config-common = { path = "../vector-config-common" } vector-config-macros = { path = "../vector-config-macros" } diff --git a/lib/vector-config/Cargo.toml b/lib/vector-config/Cargo.toml index 120f5d0b49a63..fc418ed4feded 100644 --- a/lib/vector-config/Cargo.toml +++ b/lib/vector-config/Cargo.toml @@ -26,8 +26,8 @@ snafu = { version = "0.7.4", default-features = false } toml = { version = "0.7.3", default-features = false } tracing = { version = "0.1.34", default-features = false } url = { version = "2.3.1", default-features = false, features = ["serde"] } -vrl-compiler = { package = "vrl-compiler", path = "/home/nathan_fox/repos/vrl/lib/compiler" } -vrl-core = { package = "vrl-core", path = "/home/nathan_fox/repos/vrl/lib/core", default-features = false, features = ["serde"] } +vrl-compiler = { package = "vrl-compiler", path = "/home/nathan/repos/vrl/lib/compiler" } +vrl-core = { package = "vrl-core", path = "/home/nathan/repos/vrl/lib/core", default-features = false, features = ["serde"] } vector-config-common = { path = "../vector-config-common" } vector-config-macros = { path = "../vector-config-macros" } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 6d7554db94dc6..4d427545b27e8 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -60,7 +60,7 @@ tracing-subscriber = { version = "0.3.16", default-features = false, features = typetag = { version = "0.2.7", default-features = false } twox-hash = { version = "1.6.3", default-features = false } url = { version = "2", default-features = false } -value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value", default-features = false, features = ["lua", "toml", "json", "api"] } +value = { package = "value", path = "/home/nathan/repos/vrl/lib/value", default-features = false, features = ["lua", "toml", "json", "api"] } vector-buffers = { path = "../vector-buffers", default-features = false } vector-common = { path = "../vector-common" } vector-config = { path = "../vector-config" } @@ -68,7 +68,7 @@ vector-config-common = { path = "../vector-config-common" } vector-config-macros = { path = "../vector-config-macros" } # Rename to "vrl" once we use a release with stable `-Z namespaced-features`: # https://doc.rust-lang.org/cargo/reference/unstable.html#namespaced-features -vrl-lib = { package = "vrl", path = "/home/nathan_fox/repos/vrl" } +vrl-lib = { package = "vrl", path = "/home/nathan/repos/vrl" } [target.'cfg(target_os = "macos")'.dependencies] security-framework = "2.8.2" @@ -97,7 +97,7 @@ rand = "0.8.5" rand_distr = "0.4.3" tracing-subscriber = { version = "0.3.16", default-features = false, features = ["env-filter", "fmt", "ansi", "registry"] } vector-common = { path = "../vector-common", default-features = false, features = ["test"] } -value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value", default-features = false, features = ["lua", "toml", "json", "api", "arbitrary", "test"] } +value = { package = "value", path = "/home/nathan/repos/vrl/lib/value", default-features = false, features = ["lua", "toml", "json", "api", "arbitrary", "test"] } [features] api = ["dep:async-graphql", "value/api"] diff --git a/lib/vector-lookup/Cargo.toml b/lib/vector-lookup/Cargo.toml index 61d3068eb67b3..51499c498a47b 100644 --- a/lib/vector-lookup/Cargo.toml +++ b/lib/vector-lookup/Cargo.toml @@ -10,4 +10,4 @@ license = "MPL-2.0" serde = { version = "1.0.159", default-features = false, features = ["derive", "alloc"] } vector-config = { path = "../vector-config" } vector-config-macros = { path = "../vector-config-macros" } -vrl = { path = "/home/nathan_fox/repos/vrl", default-features = false, features = ["path"] } +vrl = { path = "/home/nathan/repos/vrl", default-features = false, features = ["path"] } diff --git a/lib/vector-vrl/cli/Cargo.toml b/lib/vector-vrl/cli/Cargo.toml index bda7a6a42f7b6..215d3ad47f884 100644 --- a/lib/vector-vrl/cli/Cargo.toml +++ b/lib/vector-vrl/cli/Cargo.toml @@ -7,7 +7,7 @@ publish = false license = "MPL-2.0" [dependencies] -vrl-cli = { package = "vrl-cli", path = "/home/nathan_fox/repos/vrl/lib/cli" } +vrl-cli = { package = "vrl-cli", path = "/home/nathan/repos/vrl/lib/cli" } clap = { version = "4.1.14", features = ["derive"] } -vrl-stdlib = { package = "vrl-stdlib", path = "/home/nathan_fox/repos/vrl/lib/stdlib" } +vrl-stdlib = { package = "vrl-stdlib", path = "/home/nathan/repos/vrl/lib/stdlib" } vector-vrl-functions = { path = "../functions" } diff --git a/lib/vector-vrl/functions/Cargo.toml b/lib/vector-vrl/functions/Cargo.toml index 84e324f9533c3..c4619a61b6264 100644 --- a/lib/vector-vrl/functions/Cargo.toml +++ b/lib/vector-vrl/functions/Cargo.toml @@ -7,4 +7,4 @@ publish = false license = "MPL-2.0" [dependencies] -vrl = { path = "/home/nathan_fox/repos/vrl" } +vrl = { path = "/home/nathan/repos/vrl" } diff --git a/lib/vector-vrl/tests/Cargo.toml b/lib/vector-vrl/tests/Cargo.toml index 2726d3e22cef8..caf34283e5981 100644 --- a/lib/vector-vrl/tests/Cargo.toml +++ b/lib/vector-vrl/tests/Cargo.toml @@ -7,11 +7,11 @@ publish = false [dependencies] enrichment = { path = "../../enrichment" } -lookup = { package = "path", path = "/home/nathan_fox/repos/vrl/lib/path" } -stdlib = { package = "vrl-stdlib", path = "/home/nathan_fox/repos/vrl/lib/stdlib" } -vrl = { package = "vrl", path = "/home/nathan_fox/repos/vrl" } -value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value" } -vrl-tests = { package = "vrl-tests", path = "/home/nathan_fox/repos/vrl/lib/tests" } +lookup = { package = "path", path = "/home/nathan/repos/vrl/lib/path" } +stdlib = { package = "vrl-stdlib", path = "/home/nathan/repos/vrl/lib/stdlib" } +vrl = { package = "vrl", path = "/home/nathan/repos/vrl" } +value = { package = "value", path = "/home/nathan/repos/vrl/lib/value" } +vrl-tests = { package = "vrl-tests", path = "/home/nathan/repos/vrl/lib/tests" } vector-vrl-functions = { path = "../../vector-vrl/functions" } ansi_term = "0.12" diff --git a/lib/vector-vrl/web-playground/Cargo.toml b/lib/vector-vrl/web-playground/Cargo.toml index 3804b3cdaa3c9..6fec3444556e0 100644 --- a/lib/vector-vrl/web-playground/Cargo.toml +++ b/lib/vector-vrl/web-playground/Cargo.toml @@ -10,9 +10,9 @@ crate-type = ["cdylib"] [dependencies] wasm-bindgen = "0.2" -vrl = { package = "vrl", path = "/home/nathan_fox/repos/vrl", default-features = false} -value = { package = "value", path = "/home/nathan_fox/repos/vrl/lib/value", default-features = false} -stdlib = { package = "vrl-stdlib", path = "/home/nathan_fox/repos/vrl/lib/stdlib"} +vrl = { package = "vrl", path = "/home/nathan/repos/vrl", default-features = false} +value = { package = "value", path = "/home/nathan/repos/vrl/lib/value", default-features = false} +stdlib = { package = "vrl-stdlib", path = "/home/nathan/repos/vrl/lib/stdlib"} serde = { version = "1.0", features = ["derive"] } serde-wasm-bindgen = "0.5" gloo-utils = { version = "0.1", features = ["serde"] } diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index 4c4a420ed2dc5..882a8f1ee3189 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -25,6 +25,7 @@ use vector_core::{ event::{metric::TagValue, MetricTags}, metric_tags, }; +use vrl::compiler::value::Collection; use crate::schema::Definition; use crate::{ diff --git a/src/sources/docker_logs/tests.rs b/src/sources/docker_logs/tests.rs index e61f30f740591..22634b432eb83 100644 --- a/src/sources/docker_logs/tests.rs +++ b/src/sources/docker_logs/tests.rs @@ -25,16 +25,6 @@ fn exclude_self() { #[cfg(all(test, feature = "docker-logs-integration-tests"))] mod integration_tests { - use bollard::{ - container::{ - Config as ContainerConfig, CreateContainerOptions, KillContainerOptions, - RemoveContainerOptions, StartContainerOptions, WaitContainerOptions, - }, - image::{CreateImageOptions, ListImagesOptions}, - }; - use futures::{stream::TryStreamExt, FutureExt}; - use similar_asserts::assert_eq; - use crate::sources::docker_logs::*; use crate::sources::docker_logs::{CONTAINER, CREATED_AT, IMAGE, NAME}; use crate::{ @@ -46,6 +36,16 @@ mod integration_tests { }, SourceSender, }; + use bollard::{ + container::{ + Config as ContainerConfig, CreateContainerOptions, KillContainerOptions, + RemoveContainerOptions, StartContainerOptions, WaitContainerOptions, + }, + image::{CreateImageOptions, ListImagesOptions}, + }; + use futures::{stream::TryStreamExt, FutureExt}; + use similar_asserts::assert_eq; + use vrl::compiler::value; /// None if docker is not present on the system async fn source_with<'a, L: Into>>( @@ -315,7 +315,7 @@ mod integration_tests { schema_definitions .unwrap() .assert_valid_for_event(&events[0]); - assert_eq!(events[0].as_log().get(".").unwrap(), &vrl::value!(message)); + assert_eq!(events[0].as_log().get(".").unwrap(), &value!(message)); }) .await; } @@ -382,28 +382,28 @@ mod integration_tests { let log = events[0].as_log(); let meta = log.metadata().value(); - assert_eq!(log.get(".").unwrap(), &vrl::value!(message)); + assert_eq!(log.get(".").unwrap(), &value!(message)); assert_eq!( meta.get(path!(DockerLogsConfig::NAME, CONTAINER)).unwrap(), - &vrl::value!(id) + &value!(id) ); assert!(meta .get(path!(DockerLogsConfig::NAME, CREATED_AT)) .is_some()); assert_eq!( meta.get(path!(DockerLogsConfig::NAME, IMAGE)).unwrap(), - &vrl::value!("busybox") + &value!("busybox") ); assert!(meta .get(path!(DockerLogsConfig::NAME, "labels", label)) .is_some()); assert_eq!( meta.get(path!(DockerLogsConfig::NAME, NAME)).unwrap(), - &vrl::value!(name) + &value!(name) ); assert_eq!( meta.get(path!("vector", "source_type")).unwrap(), - &vrl::value!(DockerLogsConfig::NAME) + &value!(DockerLogsConfig::NAME) ); assert!(meta .get(path!("vector", "ingest_timestamp")) diff --git a/src/sources/exec/mod.rs b/src/sources/exec/mod.rs index 0bb58985c73f7..74787c2f59a2e 100644 --- a/src/sources/exec/mod.rs +++ b/src/sources/exec/mod.rs @@ -722,6 +722,7 @@ mod tests { use bytes::Bytes; use std::io::Cursor; use vector_core::event::EventMetadata; + use vrl::compiler::value; #[cfg(unix)] use futures::task::Poll; @@ -776,7 +777,7 @@ mod tests { let pid = Some(8888_u32); let mut event: Event = - LogEvent::from_parts(vrl::value!("hello world"), EventMetadata::default()).into(); + LogEvent::from_parts(value!("hello world"), EventMetadata::default()).into(); handle_event( &config, @@ -792,24 +793,24 @@ mod tests { assert_eq!( meta.get(path!(ExecConfig::NAME, "host")).unwrap(), - &vrl::value!("Some.Machine") + &value!("Some.Machine") ); assert_eq!( meta.get(path!(ExecConfig::NAME, STREAM_KEY)).unwrap(), - &vrl::value!(STDOUT) + &value!(STDOUT) ); assert_eq!( meta.get(path!(ExecConfig::NAME, PID_KEY)).unwrap(), - &vrl::value!(8888_i64) + &value!(8888_i64) ); assert_eq!( meta.get(path!(ExecConfig::NAME, COMMAND_KEY)).unwrap(), - &vrl::value!(config.command) + &value!(config.command) ); - assert_eq!(log.value(), &vrl::value!("hello world")); + assert_eq!(log.value(), &value!("hello world")); assert_eq!( meta.get(path!("vector", "source_type")).unwrap(), - &vrl::value!("exec") + &value!("exec") ); assert!(meta .get(path!("vector", "ingest_timestamp")) @@ -857,7 +858,7 @@ mod tests { let pid = Some(8888_u32); let mut event: Event = - LogEvent::from_parts(vrl::value!("hello world"), EventMetadata::default()).into(); + LogEvent::from_parts(value!("hello world"), EventMetadata::default()).into(); handle_event( &config, @@ -873,24 +874,24 @@ mod tests { assert_eq!( meta.get(path!(ExecConfig::NAME, "host")).unwrap(), - &vrl::value!("Some.Machine") + &value!("Some.Machine") ); assert_eq!( meta.get(path!(ExecConfig::NAME, STREAM_KEY)).unwrap(), - &vrl::value!(STDOUT) + &value!(STDOUT) ); assert_eq!( meta.get(path!(ExecConfig::NAME, PID_KEY)).unwrap(), - &vrl::value!(8888_i64) + &value!(8888_i64) ); assert_eq!( meta.get(path!(ExecConfig::NAME, COMMAND_KEY)).unwrap(), - &vrl::value!(config.command) + &value!(config.command) ); - assert_eq!(log.value(), &vrl::value!("hello world")); + assert_eq!(log.value(), &value!("hello world")); assert_eq!( meta.get(path!("vector", "source_type")).unwrap(), - &vrl::value!("exec") + &value!("exec") ); assert!(meta .get(path!("vector", "ingest_timestamp")) diff --git a/src/sources/file.rs b/src/sources/file.rs index b3644a6aec5d1..a6a96779cff1a 100644 --- a/src/sources/file.rs +++ b/src/sources/file.rs @@ -807,8 +807,8 @@ mod tests { use similar_asserts::assert_eq; use tempfile::tempdir; use tokio::time::{sleep, timeout, Duration}; - use value::kind::Collection; use vector_core::schema::Definition; + use vrl::value::kind::Collection; use super::*; use crate::{ @@ -818,6 +818,7 @@ mod tests { sources::file, test_util::components::{assert_source_compliance, FILE_SOURCE_TAGS}, }; + use vrl::compiler::value; #[test] fn generate_config() { @@ -1069,14 +1070,14 @@ mod tests { }; let log = create_event(line, offset, file, &meta, LogNamespace::Vector); - assert_eq!(log.value(), &vrl::value!("hello world")); + assert_eq!(log.value(), &value!("hello world")); assert_eq!( log.metadata() .value() .get(path!("vector", "source_type")) .unwrap(), - &vrl::value!("file") + &value!("file") ); assert!(log .metadata() @@ -1090,21 +1091,21 @@ mod tests { .value() .get(path!(FileConfig::NAME, "host")) .unwrap(), - &vrl::value!("Some.Machine") + &value!("Some.Machine") ); assert_eq!( log.metadata() .value() .get(path!(FileConfig::NAME, "offset")) .unwrap(), - &vrl::value!(0) + &value!(0) ); assert_eq!( log.metadata() .value() .get(path!(FileConfig::NAME, "path")) .unwrap(), - &vrl::value!("some_file.rs") + &value!("some_file.rs") ); } diff --git a/src/sources/file_descriptors/file_descriptor.rs b/src/sources/file_descriptors/file_descriptor.rs index 008ac2b43773a..f550def9160c9 100644 --- a/src/sources/file_descriptors/file_descriptor.rs +++ b/src/sources/file_descriptors/file_descriptor.rs @@ -112,6 +112,7 @@ mod tests { SourceSender, }; use futures::StreamExt; + use vrl::compiler::value; #[test] fn generate_config() { @@ -189,10 +190,10 @@ mod tests { let log = event.as_log(); let meta = log.metadata().value(); - assert_eq!(&vrl::value!("hello world"), log.value()); + assert_eq!(&value!("hello world"), log.value()); assert_eq!( meta.get(path!("vector", "source_type")).unwrap(), - &vrl::value!("file_descriptor") + &value!("file_descriptor") ); assert!(meta .get(path!("vector", "ingest_timestamp")) @@ -203,7 +204,7 @@ mod tests { let event = event.unwrap(); let log = event.as_log(); - assert_eq!(&vrl::value!("hello world again"), log.value()); + assert_eq!(&value!("hello world again"), log.value()); let event = stream.next().await; assert!(event.is_none()); diff --git a/src/sources/file_descriptors/stdin.rs b/src/sources/file_descriptors/stdin.rs index c3ed61d901e63..4119f17e8743a 100644 --- a/src/sources/file_descriptors/stdin.rs +++ b/src/sources/file_descriptors/stdin.rs @@ -117,6 +117,7 @@ mod tests { }; use futures::StreamExt; use lookup::path; + use vrl::compiler::value; #[test] fn generate_config() { @@ -180,10 +181,10 @@ mod tests { let log = event.as_log(); let meta = log.metadata().value(); - assert_eq!(&vrl::value!("hello world"), log.value()); + assert_eq!(&value!("hello world"), log.value()); assert_eq!( meta.get(path!("vector", "source_type")).unwrap(), - &vrl::value!("stdin") + &value!("stdin") ); assert!(meta .get(path!("vector", "ingest_timestamp")) @@ -194,7 +195,7 @@ mod tests { let event = event.unwrap(); let log = event.as_log(); - assert_eq!(&vrl::value!("hello world again"), log.value()); + assert_eq!(&value!("hello world again"), log.value()); let event = stream.next().await; assert!(event.is_none()); diff --git a/src/sources/kafka.rs b/src/sources/kafka.rs index ba314231e41ad..09b88cd6b0088 100644 --- a/src/sources/kafka.rs +++ b/src/sources/kafka.rs @@ -912,6 +912,7 @@ mod integration_test { use tokio::time::sleep; use vector_buffers::topology::channel::BufferReceiver; use vector_core::event::EventStatus; + use vrl::compiler::value; use super::{test::*, *}; use crate::{ @@ -1066,7 +1067,7 @@ mod integration_test { assert_eq!( meta.get(path!("vector", "source_type")).unwrap(), - &vrl::value!(KafkaSourceConfig::NAME) + &value!(KafkaSourceConfig::NAME) ); assert!(meta .get(path!("vector", "ingest_timestamp")) @@ -1075,20 +1076,20 @@ mod integration_test { assert_eq!( event.as_log().value(), - &vrl::value!(format!("{} {:03}", TEXT, i)) + &value!(format!("{} {:03}", TEXT, i)) ); assert_eq!( meta.get(path!("kafka", "message_key")).unwrap(), - &vrl::value!(format!("{} {}", KEY, i)) + &value!(format!("{} {}", KEY, i)) ); assert_eq!( meta.get(path!("kafka", "timestamp")).unwrap(), - &vrl::value!(now.trunc_subsecs(3)) + &value!(now.trunc_subsecs(3)) ); assert_eq!( meta.get(path!("kafka", "topic")).unwrap(), - &vrl::value!(topic.clone()) + &value!(topic.clone()) ); assert!(meta.get(path!("kafka", "partition")).unwrap().is_integer(),); assert!(meta.get(path!("kafka", "offset")).unwrap().is_integer(),); diff --git a/src/sources/kubernetes_logs/parser/cri.rs b/src/sources/kubernetes_logs/parser/cri.rs index 47519e1637596..a4d9420865988 100644 --- a/src/sources/kubernetes_logs/parser/cri.rs +++ b/src/sources/kubernetes_logs/parser/cri.rs @@ -189,6 +189,7 @@ pub mod tests { use super::{super::test_util, *}; use crate::{event::LogEvent, test_util::trace_init, transforms::Transform}; + use vrl::compiler::value; fn make_long_string(base: &str, len: usize) -> String { base.chars().cycle().take(len).collect() @@ -202,7 +203,7 @@ pub mod tests { "2016-10-06T00:17:09.669794202Z stdout F The content of the log entry 1", ), vec![test_util::make_log_event( - vrl::value!("The content of the log entry 1"), + value!("The content of the log entry 1"), "2016-10-06T00:17:09.669794202Z", "stdout", false, @@ -212,7 +213,7 @@ pub mod tests { ( Bytes::from("2016-10-06T00:17:09.669794202Z stdout P First line of log entry 2"), vec![test_util::make_log_event( - vrl::value!("First line of log entry 2"), + value!("First line of log entry 2"), "2016-10-06T00:17:09.669794202Z", "stdout", true, @@ -224,7 +225,7 @@ pub mod tests { "2016-10-06T00:17:09.669794202Z stdout P Second line of the log entry 2", ), vec![test_util::make_log_event( - vrl::value!("Second line of the log entry 2"), + value!("Second line of the log entry 2"), "2016-10-06T00:17:09.669794202Z", "stdout", true, @@ -234,7 +235,7 @@ pub mod tests { ( Bytes::from("2016-10-06T00:17:10.113242941Z stderr F Last line of the log entry 2"), vec![test_util::make_log_event( - vrl::value!("Last line of the log entry 2"), + value!("Last line of the log entry 2"), "2016-10-06T00:17:10.113242941Z", "stderr", false, @@ -251,7 +252,7 @@ pub mod tests { .join(""), ), vec![test_util::make_log_event( - vrl::value!(make_long_string("very long message ", 16 * 1024)), + value!(make_long_string("very long message ", 16 * 1024)), "2016-10-06T00:17:10.113242941Z", "stdout", true, @@ -268,7 +269,7 @@ pub mod tests { 128, 208, 184, 208, 178, 208, 181, 209, 130, 32, 208, 156, 208, 184, 209, 10, ]), vec![test_util::make_log_event( - vrl::value!(Bytes::from(vec![ + value!(Bytes::from(vec![ 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 32, 208, 159, 209, 128, 208, 184, 208, 178, 208, 181, 209, 130, 32, 208, 156, 208, 184, 209, ])), @@ -286,7 +287,7 @@ pub mod tests { trace_init(); test_util::test_parser( || Transform::function(Cri::new(LogNamespace::Vector)), - |bytes| Event::Log(LogEvent::from(vrl::value!(bytes))), + |bytes| Event::Log(LogEvent::from(value!(bytes))), valid_cases(LogNamespace::Vector), ); } diff --git a/src/sources/kubernetes_logs/parser/docker.rs b/src/sources/kubernetes_logs/parser/docker.rs index 382c7350626f4..a9050ff911dcd 100644 --- a/src/sources/kubernetes_logs/parser/docker.rs +++ b/src/sources/kubernetes_logs/parser/docker.rs @@ -207,6 +207,7 @@ enum NormalizationError { pub mod tests { use super::{super::test_util, *}; use crate::{test_util::trace_init, transforms::Transform}; + use vrl::compiler::value; fn make_long_string(base: &str, len: usize) -> String { base.chars().cycle().take(len).collect() @@ -220,7 +221,7 @@ pub mod tests { r#"{"log": "The actual log line\n", "stream": "stderr", "time": "2016-10-05T00:00:30.082640485Z"}"#, ), vec![test_util::make_log_event( - vrl::value!("The actual log line"), + value!("The actual log line"), "2016-10-05T00:00:30.082640485Z", "stderr", false, @@ -232,7 +233,7 @@ pub mod tests { r#"{"log": "A line without newline char at the end", "stream": "stdout", "time": "2016-10-05T00:00:30.082640485Z"}"#, ), vec![test_util::make_log_event( - vrl::value!("A line without newline char at the end"), + value!("A line without newline char at the end"), "2016-10-05T00:00:30.082640485Z", "stdout", false, @@ -250,7 +251,7 @@ pub mod tests { .join(""), ), vec![test_util::make_log_event( - vrl::value!(make_long_string("partial ", 16 * 1024)), + value!(make_long_string("partial ", 16 * 1024)), "2016-10-05T00:00:30.082640485Z", "stdout", true, @@ -270,7 +271,7 @@ pub mod tests { .join(""), ), vec![test_util::make_log_event( - vrl::value!(make_long_string("non-partial ", 16 * 1024 - 1)), + value!(make_long_string("non-partial ", 16 * 1024 - 1)), "2016-10-05T00:00:30.082640485Z", "stdout", false, @@ -319,7 +320,7 @@ pub mod tests { log_namespace: LogNamespace::Vector, }) }, - |bytes| Event::Log(LogEvent::from(vrl::value!(bytes))), + |bytes| Event::Log(LogEvent::from(value!(bytes))), valid_cases(LogNamespace::Vector), ); } @@ -347,7 +348,7 @@ pub mod tests { for bytes in cases { let mut parser = Docker::new(LogNamespace::Vector); - let input = LogEvent::from(vrl::value!(bytes)); + let input = LogEvent::from(value!(bytes)); let mut output = OutputBuffer::default(); parser.transform(&mut output, input.into()); diff --git a/src/sources/kubernetes_logs/parser/mod.rs b/src/sources/kubernetes_logs/parser/mod.rs index dc5920fdfac76..1a0108308bd8e 100644 --- a/src/sources/kubernetes_logs/parser/mod.rs +++ b/src/sources/kubernetes_logs/parser/mod.rs @@ -84,6 +84,7 @@ impl FunctionTransform for Parser { mod tests { use bytes::Bytes; use lookup::event_path; + use vrl::compiler::value; use super::*; use crate::{event::Event, event::LogEvent, test_util::trace_init, transforms::Transform}; @@ -107,7 +108,7 @@ mod tests { trace_init(); test_util::test_parser( || Transform::function(Parser::new(LogNamespace::Vector)), - |bytes| Event::Log(LogEvent::from(vrl::value!(bytes))), + |bytes| Event::Log(LogEvent::from(value!(bytes))), valid_cases(LogNamespace::Vector), ); } @@ -146,7 +147,7 @@ mod tests { // No `message` field. (LogEvent::default(), LogNamespace::Legacy), // Non-bytes `message` field. - (LogEvent::from(vrl::value!(123)), LogNamespace::Vector), + (LogEvent::from(value!(123)), LogNamespace::Vector), ( { let mut input = LogEvent::default(); diff --git a/src/sources/kubernetes_logs/parser/test_util.rs b/src/sources/kubernetes_logs/parser/test_util.rs index 7e11168672505..07deeb48c8e4c 100644 --- a/src/sources/kubernetes_logs/parser/test_util.rs +++ b/src/sources/kubernetes_logs/parser/test_util.rs @@ -3,8 +3,9 @@ use similar_asserts::assert_eq; use chrono::{DateTime, Utc}; use lookup::{event_path, metadata_path}; -use value::Value; use vector_core::{config::LogNamespace, event}; +use vrl::compiler::value; +use vrl::value::Value; use crate::{ event::{Event, LogEvent}, @@ -29,7 +30,7 @@ pub fn make_log_event( let log = match log_namespace { LogNamespace::Vector => { - let mut log = LogEvent::from(vrl::value!(message)); + let mut log = LogEvent::from(value!(message)); log.insert(metadata_path!(Config::NAME, "timestamp"), timestamp); log.insert(metadata_path!(Config::NAME, "stream"), stream); if is_partial { diff --git a/src/sources/redis/mod.rs b/src/sources/redis/mod.rs index e2134e78ba9ee..ff58afcd08afa 100644 --- a/src/sources/redis/mod.rs +++ b/src/sources/redis/mod.rs @@ -321,6 +321,7 @@ mod integration_test { }, SourceSender, }; + use vrl::compiler::value; const REDIS_SERVER: &str = "redis://redis:6379/0"; @@ -393,7 +394,7 @@ mod integration_test { meta.value() .get(path!(RedisSourceConfig::NAME, "key")) .unwrap(), - &vrl::value!(key) + &value!(key) ); } diff --git a/src/sources/socket/mod.rs b/src/sources/socket/mod.rs index f1b4ced6d4cc7..a54dfd99c7009 100644 --- a/src/sources/socket/mod.rs +++ b/src/sources/socket/mod.rs @@ -345,6 +345,7 @@ mod test { }; use vector_core::event::EventContainer; use vrl::compiler::value; + use vrl::value::{btreemap, Value}; #[cfg(unix)] use { @@ -587,7 +588,7 @@ mod test { "one line".into() ); - let tls_meta: BTreeMap = btreemap!( + let tls_meta: BTreeMap = btreemap!( "subject" => "CN=localhost,OU=Vector,O=Datadog,L=New York,ST=New York,C=US" ); From bdf1c9331e1a9a863f0b5902baacf50d393be48c Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Wed, 3 May 2023 14:33:04 -0400 Subject: [PATCH 05/23] save --- Cargo.lock | 8 +++++--- lib/enrichment/src/find_enrichment_table_records.rs | 2 +- lib/enrichment/src/get_enrichment_table_record.rs | 3 +-- lib/vector-common/Cargo.toml | 1 + lib/vector-common/src/lib.rs | 4 ++-- lib/vector-config/Cargo.toml | 3 +-- lib/vector-config/src/external/datetime.rs | 2 +- lib/vector-config/src/external/vrl.rs | 2 +- lib/vector-core/src/event/log_event.rs | 2 +- lib/vector-vrl/tests/src/main.rs | 3 +-- lib/vector-vrl/web-playground/src/lib.rs | 2 +- src/sinks/influxdb/logs.rs | 2 +- src/sources/aws_kinesis_firehose/mod.rs | 2 +- src/sources/docker_logs/tests.rs | 2 +- src/sources/exec/mod.rs | 2 +- src/sources/file.rs | 2 +- src/sources/file_descriptors/file_descriptor.rs | 2 +- src/sources/file_descriptors/stdin.rs | 2 +- src/sources/kafka.rs | 2 +- src/sources/kubernetes_logs/parser/cri.rs | 2 +- src/sources/kubernetes_logs/parser/docker.rs | 2 +- src/sources/kubernetes_logs/parser/mod.rs | 2 +- src/sources/kubernetes_logs/parser/test_util.rs | 3 +-- src/sources/opentelemetry/tests.rs | 2 +- src/sources/redis/mod.rs | 2 +- src/sources/socket/mod.rs | 2 +- 26 files changed, 31 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c1d5c3ad2f9fb..b6f801851178c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9413,6 +9413,7 @@ dependencies = [ "vector-config", "vector-config-common", "vector-config-macros", + "vrl-compiler", "vrl-core", ] @@ -9438,8 +9439,7 @@ dependencies = [ "url", "vector-config-common", "vector-config-macros", - "vrl-compiler", - "vrl-core", + "vrl", ] [[package]] @@ -9679,6 +9679,7 @@ dependencies = [ "anymap", "bytes 1.4.0", "chrono", + "chrono-tz", "dyn-clone", "getrandom 0.2.9", "lalrpop-util", @@ -9687,9 +9688,10 @@ dependencies = [ "path", "regex", "serde", + "snafu", "thiserror", + "tracing 0.1.37", "value", - "vrl-core", "vrl-diagnostic", "vrl-parser", ] diff --git a/lib/enrichment/src/find_enrichment_table_records.rs b/lib/enrichment/src/find_enrichment_table_records.rs index 9e46582a95d57..50d43194129a1 100644 --- a/lib/enrichment/src/find_enrichment_table_records.rs +++ b/lib/enrichment/src/find_enrichment_table_records.rs @@ -20,7 +20,7 @@ fn find_enrichment_table_records( .iter() .map(|value| Ok(value.try_bytes_utf8_lossy()?.to_string())) .collect::, _>>(), - value => Err(value::Error::Expected { + value => Err(ValueError::Expected { got: value.kind(), expected: Kind::array(Collection::any()), }), diff --git a/lib/enrichment/src/get_enrichment_table_record.rs b/lib/enrichment/src/get_enrichment_table_record.rs index da590ae5749f6..8f4e9e2a7845e 100644 --- a/lib/enrichment/src/get_enrichment_table_record.rs +++ b/lib/enrichment/src/get_enrichment_table_record.rs @@ -1,5 +1,4 @@ use std::collections::BTreeMap; -use vrl::compiler::value::Error; use vrl::stdlib::prelude::*; use crate::{ @@ -21,7 +20,7 @@ fn get_enrichment_table_record( .iter() .map(|value| Ok(value.try_bytes_utf8_lossy()?.to_string())) .collect::, _>>(), - value => Err(Error::Expected { + value => Err(ValueError::Expected { got: value.kind(), expected: Kind::array(Collection::any()), }), diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 8c488d9baf097..4a5534df06e1a 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -64,6 +64,7 @@ tokio = { version = "1.28.0", default-features = false, features = ["macros", "t tracing = { version = "0.1.34", default-features = false } value = { package = "value", path = "/home/nathan/repos/vrl/lib/value", features = ["json", "arbitrary"] } vrl-core = { package = "vrl-core", path = "/home/nathan/repos/vrl/lib/core", default-features = false } +vrl-compiler = { path = "/home/nathan/repos/vrl/lib/compiler", default-features = false } vector-config = { path = "../vector-config" } vector-config-common = { path = "../vector-config-common" } vector-config-macros = { path = "../vector-config-macros" } diff --git a/lib/vector-common/src/lib.rs b/lib/vector-common/src/lib.rs index 867b5a9ce5565..4e7b537bf1cfb 100644 --- a/lib/vector-common/src/lib.rs +++ b/lib/vector-common/src/lib.rs @@ -21,7 +21,7 @@ pub mod byte_size_of; pub mod config; #[cfg(feature = "conversion")] -pub use vrl_core::TimeZone; +pub use vrl_compiler::TimeZone; #[cfg(feature = "encoding")] pub mod encode_logfmt { @@ -29,7 +29,7 @@ pub mod encode_logfmt { } pub mod conversion { - pub use vrl_core::conversion::*; + pub use vrl_compiler::conversion::*; } pub mod event_data_eq; diff --git a/lib/vector-config/Cargo.toml b/lib/vector-config/Cargo.toml index fc418ed4feded..67441b4887c1d 100644 --- a/lib/vector-config/Cargo.toml +++ b/lib/vector-config/Cargo.toml @@ -26,8 +26,7 @@ snafu = { version = "0.7.4", default-features = false } toml = { version = "0.7.3", default-features = false } tracing = { version = "0.1.34", default-features = false } url = { version = "2.3.1", default-features = false, features = ["serde"] } -vrl-compiler = { package = "vrl-compiler", path = "/home/nathan/repos/vrl/lib/compiler" } -vrl-core = { package = "vrl-core", path = "/home/nathan/repos/vrl/lib/core", default-features = false, features = ["serde"] } +vrl = { path = "/home/nathan/repos/vrl", default-features = false, features = ["compiler"] } vector-config-common = { path = "../vector-config-common" } vector-config-macros = { path = "../vector-config-macros" } diff --git a/lib/vector-config/src/external/datetime.rs b/lib/vector-config/src/external/datetime.rs index 341df8076a950..45575f2846564 100644 --- a/lib/vector-config/src/external/datetime.rs +++ b/lib/vector-config/src/external/datetime.rs @@ -9,7 +9,7 @@ use chrono_tz::Tz; use serde_json::Value; use std::cell::RefCell; use vector_config_common::{attributes::CustomAttribute, constants}; -use vrl_core::TimeZone; +use vrl::compiler::TimeZone; // TODO: Consider an approach for generating schema of "fixed string value, or remainder" structure // used by this type. diff --git a/lib/vector-config/src/external/vrl.rs b/lib/vector-config/src/external/vrl.rs index 87352e018f263..45303c52ce887 100644 --- a/lib/vector-config/src/external/vrl.rs +++ b/lib/vector-config/src/external/vrl.rs @@ -1,7 +1,7 @@ use std::cell::RefCell; use serde_json::Value; -use vrl_compiler::VrlRuntime; +use vrl::compiler::VrlRuntime; use crate::{ schema::{generate_string_schema, SchemaGenerator, SchemaObject}, diff --git a/lib/vector-core/src/event/log_event.rs b/lib/vector-core/src/event/log_event.rs index e775a3235fb9e..bcd70b5e1ea19 100644 --- a/lib/vector-core/src/event/log_event.rs +++ b/lib/vector-core/src/event/log_event.rs @@ -694,7 +694,7 @@ mod test { use super::*; use crate::test_util::open_fixture; use lookup::event_path; - use vrl_lib::compiler::value; + use vrl_lib::value::value; // The following two tests assert that renaming a key has no effect if the // keys are equivalent, whether the key exists in the log or not. diff --git a/lib/vector-vrl/tests/src/main.rs b/lib/vector-vrl/tests/src/main.rs index e4c4dfe4b9583..c0ef1dc6aa704 100644 --- a/lib/vector-vrl/tests/src/main.rs +++ b/lib/vector-vrl/tests/src/main.rs @@ -9,8 +9,7 @@ use vrl_tests::{get_tests_from_functions, run_tests, Test, TestConfig}; use chrono_tz::Tz; use clap::Parser; use glob::glob; -use vrl::compiler::{CompileConfig, VrlRuntime}; -use vrl::core::TimeZone; +use vrl::compiler::{CompileConfig, TimeZone, VrlRuntime}; #[cfg(not(target_env = "msvc"))] #[global_allocator] diff --git a/lib/vector-vrl/web-playground/src/lib.rs b/lib/vector-vrl/web-playground/src/lib.rs index 403ea5bf1a6c7..c2a9d1775eed4 100644 --- a/lib/vector-vrl/web-playground/src/lib.rs +++ b/lib/vector-vrl/web-playground/src/lib.rs @@ -4,8 +4,8 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use value::Secrets; use vrl::compiler::runtime::{Runtime, Terminate}; +use vrl::compiler::TimeZone; use vrl::compiler::{compile_with_state, CompileConfig, TargetValue, TypeState}; -use vrl::core::TimeZone; use vrl::diagnostic::DiagnosticList; use vrl::diagnostic::Formatter; use wasm_bindgen::prelude::*; diff --git a/src/sinks/influxdb/logs.rs b/src/sinks/influxdb/logs.rs index c2cbc70ed2ad4..08ef59618b6f7 100644 --- a/src/sinks/influxdb/logs.rs +++ b/src/sinks/influxdb/logs.rs @@ -883,7 +883,7 @@ mod integration_tests { use std::sync::Arc; use vector_core::config::{LegacyKey, LogNamespace}; use vector_core::event::{BatchNotifier, BatchStatus, Event, LogEvent}; - use vrl::compiler::value; + use vrl::value::value; use super::*; use crate::{ diff --git a/src/sources/aws_kinesis_firehose/mod.rs b/src/sources/aws_kinesis_firehose/mod.rs index c14fdc3ce5003..0c55614d4acac 100644 --- a/src/sources/aws_kinesis_firehose/mod.rs +++ b/src/sources/aws_kinesis_firehose/mod.rs @@ -255,7 +255,7 @@ mod tests { use similar_asserts::assert_eq; use tokio::time::{sleep, Duration}; use vector_common::assert_event_data_eq; - use vrl::compiler::value; + use vrl::value::value; use super::*; use crate::{ diff --git a/src/sources/docker_logs/tests.rs b/src/sources/docker_logs/tests.rs index 22634b432eb83..e055431271886 100644 --- a/src/sources/docker_logs/tests.rs +++ b/src/sources/docker_logs/tests.rs @@ -45,7 +45,7 @@ mod integration_tests { }; use futures::{stream::TryStreamExt, FutureExt}; use similar_asserts::assert_eq; - use vrl::compiler::value; + use vrl::value::value; /// None if docker is not present on the system async fn source_with<'a, L: Into>>( diff --git a/src/sources/exec/mod.rs b/src/sources/exec/mod.rs index 74787c2f59a2e..bbd6022ed6c49 100644 --- a/src/sources/exec/mod.rs +++ b/src/sources/exec/mod.rs @@ -722,7 +722,7 @@ mod tests { use bytes::Bytes; use std::io::Cursor; use vector_core::event::EventMetadata; - use vrl::compiler::value; + use vrl::value::value; #[cfg(unix)] use futures::task::Poll; diff --git a/src/sources/file.rs b/src/sources/file.rs index a6a96779cff1a..4cad2b3c0bd08 100644 --- a/src/sources/file.rs +++ b/src/sources/file.rs @@ -818,7 +818,7 @@ mod tests { sources::file, test_util::components::{assert_source_compliance, FILE_SOURCE_TAGS}, }; - use vrl::compiler::value; + use vrl::value::value; #[test] fn generate_config() { diff --git a/src/sources/file_descriptors/file_descriptor.rs b/src/sources/file_descriptors/file_descriptor.rs index f550def9160c9..9a97591f0644c 100644 --- a/src/sources/file_descriptors/file_descriptor.rs +++ b/src/sources/file_descriptors/file_descriptor.rs @@ -112,7 +112,7 @@ mod tests { SourceSender, }; use futures::StreamExt; - use vrl::compiler::value; + use vrl::value::value; #[test] fn generate_config() { diff --git a/src/sources/file_descriptors/stdin.rs b/src/sources/file_descriptors/stdin.rs index 4119f17e8743a..3b6d5aecebbae 100644 --- a/src/sources/file_descriptors/stdin.rs +++ b/src/sources/file_descriptors/stdin.rs @@ -117,7 +117,7 @@ mod tests { }; use futures::StreamExt; use lookup::path; - use vrl::compiler::value; + use vrl::value::value; #[test] fn generate_config() { diff --git a/src/sources/kafka.rs b/src/sources/kafka.rs index 09b88cd6b0088..94c561dffd36c 100644 --- a/src/sources/kafka.rs +++ b/src/sources/kafka.rs @@ -912,7 +912,7 @@ mod integration_test { use tokio::time::sleep; use vector_buffers::topology::channel::BufferReceiver; use vector_core::event::EventStatus; - use vrl::compiler::value; + use vrl::value::value; use super::{test::*, *}; use crate::{ diff --git a/src/sources/kubernetes_logs/parser/cri.rs b/src/sources/kubernetes_logs/parser/cri.rs index a4d9420865988..c669fd44f5b38 100644 --- a/src/sources/kubernetes_logs/parser/cri.rs +++ b/src/sources/kubernetes_logs/parser/cri.rs @@ -189,7 +189,7 @@ pub mod tests { use super::{super::test_util, *}; use crate::{event::LogEvent, test_util::trace_init, transforms::Transform}; - use vrl::compiler::value; + use vrl::value::value; fn make_long_string(base: &str, len: usize) -> String { base.chars().cycle().take(len).collect() diff --git a/src/sources/kubernetes_logs/parser/docker.rs b/src/sources/kubernetes_logs/parser/docker.rs index a9050ff911dcd..97724e4b661c8 100644 --- a/src/sources/kubernetes_logs/parser/docker.rs +++ b/src/sources/kubernetes_logs/parser/docker.rs @@ -207,7 +207,7 @@ enum NormalizationError { pub mod tests { use super::{super::test_util, *}; use crate::{test_util::trace_init, transforms::Transform}; - use vrl::compiler::value; + use vrl::value::value; fn make_long_string(base: &str, len: usize) -> String { base.chars().cycle().take(len).collect() diff --git a/src/sources/kubernetes_logs/parser/mod.rs b/src/sources/kubernetes_logs/parser/mod.rs index 1a0108308bd8e..37cb569fe7587 100644 --- a/src/sources/kubernetes_logs/parser/mod.rs +++ b/src/sources/kubernetes_logs/parser/mod.rs @@ -84,7 +84,7 @@ impl FunctionTransform for Parser { mod tests { use bytes::Bytes; use lookup::event_path; - use vrl::compiler::value; + use vrl::value::value; use super::*; use crate::{event::Event, event::LogEvent, test_util::trace_init, transforms::Transform}; diff --git a/src/sources/kubernetes_logs/parser/test_util.rs b/src/sources/kubernetes_logs/parser/test_util.rs index 07deeb48c8e4c..13cd4220620a0 100644 --- a/src/sources/kubernetes_logs/parser/test_util.rs +++ b/src/sources/kubernetes_logs/parser/test_util.rs @@ -4,8 +4,7 @@ use similar_asserts::assert_eq; use chrono::{DateTime, Utc}; use lookup::{event_path, metadata_path}; use vector_core::{config::LogNamespace, event}; -use vrl::compiler::value; -use vrl::value::Value; +use vrl::value::{value, Value}; use crate::{ event::{Event, LogEvent}, diff --git a/src/sources/opentelemetry/tests.rs b/src/sources/opentelemetry/tests.rs index 1093db34d08e3..a6085cff20769 100644 --- a/src/sources/opentelemetry/tests.rs +++ b/src/sources/opentelemetry/tests.rs @@ -12,7 +12,7 @@ use similar_asserts::assert_eq; use std::collections::BTreeMap; use tonic::Request; use vector_core::config::LogNamespace; -use vrl::compiler::value; +use vrl::value::value; use crate::{ config::{SourceConfig, SourceContext}, diff --git a/src/sources/redis/mod.rs b/src/sources/redis/mod.rs index ff58afcd08afa..b087b0f5f9b11 100644 --- a/src/sources/redis/mod.rs +++ b/src/sources/redis/mod.rs @@ -321,7 +321,7 @@ mod integration_test { }, SourceSender, }; - use vrl::compiler::value; + use vrl::value::value; const REDIS_SERVER: &str = "redis://redis:6379/0"; diff --git a/src/sources/socket/mod.rs b/src/sources/socket/mod.rs index a54dfd99c7009..7f7c985b1a67a 100644 --- a/src/sources/socket/mod.rs +++ b/src/sources/socket/mod.rs @@ -344,7 +344,7 @@ mod test { time::{timeout, Duration, Instant}, }; use vector_core::event::EventContainer; - use vrl::compiler::value; + use vrl::value::value; use vrl::value::{btreemap, Value}; #[cfg(unix)] From 9755a72a8a60aa58d24a0a34b62c48770b123f90 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Wed, 3 May 2023 15:34:31 -0400 Subject: [PATCH 06/23] save --- Cargo.lock | 2 +- lib/vector-vrl/functions/Cargo.toml | 2 +- lib/vector-vrl/functions/src/get_secret.rs | 2 +- lib/vector-vrl/functions/src/remove_secret.rs | 2 +- lib/vector-vrl/functions/src/set_secret.rs | 4 +--- lib/vector-vrl/functions/src/set_semantic_meaning.rs | 2 +- 6 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b6f801851178c..04c61b3466c07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9682,6 +9682,7 @@ dependencies = [ "chrono-tz", "dyn-clone", "getrandom 0.2.9", + "indoc", "lalrpop-util", "ordered-float 3.7.0", "paste", @@ -9710,7 +9711,6 @@ dependencies = [ "serde", "serde_json", "snafu", - "tracing 0.1.37", "value", "vrl-diagnostic", ] diff --git a/lib/vector-vrl/functions/Cargo.toml b/lib/vector-vrl/functions/Cargo.toml index c4619a61b6264..77b73f8cb7312 100644 --- a/lib/vector-vrl/functions/Cargo.toml +++ b/lib/vector-vrl/functions/Cargo.toml @@ -7,4 +7,4 @@ publish = false license = "MPL-2.0" [dependencies] -vrl = { path = "/home/nathan/repos/vrl" } +vrl = { path = "/home/nathan/repos/vrl", default-features = false, features = ["compiler", "path", "diagnostic"] } diff --git a/lib/vector-vrl/functions/src/get_secret.rs b/lib/vector-vrl/functions/src/get_secret.rs index 1ef9b8b89f0b9..be85c557c94ba 100644 --- a/lib/vector-vrl/functions/src/get_secret.rs +++ b/lib/vector-vrl/functions/src/get_secret.rs @@ -1,4 +1,4 @@ -use vrl::stdlib::prelude::*; +use vrl::compiler::prelude::*; fn get_secret(ctx: &mut Context, key: Value) -> std::result::Result { let key_bytes = key.as_bytes().expect("argument must be a string"); diff --git a/lib/vector-vrl/functions/src/remove_secret.rs b/lib/vector-vrl/functions/src/remove_secret.rs index 71f81439734c5..198c0d2a03bcc 100644 --- a/lib/vector-vrl/functions/src/remove_secret.rs +++ b/lib/vector-vrl/functions/src/remove_secret.rs @@ -1,4 +1,4 @@ -use vrl::stdlib::prelude::*; +use vrl::compiler::prelude::*; fn remove_secret(ctx: &mut Context, key: Value) -> std::result::Result { let key_bytes = key.as_bytes().expect("argument must be a string"); diff --git a/lib/vector-vrl/functions/src/set_secret.rs b/lib/vector-vrl/functions/src/set_secret.rs index 4254043ff65c8..a2a1f03fa7f72 100644 --- a/lib/vector-vrl/functions/src/set_secret.rs +++ b/lib/vector-vrl/functions/src/set_secret.rs @@ -1,6 +1,4 @@ -use vrl::stdlib::prelude::*; -// use vrl::value::kind: -use vrl::compiler::TypeDef; +use vrl::compiler::prelude::*; fn set_secret( ctx: &mut Context, diff --git a/lib/vector-vrl/functions/src/set_semantic_meaning.rs b/lib/vector-vrl/functions/src/set_semantic_meaning.rs index 861b13d990c96..3cdd53ac3324b 100644 --- a/lib/vector-vrl/functions/src/set_semantic_meaning.rs +++ b/lib/vector-vrl/functions/src/set_semantic_meaning.rs @@ -1,8 +1,8 @@ use std::collections::BTreeMap; use std::ops::{Deref, DerefMut}; +use vrl::compiler::prelude::*; use vrl::diagnostic::Label; use vrl::path::OwnedValuePath; -use vrl::stdlib::prelude::*; #[derive(Debug, Default, Clone)] pub struct MeaningList(pub BTreeMap); From 903e879ac53c69fd9d1385ac84199939d7be66c9 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Wed, 3 May 2023 15:39:52 -0400 Subject: [PATCH 07/23] save --- lib/enrichment/src/find_enrichment_table_records.rs | 2 +- lib/enrichment/src/get_enrichment_table_record.rs | 2 +- lib/enrichment/src/vrl_util.rs | 2 +- lib/vector-vrl/functions/src/get_secret.rs | 2 +- lib/vector-vrl/functions/src/remove_secret.rs | 2 +- lib/vector-vrl/functions/src/set_secret.rs | 2 +- lib/vector-vrl/functions/src/set_semantic_meaning.rs | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/enrichment/src/find_enrichment_table_records.rs b/lib/enrichment/src/find_enrichment_table_records.rs index 50d43194129a1..f4fbfd03c3ea6 100644 --- a/lib/enrichment/src/find_enrichment_table_records.rs +++ b/lib/enrichment/src/find_enrichment_table_records.rs @@ -1,5 +1,5 @@ use std::collections::BTreeMap; -use vrl::stdlib::prelude::*; +use vrl::prelude::*; use crate::{ vrl_util::{self, add_index, evaluate_condition}, diff --git a/lib/enrichment/src/get_enrichment_table_record.rs b/lib/enrichment/src/get_enrichment_table_record.rs index 8f4e9e2a7845e..93c432ccdd09f 100644 --- a/lib/enrichment/src/get_enrichment_table_record.rs +++ b/lib/enrichment/src/get_enrichment_table_record.rs @@ -1,5 +1,5 @@ use std::collections::BTreeMap; -use vrl::stdlib::prelude::*; +use vrl::prelude::*; use crate::{ vrl_util::{self, add_index, evaluate_condition}, diff --git a/lib/enrichment/src/vrl_util.rs b/lib/enrichment/src/vrl_util.rs index 3f73094406b32..f6aaa4dac02ce 100644 --- a/lib/enrichment/src/vrl_util.rs +++ b/lib/enrichment/src/vrl_util.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use crate::{Case, Condition, IndexHandle, TableRegistry}; use vrl::diagnostic::{Label, Span}; -use vrl::stdlib::prelude::*; +use vrl::prelude::*; #[derive(Debug)] pub enum Error { diff --git a/lib/vector-vrl/functions/src/get_secret.rs b/lib/vector-vrl/functions/src/get_secret.rs index be85c557c94ba..f559925daf2dc 100644 --- a/lib/vector-vrl/functions/src/get_secret.rs +++ b/lib/vector-vrl/functions/src/get_secret.rs @@ -1,4 +1,4 @@ -use vrl::compiler::prelude::*; +use vrl::prelude::*; fn get_secret(ctx: &mut Context, key: Value) -> std::result::Result { let key_bytes = key.as_bytes().expect("argument must be a string"); diff --git a/lib/vector-vrl/functions/src/remove_secret.rs b/lib/vector-vrl/functions/src/remove_secret.rs index 198c0d2a03bcc..1e6b6bd7f8349 100644 --- a/lib/vector-vrl/functions/src/remove_secret.rs +++ b/lib/vector-vrl/functions/src/remove_secret.rs @@ -1,4 +1,4 @@ -use vrl::compiler::prelude::*; +use vrl::prelude::*; fn remove_secret(ctx: &mut Context, key: Value) -> std::result::Result { let key_bytes = key.as_bytes().expect("argument must be a string"); diff --git a/lib/vector-vrl/functions/src/set_secret.rs b/lib/vector-vrl/functions/src/set_secret.rs index a2a1f03fa7f72..d7c0d71f0ad21 100644 --- a/lib/vector-vrl/functions/src/set_secret.rs +++ b/lib/vector-vrl/functions/src/set_secret.rs @@ -1,4 +1,4 @@ -use vrl::compiler::prelude::*; +use vrl::prelude::*; fn set_secret( ctx: &mut Context, diff --git a/lib/vector-vrl/functions/src/set_semantic_meaning.rs b/lib/vector-vrl/functions/src/set_semantic_meaning.rs index 3cdd53ac3324b..1f9d09f122526 100644 --- a/lib/vector-vrl/functions/src/set_semantic_meaning.rs +++ b/lib/vector-vrl/functions/src/set_semantic_meaning.rs @@ -1,8 +1,8 @@ use std::collections::BTreeMap; use std::ops::{Deref, DerefMut}; -use vrl::compiler::prelude::*; use vrl::diagnostic::Label; use vrl::path::OwnedValuePath; +use vrl::prelude::*; #[derive(Debug, Default, Clone)] pub struct MeaningList(pub BTreeMap); From 012a3914fadd49b211c3fa88a19728ab84d85f9c Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Wed, 3 May 2023 16:05:46 -0400 Subject: [PATCH 08/23] save --- lib/vector-core/Cargo.toml | 5 +---- lib/vector-core/src/config/mod.rs | 2 +- .../src/event/estimated_json_encoded_size_of.rs | 2 +- lib/vector-core/src/event/log_event.rs | 2 +- lib/vector-core/src/event/metadata.rs | 2 +- lib/vector-core/src/event/metric/mod.rs | 9 ++++----- lib/vector-core/src/event/metric/value.rs | 4 ++-- lib/vector-core/src/event/mod.rs | 2 +- lib/vector-core/src/event/proto.rs | 10 +++++----- lib/vector-core/src/event/vrl_target.rs | 17 +++++++---------- lib/vector-core/src/lib.rs | 5 ++--- lib/vector-core/src/schema/definition.rs | 2 +- lib/vector-core/src/schema/requirement.rs | 2 +- lib/vector-core/src/vrl.rs | 6 ++---- 14 files changed, 30 insertions(+), 40 deletions(-) diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 4d427545b27e8..f8c2e9b05235b 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -60,15 +60,12 @@ tracing-subscriber = { version = "0.3.16", default-features = false, features = typetag = { version = "0.2.7", default-features = false } twox-hash = { version = "1.6.3", default-features = false } url = { version = "2", default-features = false } -value = { package = "value", path = "/home/nathan/repos/vrl/lib/value", default-features = false, features = ["lua", "toml", "json", "api"] } vector-buffers = { path = "../vector-buffers", default-features = false } vector-common = { path = "../vector-common" } vector-config = { path = "../vector-config" } vector-config-common = { path = "../vector-config-common" } vector-config-macros = { path = "../vector-config-macros" } -# Rename to "vrl" once we use a release with stable `-Z namespaced-features`: -# https://doc.rust-lang.org/cargo/reference/unstable.html#namespaced-features -vrl-lib = { package = "vrl", path = "/home/nathan/repos/vrl" } +vrl = { path = "/home/nathan/repos/vrl" } [target.'cfg(target_os = "macos")'.dependencies] security-framework = "2.8.2" diff --git a/lib/vector-core/src/config/mod.rs b/lib/vector-core/src/config/mod.rs index 27c653380bfb0..c2f8d66c715c0 100644 --- a/lib/vector-core/src/config/mod.rs +++ b/lib/vector-core/src/config/mod.rs @@ -15,9 +15,9 @@ pub use log_schema::{init_log_schema, log_schema, LogSchema}; use lookup::{lookup_v2::ValuePath, path, PathPrefix}; pub use output_id::OutputId; use serde::{Deserialize, Serialize}; -use value::Value; pub use vector_common::config::ComponentKey; use vector_config::configurable_component; +use vrl::value::Value; use crate::schema; diff --git a/lib/vector-core/src/event/estimated_json_encoded_size_of.rs b/lib/vector-core/src/event/estimated_json_encoded_size_of.rs index 9fced6f106d5e..8ee523d13f5cd 100644 --- a/lib/vector-core/src/event/estimated_json_encoded_size_of.rs +++ b/lib/vector-core/src/event/estimated_json_encoded_size_of.rs @@ -4,7 +4,7 @@ use bytes::Bytes; use chrono::{DateTime, Timelike, Utc}; use ordered_float::NotNan; use smallvec::SmallVec; -use value::Value; +use vrl::value::Value; const NULL_SIZE: usize = 4; const TRUE_SIZE: usize = 4; diff --git a/lib/vector-core/src/event/log_event.rs b/lib/vector-core/src/event/log_event.rs index bcd70b5e1ea19..a1e1642e74094 100644 --- a/lib/vector-core/src/event/log_event.rs +++ b/lib/vector-core/src/event/log_event.rs @@ -694,7 +694,7 @@ mod test { use super::*; use crate::test_util::open_fixture; use lookup::event_path; - use vrl_lib::value::value; + use vrl::value::value; // The following two tests assert that renaming a key has no effect if the // keys are equivalent, whether the key exists in the log or not. diff --git a/lib/vector-core/src/event/metadata.rs b/lib/vector-core/src/event/metadata.rs index ee71c8a928467..e24fc5dd330a3 100644 --- a/lib/vector-core/src/event/metadata.rs +++ b/lib/vector-core/src/event/metadata.rs @@ -4,8 +4,8 @@ use std::collections::BTreeMap; use std::sync::Arc; use serde::{Deserialize, Serialize}; -use value::{Kind, Secrets, Value}; use vector_common::EventDataEq; +use vrl::value::{Kind, Secrets, Value}; use super::{BatchNotifier, EventFinalizer, EventFinalizers, EventStatus}; use crate::config::LogNamespace; diff --git a/lib/vector-core/src/event/metric/mod.rs b/lib/vector-core/src/event/metric/mod.rs index 6f63a61febee6..0253ef7617e0d 100644 --- a/lib/vector-core/src/event/metric/mod.rs +++ b/lib/vector-core/src/event/metric/mod.rs @@ -9,8 +9,7 @@ use std::{ use chrono::{DateTime, Utc}; use vector_common::EventDataEq; use vector_config::configurable_component; -#[cfg(feature = "vrl")] -use vrl_lib::compiler::value::VrlValueConvert; +use vrl::compiler::value::VrlValueConvert; use crate::{ event::{ @@ -495,10 +494,10 @@ pub enum MetricKind { } #[cfg(feature = "vrl")] -impl TryFrom<::value::Value> for MetricKind { +impl TryFrom for MetricKind { type Error = String; - fn try_from(value: ::value::Value) -> Result { + fn try_from(value: vrl::value::Value) -> Result { let value = value.try_bytes().map_err(|e| e.to_string())?; match std::str::from_utf8(&value).map_err(|e| e.to_string())? { "incremental" => Ok(Self::Incremental), @@ -511,7 +510,7 @@ impl TryFrom<::value::Value> for MetricKind { } #[cfg(feature = "vrl")] -impl From for ::value::Value { +impl From for vrl::value::Value { fn from(kind: MetricKind) -> Self { match kind { MetricKind::Incremental => "incremental".into(), diff --git a/lib/vector-core/src/event/metric/value.rs b/lib/vector-core/src/event/metric/value.rs index d2d071b0c8c30..6e27e61f3848d 100644 --- a/lib/vector-core/src/event/metric/value.rs +++ b/lib/vector-core/src/event/metric/value.rs @@ -506,7 +506,7 @@ impl From for MetricValue { // Currently, VRL can only read the type of the value and doesn't consider any actual metric values. #[cfg(feature = "vrl")] -impl From for ::value::Value { +impl From for vrl::value::Value { fn from(value: MetricValue) -> Self { value.as_name().into() } @@ -568,7 +568,7 @@ impl ByteSizeOf for MetricSketch { // Currently, VRL can only read the type of the value and doesn't consider ny actual metric values. #[cfg(feature = "vrl")] -impl From for ::value::Value { +impl From for vrl::value::Value { fn from(value: MetricSketch) -> Self { value.as_name().into() } diff --git a/lib/vector-core/src/event/mod.rs b/lib/vector-core/src/event/mod.rs index 03947095037f3..028d3cea7cf91 100644 --- a/lib/vector-core/src/event/mod.rs +++ b/lib/vector-core/src/event/mod.rs @@ -5,7 +5,6 @@ use std::{ }; use crate::ByteSizeOf; -pub use ::value::Value; pub use array::{into_event_stream, EventArray, EventContainer, LogArray, MetricArray, TraceArray}; pub use estimated_json_encoded_size_of::EstimatedJsonEncodedSizeOf; pub use finalization::{ @@ -20,6 +19,7 @@ use serde::{Deserialize, Serialize}; pub use trace::TraceEvent; use vector_buffers::EventCount; use vector_common::{finalization, EventDataEq}; +pub use vrl::value::Value; #[cfg(feature = "vrl")] pub use vrl_target::{TargetEvents, VrlTarget}; diff --git a/lib/vector-core/src/event/proto.rs b/lib/vector-core/src/event/proto.rs index 97c0414688f44..a00a2c7a317fd 100644 --- a/lib/vector-core/src/event/proto.rs +++ b/lib/vector-core/src/event/proto.rs @@ -10,10 +10,10 @@ use crate::{ mod proto_event { include!(concat!(env!("OUT_DIR"), "/event.rs")); } -pub use proto_event::*; - pub use event_wrapper::Event; pub use metric::Value as MetricValue; +pub use proto_event::*; +use vrl::value::Value as VrlValue; use super::{array, metric::MetricSketch}; @@ -92,7 +92,7 @@ impl From for Event { impl From for event::LogEvent { fn from(log: Log) -> Self { let mut event_log = if let Some(value) = log.value { - Self::from(decode_value(value).unwrap_or(::value::Value::Null)) + Self::from(decode_value(value).unwrap_or(VrlValue::Null)) } else { // This is for backwards compatibility. Only `value` should be set let fields = log @@ -285,7 +285,7 @@ impl From for WithMetadata { // Once this backwards compatibility is no longer required, "fields" can // be entirely removed from the Log object - let data = if let ::value::Value::Object(fields) = value { + let data = if let VrlValue::Object(fields) = value { // using only "fields" to prevent having to use the dummy value Log { fields: fields @@ -299,7 +299,7 @@ impl From for WithMetadata { let mut dummy = BTreeMap::new(); // must insert at least 1 field, otherwise it is emitted entirely. // this value is ignored in the decoding step (since value is provided) - dummy.insert(".".to_owned(), encode_value(::value::Value::Null)); + dummy.insert(".".to_owned(), encode_value(VrlValue::Null)); Log { fields: dummy, value: Some(encode_value(value)), diff --git a/lib/vector-core/src/event/vrl_target.rs b/lib/vector-core/src/event/vrl_target.rs index d320d068f472c..a24cb9d5f72b5 100644 --- a/lib/vector-core/src/event/vrl_target.rs +++ b/lib/vector-core/src/event/vrl_target.rs @@ -4,10 +4,11 @@ use std::{collections::BTreeMap, convert::TryFrom, marker::PhantomData}; use lookup::lookup_v2::OwnedSegment; use lookup::{OwnedTargetPath, OwnedValuePath, PathPrefix}; use snafu::Snafu; -use vrl_lib::compiler::value::VrlValueConvert; -use vrl_lib::compiler::{ProgramInfo, SecretTarget, Target}; +use vrl::compiler::value::VrlValueConvert; +use vrl::compiler::{ProgramInfo, SecretTarget, Target}; +use vrl::value::Value; -use super::{Event, EventMetadata, LogEvent, Metric, MetricKind, TraceEvent, Value}; +use super::{Event, EventMetadata, LogEvent, Metric, MetricKind, TraceEvent}; use crate::config::log_schema; use crate::event::metric::TagValue; @@ -200,11 +201,7 @@ fn set_metric_tag_values(name: String, value: &Value, metric: &mut Metric, multi } impl Target for VrlTarget { - fn target_insert( - &mut self, - target_path: &OwnedTargetPath, - value: ::value::Value, - ) -> Result<(), String> { + fn target_insert(&mut self, target_path: &OwnedTargetPath, value: Value) -> Result<(), String> { let path = &target_path.path; match target_path.prefix { PathPrefix::Event => match self { @@ -327,7 +324,7 @@ impl Target for VrlTarget { &mut self, target_path: &OwnedTargetPath, compact: bool, - ) -> Result, String> { + ) -> Result, String> { match target_path.prefix { PathPrefix::Event => match self { VrlTarget::LogEvent(ref mut log, _) | VrlTarget::Trace(ref mut log, _) => { @@ -353,7 +350,7 @@ impl Target for VrlTarget { ["tags"] => metric.series.tags.take().map(|map| { map.into_iter_single() .map(|(k, v)| (k, v.into())) - .collect::<::value::Value>() + .collect::() }), ["tags", field] => metric.remove_tag(field).map(Into::into), _ => { diff --git a/lib/vector-core/src/lib.rs b/lib/vector-core/src/lib.rs index 0002dc8b2df81..a6b0488b018ab 100644 --- a/lib/vector-core/src/lib.rs +++ b/lib/vector-core/src/lib.rs @@ -46,12 +46,11 @@ pub mod transform; #[cfg(feature = "vrl")] mod vrl; -use std::path::PathBuf; - use float_eq::FloatEq; +use std::path::PathBuf; #[cfg(feature = "vrl")] -pub use vrl::compile_vrl; +pub use crate::vrl::compile_vrl; pub use vector_buffers as buffers; #[cfg(any(test, feature = "test"))] diff --git a/lib/vector-core/src/schema/definition.rs b/lib/vector-core/src/schema/definition.rs index 0442337bba094..f499764fa5cfd 100644 --- a/lib/vector-core/src/schema/definition.rs +++ b/lib/vector-core/src/schema/definition.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use crate::config::{log_schema, LegacyKey, LogNamespace}; use lookup::lookup_v2::{parse_value_path, TargetPath}; use lookup::{owned_value_path, OwnedTargetPath, OwnedValuePath, PathPrefix}; -use value::{kind::Collection, Kind}; +use vrl::value::{kind::Collection, Kind}; /// The definition of a schema. /// diff --git a/lib/vector-core/src/schema/requirement.rs b/lib/vector-core/src/schema/requirement.rs index 6a9b645f2bca1..6b3721d606b21 100644 --- a/lib/vector-core/src/schema/requirement.rs +++ b/lib/vector-core/src/schema/requirement.rs @@ -1,7 +1,7 @@ use std::collections::{BTreeMap, BTreeSet}; use lookup::OwnedTargetPath; -use value::Kind; +use vrl::value::Kind; use crate::config::LogNamespace; diff --git a/lib/vector-core/src/vrl.rs b/lib/vector-core/src/vrl.rs index 1f6424a399346..22f649929c181 100644 --- a/lib/vector-core/src/vrl.rs +++ b/lib/vector-core/src/vrl.rs @@ -1,8 +1,6 @@ use lookup::{owned_value_path, OwnedTargetPath}; -use vrl_lib::compiler::{ - compile_with_state, CompilationResult, CompileConfig, Function, TypeState, -}; -use vrl_lib::diagnostic::DiagnosticList; +use vrl::compiler::{compile_with_state, CompilationResult, CompileConfig, Function, TypeState}; +use vrl::diagnostic::DiagnosticList; /// Compiles a VRL program /// Vector metadata is set to read-only to prevent it from being mutated From eef3cd11b0ae7aeb3b6fa3d4fcd227067b0b2233 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Wed, 3 May 2023 16:12:22 -0400 Subject: [PATCH 09/23] save --- Cargo.lock | 2 +- lib/codecs/Cargo.toml | 2 +- lib/codecs/src/decoding/format/bytes.rs | 4 ++-- lib/codecs/src/decoding/format/gelf.rs | 12 +++++------- lib/codecs/src/decoding/format/json.rs | 2 +- lib/codecs/src/decoding/format/native.rs | 2 +- lib/codecs/src/decoding/format/native_json.rs | 4 ++-- lib/codecs/src/decoding/format/syslog.rs | 2 +- lib/codecs/src/encoding/format/avro.rs | 2 +- lib/codecs/src/encoding/format/gelf.rs | 4 ++-- lib/codecs/src/encoding/format/json.rs | 2 +- lib/codecs/src/encoding/format/logfmt.rs | 2 +- lib/codecs/src/encoding/format/native_json.rs | 2 +- lib/codecs/src/encoding/format/raw_message.rs | 2 +- lib/codecs/src/encoding/format/text.rs | 2 +- lib/vector-core/src/event/metric/mod.rs | 5 ++++- 16 files changed, 26 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 04c61b3466c07..9e5523c623d7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2011,13 +2011,13 @@ dependencies = [ "tokio", "tokio-util", "tracing 0.1.37", - "value", "vector-common", "vector-config", "vector-config-common", "vector-config-macros", "vector-core", "vector-lookup", + "vrl", ] [[package]] diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 3e3ef5c9ac6a9..f330d612fc649 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -25,7 +25,7 @@ snafu = { version = "0.7.4", default-features = false, features = ["futures"] } syslog_loose = { version = "0.18", default-features = false, optional = true } tokio-util = { version = "0.7", default-features = false, features = ["codec"] } tracing = { version = "0.1", default-features = false } -value = { package = "value", path = "/home/nathan/repos/vrl/lib/value", default-features = false } +vrl = { path = "/home/nathan/repos/vrl", default-features = false, features = ["value"] } vector-common = { path = "../vector-common", default-features = false } vector-config = { path = "../vector-config", default-features = false } vector-config-common = { path = "../vector-config-common", default-features = false } diff --git a/lib/codecs/src/decoding/format/bytes.rs b/lib/codecs/src/decoding/format/bytes.rs index 7242d042abacd..e27df861cf1e0 100644 --- a/lib/codecs/src/decoding/format/bytes.rs +++ b/lib/codecs/src/decoding/format/bytes.rs @@ -3,13 +3,13 @@ use lookup::lookup_v2::parse_value_path; use lookup::OwnedTargetPath; use serde::{Deserialize, Serialize}; use smallvec::{smallvec, SmallVec}; -use value::Kind; use vector_core::config::LogNamespace; use vector_core::{ config::{log_schema, DataType}, event::{Event, LogEvent}, schema, }; +use vrl::value::Kind; use super::Deserializer; @@ -99,8 +99,8 @@ impl Deserializer for BytesDeserializer { #[cfg(test)] mod tests { - use value::Value; use vector_core::config::log_schema; + use vrl::value::Value; use super::*; diff --git a/lib/codecs/src/decoding/format/gelf.rs b/lib/codecs/src/decoding/format/gelf.rs index a2fa7f08eb167..050bcdfc21912 100644 --- a/lib/codecs/src/decoding/format/gelf.rs +++ b/lib/codecs/src/decoding/format/gelf.rs @@ -4,8 +4,6 @@ use lookup::{event_path, owned_value_path, PathPrefix}; use serde::{Deserialize, Serialize}; use smallvec::{smallvec, SmallVec}; use std::collections::HashMap; -use value::kind::Collection; -use value::Kind; use vector_core::config::LogNamespace; use vector_core::{ config::{log_schema, DataType}, @@ -13,6 +11,8 @@ use vector_core::{ event::LogEvent, schema, }; +use vrl::value::kind::Collection; +use vrl::value::{Kind, Value}; use super::Deserializer; use crate::{gelf_fields::*, VALID_FIELD_REGEX}; @@ -124,9 +124,7 @@ impl GelfDeserializer { if let Some(line) = parsed.line { log.insert( LINE, - value::Value::Float( - ordered_float::NotNan::new(line).expect("JSON doesn't allow NaNs"), - ), + Value::Float(ordered_float::NotNan::new(line).expect("JSON doesn't allow NaNs")), ); } if let Some(file) = &parsed.file { @@ -156,7 +154,7 @@ impl GelfDeserializer { // per GELF spec, Additional field values must be either strings or numbers if val.is_string() || val.is_number() { - let vector_val: value::Value = val.into(); + let vector_val: Value = val.into(); log.insert(event_path!(key.as_str()), vector_val); } else { let type_ = match val { @@ -216,8 +214,8 @@ mod tests { use serde_json::json; use similar_asserts::assert_eq; use smallvec::SmallVec; - use value::Value; use vector_core::{config::log_schema, event::Event}; + use vrl::value::Value; fn deserialize_gelf_input( input: &serde_json::Value, diff --git a/lib/codecs/src/decoding/format/json.rs b/lib/codecs/src/decoding/format/json.rs index 6b2ad8a804732..32f28e5e58436 100644 --- a/lib/codecs/src/decoding/format/json.rs +++ b/lib/codecs/src/decoding/format/json.rs @@ -5,12 +5,12 @@ use chrono::Utc; use lookup::PathPrefix; use serde::{Deserialize, Serialize}; use smallvec::{smallvec, SmallVec}; -use value::Kind; use vector_core::{ config::{log_schema, DataType, LogNamespace}, event::Event, schema, }; +use vrl::value::Kind; use super::Deserializer; diff --git a/lib/codecs/src/decoding/format/native.rs b/lib/codecs/src/decoding/format/native.rs index bbaa2ba51f9d4..1f47414e103f5 100644 --- a/lib/codecs/src/decoding/format/native.rs +++ b/lib/codecs/src/decoding/format/native.rs @@ -2,13 +2,13 @@ use bytes::Bytes; use prost::Message; use serde::{Deserialize, Serialize}; use smallvec::{smallvec, SmallVec}; -use value::Kind; use vector_core::config::LogNamespace; use vector_core::{ config::DataType, event::{proto, Event, EventArray, EventContainer}, schema, }; +use vrl::value::Kind; use super::Deserializer; diff --git a/lib/codecs/src/decoding/format/native_json.rs b/lib/codecs/src/decoding/format/native_json.rs index 5e1a351d9e787..038a36c69fa28 100644 --- a/lib/codecs/src/decoding/format/native_json.rs +++ b/lib/codecs/src/decoding/format/native_json.rs @@ -1,9 +1,9 @@ use bytes::Bytes; use serde::{Deserialize, Serialize}; use smallvec::{smallvec, SmallVec}; -use value::kind::Collection; -use value::Kind; use vector_core::{config::DataType, event::Event, schema}; +use vrl::value::kind::Collection; +use vrl::value::Kind; use super::Deserializer; use vector_core::config::LogNamespace; diff --git a/lib/codecs/src/decoding/format/syslog.rs b/lib/codecs/src/decoding/format/syslog.rs index c9e52aa083599..68824b0e923d5 100644 --- a/lib/codecs/src/decoding/format/syslog.rs +++ b/lib/codecs/src/decoding/format/syslog.rs @@ -6,13 +6,13 @@ use serde::{Deserialize, Serialize}; use smallvec::{smallvec, SmallVec}; use std::collections::BTreeMap; use syslog_loose::{IncompleteDate, Message, ProcId, Protocol}; -use value::{kind::Collection, Kind}; use vector_core::config::{LegacyKey, LogNamespace}; use vector_core::{ config::{log_schema, DataType}, event::{Event, LogEvent, Value}, schema, }; +use vrl::value::{kind::Collection, Kind}; use super::Deserializer; diff --git a/lib/codecs/src/encoding/format/avro.rs b/lib/codecs/src/encoding/format/avro.rs index c66b19d2931eb..35bf45890043d 100644 --- a/lib/codecs/src/encoding/format/avro.rs +++ b/lib/codecs/src/encoding/format/avro.rs @@ -80,8 +80,8 @@ impl Encoder for AvroSerializer { mod tests { use bytes::BytesMut; use indoc::indoc; - use value::btreemap; use vector_core::event::{LogEvent, Value}; + use vrl::value::btreemap; use super::*; diff --git a/lib/codecs/src/encoding/format/gelf.rs b/lib/codecs/src/encoding/format/gelf.rs index 96a85f81f3eff..799fe41adb526 100644 --- a/lib/codecs/src/encoding/format/gelf.rs +++ b/lib/codecs/src/encoding/format/gelf.rs @@ -240,9 +240,9 @@ mod tests { use super::*; use chrono::{DateTime, NaiveDateTime, Utc}; - use value::btreemap; - use value::Value; use vector_core::event::{Event, EventMetadata}; + use vrl::value::btreemap; + use vrl::value::Value; fn do_serialize( expect_success: bool, diff --git a/lib/codecs/src/encoding/format/json.rs b/lib/codecs/src/encoding/format/json.rs index eb79edc417c91..73ab79c78c095 100644 --- a/lib/codecs/src/encoding/format/json.rs +++ b/lib/codecs/src/encoding/format/json.rs @@ -89,9 +89,9 @@ impl Encoder for JsonSerializer { mod tests { use bytes::{Bytes, BytesMut}; use chrono::{TimeZone, Timelike, Utc}; - use value::btreemap; use vector_core::event::{LogEvent, Metric, MetricKind, MetricValue, StatisticKind, Value}; use vector_core::metric_tags; + use vrl::value::btreemap; use super::*; diff --git a/lib/codecs/src/encoding/format/logfmt.rs b/lib/codecs/src/encoding/format/logfmt.rs index 7ed46b1b18a00..37b0351b085ee 100644 --- a/lib/codecs/src/encoding/format/logfmt.rs +++ b/lib/codecs/src/encoding/format/logfmt.rs @@ -59,8 +59,8 @@ impl Encoder for LogfmtSerializer { mod tests { use super::*; use bytes::BytesMut; - use value::btreemap; use vector_core::event::{LogEvent, Value}; + use vrl::value::btreemap; #[test] fn serialize_logfmt() { diff --git a/lib/codecs/src/encoding/format/native_json.rs b/lib/codecs/src/encoding/format/native_json.rs index b56c47dadd995..6762c92737834 100644 --- a/lib/codecs/src/encoding/format/native_json.rs +++ b/lib/codecs/src/encoding/format/native_json.rs @@ -52,8 +52,8 @@ impl Encoder for NativeJsonSerializer { #[cfg(test)] mod tests { use bytes::BytesMut; - use value::btreemap; use vector_core::event::{LogEvent, Value}; + use vrl::value::btreemap; use super::*; diff --git a/lib/codecs/src/encoding/format/raw_message.rs b/lib/codecs/src/encoding/format/raw_message.rs index 55e409fbf6f7c..b955ae8355dc6 100644 --- a/lib/codecs/src/encoding/format/raw_message.rs +++ b/lib/codecs/src/encoding/format/raw_message.rs @@ -1,12 +1,12 @@ use bytes::{BufMut, BytesMut}; use serde::{Deserialize, Serialize}; use tokio_util::codec::Encoder; -use value::Kind; use vector_core::{ config::{log_schema, DataType}, event::Event, schema, }; +use vrl::value::Kind; /// Config used to build a `RawMessageSerializer`. #[derive(Debug, Clone, Default, Deserialize, Serialize)] diff --git a/lib/codecs/src/encoding/format/text.rs b/lib/codecs/src/encoding/format/text.rs index e26dee14b9b94..0b9e49a44b976 100644 --- a/lib/codecs/src/encoding/format/text.rs +++ b/lib/codecs/src/encoding/format/text.rs @@ -1,11 +1,11 @@ use bytes::{BufMut, BytesMut}; use tokio_util::codec::Encoder; -use value::Kind; use vector_core::{ config::{log_schema, DataType}, event::Event, schema, }; +use vrl::value::Kind; use crate::MetricTagValues; diff --git a/lib/vector-core/src/event/metric/mod.rs b/lib/vector-core/src/event/metric/mod.rs index 0253ef7617e0d..849823e301035 100644 --- a/lib/vector-core/src/event/metric/mod.rs +++ b/lib/vector-core/src/event/metric/mod.rs @@ -1,5 +1,9 @@ #[cfg(feature = "vrl")] use std::convert::TryFrom; + +#[cfg(feature = "vrl")] +use vrl::compiler::value::VrlValueConvert; + use std::{ convert::AsRef, fmt::{self, Display, Formatter}, @@ -9,7 +13,6 @@ use std::{ use chrono::{DateTime, Utc}; use vector_common::EventDataEq; use vector_config::configurable_component; -use vrl::compiler::value::VrlValueConvert; use crate::{ event::{ From 1c7b5a9aca442dc504ad1c49887bf98f0ff92626 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Wed, 3 May 2023 16:16:00 -0400 Subject: [PATCH 10/23] save --- Cargo.lock | 2 +- lib/opentelemetry-proto/Cargo.toml | 2 +- lib/opentelemetry-proto/src/convert.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e5523c623d7b..681a792c93f9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5636,9 +5636,9 @@ dependencies = [ "prost-build", "tonic 0.9.2", "tonic-build", - "value", "vector-core", "vector-lookup", + "vrl", ] [[package]] diff --git a/lib/opentelemetry-proto/Cargo.toml b/lib/opentelemetry-proto/Cargo.toml index 02e5cc28cfb1f..1b7a6e568a1b1 100644 --- a/lib/opentelemetry-proto/Cargo.toml +++ b/lib/opentelemetry-proto/Cargo.toml @@ -17,5 +17,5 @@ lookup = { package = "vector-lookup", path = "../vector-lookup", default-feature ordered-float = { version = "3.7.0", default-features = false } prost = { version = "0.11", default-features = false, features = ["std"] } tonic = { version = "0.9", default-features = false, features = ["codegen", "gzip", "prost", "tls", "tls-roots", "transport"] } -value = { package = "value", path = "/home/nathan/repos/vrl/lib/value" } +vrl = { path = "/home/nathan/repos/vrl", default-features = false, features = ["value"] } vector-core = { path = "../vector-core", default-features = false } diff --git a/lib/opentelemetry-proto/src/convert.rs b/lib/opentelemetry-proto/src/convert.rs index 7ed080eef0fa2..4ccfda0b726ba 100644 --- a/lib/opentelemetry-proto/src/convert.rs +++ b/lib/opentelemetry-proto/src/convert.rs @@ -3,11 +3,11 @@ use chrono::{DateTime, TimeZone, Utc}; use lookup::path; use ordered_float::NotNan; use std::collections::BTreeMap; -use value::Value; use vector_core::{ config::{log_schema, LegacyKey, LogNamespace}, event::{Event, LogEvent}, }; +use vrl::value::Value; use super::proto::{ common::v1::{any_value::Value as PBValue, KeyValue}, From 049bda9290911d3cfc8075451a2d2fc2a2db53a9 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Wed, 3 May 2023 16:19:59 -0400 Subject: [PATCH 11/23] save --- Cargo.lock | 2 +- lib/vector-common/Cargo.toml | 2 +- lib/vector-common/src/byte_size_of.rs | 2 +- lib/vector-common/src/lib.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 681a792c93f9e..8dbe772103fc7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9409,10 +9409,10 @@ dependencies = [ "stream-cancel", "tokio", "tracing 0.1.37", - "value", "vector-config", "vector-config-common", "vector-config-macros", + "vrl", "vrl-compiler", "vrl-core", ] diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 4a5534df06e1a..3f0ac9215f28d 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -62,9 +62,9 @@ snafu = { version = "0.7", optional = true } stream-cancel = { version = "0.8.1", default-features = false } tokio = { version = "1.28.0", default-features = false, features = ["macros", "time"] } tracing = { version = "0.1.34", default-features = false } -value = { package = "value", path = "/home/nathan/repos/vrl/lib/value", features = ["json", "arbitrary"] } vrl-core = { package = "vrl-core", path = "/home/nathan/repos/vrl/lib/core", default-features = false } vrl-compiler = { path = "/home/nathan/repos/vrl/lib/compiler", default-features = false } +vrl = { path = "/home/nathan/repos/vrl", default-features = false, features = ["value"] } vector-config = { path = "../vector-config" } vector-config-common = { path = "../vector-config-common" } vector-config-macros = { path = "../vector-config-macros" } diff --git a/lib/vector-common/src/byte_size_of.rs b/lib/vector-common/src/byte_size_of.rs index bd3fc93023cd6..0b984ac41dfa3 100644 --- a/lib/vector-common/src/byte_size_of.rs +++ b/lib/vector-common/src/byte_size_of.rs @@ -7,7 +7,7 @@ use bytes::{Bytes, BytesMut}; use chrono::{DateTime, Utc}; use serde_json::{value::RawValue, Value as JsonValue}; use smallvec::SmallVec; -use value::Value; +use vrl::value::Value; pub trait ByteSizeOf { /// Returns the in-memory size of this type diff --git a/lib/vector-common/src/lib.rs b/lib/vector-common/src/lib.rs index 4e7b537bf1cfb..ffe4716c3f0c1 100644 --- a/lib/vector-common/src/lib.rs +++ b/lib/vector-common/src/lib.rs @@ -13,7 +13,7 @@ #![deny(unused_comparisons)] #[cfg(feature = "btreemap")] -pub use value::btreemap; +pub use vrl::value::btreemap; #[cfg(feature = "byte_size_of")] pub mod byte_size_of; From e92539e2264ec8ddfa847d2eee8bb8720f73b29b Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Wed, 3 May 2023 16:24:14 -0400 Subject: [PATCH 12/23] save --- Cargo.lock | 1 - lib/prometheus-parser/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8dbe772103fc7..b9b9aedc3c2f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6226,7 +6226,6 @@ dependencies = [ "prost-build", "prost-types", "snafu", - "value", "vector-common", ] diff --git a/lib/prometheus-parser/Cargo.toml b/lib/prometheus-parser/Cargo.toml index a38b1c1b0db03..7ce50ec99e88c 100644 --- a/lib/prometheus-parser/Cargo.toml +++ b/lib/prometheus-parser/Cargo.toml @@ -16,7 +16,6 @@ prost = "0.11" prost-types = "0.11" snafu = { version = "0.7" } vector-common = { path = "../vector-common", features = ["btreemap"] } -value = { package = "value", path = "/home/nathan/repos/vrl/lib/value" } [build-dependencies] prost-build = "0.11" From 32f2579d775d65661af52f32f86cec0224712575 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Wed, 3 May 2023 16:54:00 -0400 Subject: [PATCH 13/23] save --- Cargo.lock | 3 -- lib/vector-core/Cargo.toml | 4 +-- lib/vector-core/src/config/mod.rs | 2 +- lib/vector-core/src/event/vrl_target.rs | 44 +++++++++--------------- lib/vector-core/src/fanout.rs | 2 +- lib/vector-core/src/schema/definition.rs | 2 +- 6 files changed, 22 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b9b9aedc3c2f6..74b39aeae1524 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9056,7 +9056,6 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" name = "value" version = "0.1.0" dependencies = [ - "async-graphql", "bytes 1.4.0", "chrono", "mlua", @@ -9068,7 +9067,6 @@ dependencies = [ "serde", "serde_json", "snafu", - "toml 0.7.3", "tracing 0.1.37", ] @@ -9544,7 +9542,6 @@ dependencies = [ "twox-hash", "typetag", "url", - "value", "vector-buffers", "vector-common", "vector-config", diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index f8c2e9b05235b..ef4ea87b77d07 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -94,10 +94,10 @@ rand = "0.8.5" rand_distr = "0.4.3" tracing-subscriber = { version = "0.3.16", default-features = false, features = ["env-filter", "fmt", "ansi", "registry"] } vector-common = { path = "../vector-common", default-features = false, features = ["test"] } -value = { package = "value", path = "/home/nathan/repos/vrl/lib/value", default-features = false, features = ["lua", "toml", "json", "api", "arbitrary", "test"] } +vrl = { path = "/home/nathan/repos/vrl", default-features = false, features = ["value", "arbitrary", "lua"] } [features] -api = ["dep:async-graphql", "value/api"] +api = ["dep:async-graphql"] default = [] lua = ["dep:mlua", "dep:tokio-stream"] vrl = ["dep:enrichment"] diff --git a/lib/vector-core/src/config/mod.rs b/lib/vector-core/src/config/mod.rs index c2f8d66c715c0..97cbc091d8f5e 100644 --- a/lib/vector-core/src/config/mod.rs +++ b/lib/vector-core/src/config/mod.rs @@ -553,8 +553,8 @@ mod test { use crate::event::LogEvent; use chrono::Utc; use lookup::{event_path, owned_value_path, OwnedTargetPath}; - use value::Kind; use vector_common::btreemap; + use vrl::value::Kind; #[test] fn test_insert_standard_vector_source_metadata() { diff --git a/lib/vector-core/src/event/vrl_target.rs b/lib/vector-core/src/event/vrl_target.rs index a24cb9d5f72b5..861721eb7b1aa 100644 --- a/lib/vector-core/src/event/vrl_target.rs +++ b/lib/vector-core/src/event/vrl_target.rs @@ -588,7 +588,7 @@ mod test { use chrono::{offset::TimeZone, Utc}; use lookup::owned_value_path; use similar_asserts::assert_eq; - use value::btreemap; + use vrl::value::btreemap; use super::super::MetricValue; use super::*; @@ -596,8 +596,6 @@ mod test { #[test] fn log_get() { - use value::btreemap; - let cases = vec![ ( BTreeMap::new(), @@ -652,8 +650,6 @@ mod test { #[allow(clippy::too_many_lines)] #[test] fn log_insert() { - use value::btreemap; - let cases = vec![ ( BTreeMap::from([("foo".into(), "bar".into())]), @@ -747,7 +743,7 @@ mod test { }; let mut target = VrlTarget::new(Event::Log(LogEvent::from(object)), &info, false); let expect = LogEvent::from(expect); - let value: ::value::Value = value; + let value: Value = value; let path = OwnedTargetPath::event(path); assert_eq!( @@ -862,28 +858,22 @@ mod test { #[test] fn log_into_events() { - use value::btreemap; + use vrl::value::btreemap; let cases = vec![ ( - ::value::Value::from(btreemap! {"foo" => "bar"}), + Value::from(btreemap! {"foo" => "bar"}), vec![btreemap! {"foo" => "bar"}], ), - (::value::Value::from(1), vec![btreemap! {"message" => 1}]), - ( - ::value::Value::from("2"), - vec![btreemap! {"message" => "2"}], - ), - ( - ::value::Value::from(true), - vec![btreemap! {"message" => true}], - ), + (Value::from(1), vec![btreemap! {"message" => 1}]), + (Value::from("2"), vec![btreemap! {"message" => "2"}]), + (Value::from(true), vec![btreemap! {"message" => true}]), ( - ::value::Value::from(vec![ - ::value::Value::from(1), - ::value::Value::from("2"), - ::value::Value::from(true), - ::value::Value::from(btreemap! {"foo" => "bar"}), + Value::from(vec![ + Value::from(1), + Value::from("2"), + Value::from(true), + Value::from(btreemap! {"foo" => "bar"}), ]), vec![ btreemap! {"message" => 1}, @@ -983,10 +973,10 @@ mod test { let cases = vec![ ( - owned_value_path!("name"), // Path - Some(::value::Value::from("name")), // Current value - ::value::Value::from("namefoo"), // New value - false, // Test deletion + owned_value_path!("name"), // Path + Some(Value::from("name")), // Current value + Value::from("namefoo"), // New value + false, // Test deletion ), ( owned_value_path!("namespace"), @@ -1005,7 +995,7 @@ mod test { ), ( owned_value_path!("kind"), - Some(::value::Value::from("absolute")), + Some(Value::from("absolute")), "incremental".into(), false, ), diff --git a/lib/vector-core/src/fanout.rs b/lib/vector-core/src/fanout.rs index ef007e233f191..c9596990045d0 100644 --- a/lib/vector-core/src/fanout.rs +++ b/lib/vector-core/src/fanout.rs @@ -451,7 +451,6 @@ mod tests { use futures::poll; use tokio::sync::mpsc::UnboundedSender; use tokio_test::{assert_pending, assert_ready, task::spawn}; - use value::Value; use vector_buffers::{ topology::{ builder::TopologyBuilder, @@ -459,6 +458,7 @@ mod tests { }, WhenFull, }; + use vrl::value::Value; use super::{ControlMessage, Fanout}; use crate::event::{Event, EventArray, LogEvent}; diff --git a/lib/vector-core/src/schema/definition.rs b/lib/vector-core/src/schema/definition.rs index f499764fa5cfd..62a5bd3b2ff64 100644 --- a/lib/vector-core/src/schema/definition.rs +++ b/lib/vector-core/src/schema/definition.rs @@ -598,7 +598,7 @@ mod tests { use lookup::lookup_v2::parse_target_path; use lookup::owned_value_path; use std::collections::{BTreeMap, HashMap}; - use value::Value; + use vrl::value::Value; use super::*; From b291f5f38373cc6f9bf82e573faed1ffcf36afb6 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Wed, 3 May 2023 16:59:21 -0400 Subject: [PATCH 14/23] save --- Cargo.lock | 2 -- lib/vector-common/Cargo.toml | 4 +--- lib/vector-common/src/lib.rs | 6 +++--- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 74b39aeae1524..826769819f71f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9410,8 +9410,6 @@ dependencies = [ "vector-config-common", "vector-config-macros", "vrl", - "vrl-compiler", - "vrl-core", ] [[package]] diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 3f0ac9215f28d..5f6ef28b01089 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -62,9 +62,7 @@ snafu = { version = "0.7", optional = true } stream-cancel = { version = "0.8.1", default-features = false } tokio = { version = "1.28.0", default-features = false, features = ["macros", "time"] } tracing = { version = "0.1.34", default-features = false } -vrl-core = { package = "vrl-core", path = "/home/nathan/repos/vrl/lib/core", default-features = false } -vrl-compiler = { path = "/home/nathan/repos/vrl/lib/compiler", default-features = false } -vrl = { path = "/home/nathan/repos/vrl", default-features = false, features = ["value"] } +vrl = { path = "/home/nathan/repos/vrl", default-features = false, features = ["value", "core", "compiler"] } vector-config = { path = "../vector-config" } vector-config-common = { path = "../vector-config-common" } vector-config-macros = { path = "../vector-config-macros" } diff --git a/lib/vector-common/src/lib.rs b/lib/vector-common/src/lib.rs index ffe4716c3f0c1..bcf58c72de723 100644 --- a/lib/vector-common/src/lib.rs +++ b/lib/vector-common/src/lib.rs @@ -21,15 +21,15 @@ pub mod byte_size_of; pub mod config; #[cfg(feature = "conversion")] -pub use vrl_compiler::TimeZone; +pub use vrl::compiler::TimeZone; #[cfg(feature = "encoding")] pub mod encode_logfmt { - pub use vrl_core::encode_logfmt::*; + pub use vrl::core::encode_logfmt::*; } pub mod conversion { - pub use vrl_compiler::conversion::*; + pub use vrl::compiler::conversion::*; } pub mod event_data_eq; From 7e8bb712a8a8357f21c911222fd360b8063fe984 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Wed, 3 May 2023 21:29:42 -0400 Subject: [PATCH 15/23] save --- Cargo.lock | 4 ++-- lib/vector-buffers/examples/buffer_perf.rs | 2 +- lib/vector-vrl/cli/Cargo.toml | 3 +-- lib/vector-vrl/cli/src/main.rs | 6 ++---- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 826769819f71f..9e35671e9ea57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9565,8 +9565,7 @@ version = "0.1.0" dependencies = [ "clap 4.1.14", "vector-vrl-functions", - "vrl-cli", - "vrl-stdlib", + "vrl", ] [[package]] @@ -9637,6 +9636,7 @@ dependencies = [ "datadog-search-syntax", "path", "value", + "vrl-cli", "vrl-compiler", "vrl-core", "vrl-diagnostic", diff --git a/lib/vector-buffers/examples/buffer_perf.rs b/lib/vector-buffers/examples/buffer_perf.rs index 7d27eff9a9180..54916593ebeff 100644 --- a/lib/vector-buffers/examples/buffer_perf.rs +++ b/lib/vector-buffers/examples/buffer_perf.rs @@ -235,7 +235,7 @@ fn generate_record_cache(min: usize, max: usize) -> Vec { let mut records = Vec::new(); for i in 1..=200_000 { let payload_size = rng.gen_range(min..max); - let payload = (0..payload_size).map(|_| rng.gen()).collect(); + let payload = (0..payload_size).map(|_| rng.gen::()).collect(); let message = VariableMessage::new(i, payload); records.push(message); } diff --git a/lib/vector-vrl/cli/Cargo.toml b/lib/vector-vrl/cli/Cargo.toml index 215d3ad47f884..d4323e415bc1a 100644 --- a/lib/vector-vrl/cli/Cargo.toml +++ b/lib/vector-vrl/cli/Cargo.toml @@ -7,7 +7,6 @@ publish = false license = "MPL-2.0" [dependencies] -vrl-cli = { package = "vrl-cli", path = "/home/nathan/repos/vrl/lib/cli" } clap = { version = "4.1.14", features = ["derive"] } -vrl-stdlib = { package = "vrl-stdlib", path = "/home/nathan/repos/vrl/lib/stdlib" } vector-vrl-functions = { path = "../functions" } +vrl = { path = "/home/nathan/repos/vrl", default-features = false, features = ["stdlib", "cli"] } diff --git a/lib/vector-vrl/cli/src/main.rs b/lib/vector-vrl/cli/src/main.rs index 4f6e92b9a7b6f..5f08c6f09b26f 100644 --- a/lib/vector-vrl/cli/src/main.rs +++ b/lib/vector-vrl/cli/src/main.rs @@ -1,10 +1,8 @@ -extern crate vrl_cli; - use clap::Parser; -use vrl_cli::{cmd::cmd, Opts}; +use vrl::cli::{cmd::cmd, Opts}; fn main() { - let mut functions = vrl_stdlib::all(); + let mut functions = vrl::stdlib::all(); functions.extend(vector_vrl_functions::all()); std::process::exit(cmd(&Opts::parse(), functions)); From adc7a253da9cb88efb530c58592c1f604df995c8 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Wed, 3 May 2023 21:41:43 -0400 Subject: [PATCH 16/23] save --- Cargo.lock | 5 +---- lib/vector-vrl/tests/Cargo.toml | 6 +----- lib/vector-vrl/tests/src/docs.rs | 2 +- lib/vector-vrl/tests/src/main.rs | 4 ++-- lib/vector-vrl/tests/src/test_enrichment.rs | 23 +++++++++++---------- 5 files changed, 17 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e35671e9ea57..a22d96b55b379 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9585,18 +9585,14 @@ dependencies = [ "clap 4.1.14", "enrichment", "glob", - "path", "prettydiff", "regex", "serde", "serde_json", "tikv-jemallocator", "tracing-subscriber", - "value", "vector-vrl-functions", "vrl", - "vrl-stdlib", - "vrl-tests", ] [[package]] @@ -9642,6 +9638,7 @@ dependencies = [ "vrl-diagnostic", "vrl-parser", "vrl-stdlib", + "vrl-tests", ] [[package]] diff --git a/lib/vector-vrl/tests/Cargo.toml b/lib/vector-vrl/tests/Cargo.toml index caf34283e5981..d5e21c5447371 100644 --- a/lib/vector-vrl/tests/Cargo.toml +++ b/lib/vector-vrl/tests/Cargo.toml @@ -7,11 +7,7 @@ publish = false [dependencies] enrichment = { path = "../../enrichment" } -lookup = { package = "path", path = "/home/nathan/repos/vrl/lib/path" } -stdlib = { package = "vrl-stdlib", path = "/home/nathan/repos/vrl/lib/stdlib" } -vrl = { package = "vrl", path = "/home/nathan/repos/vrl" } -value = { package = "value", path = "/home/nathan/repos/vrl/lib/value" } -vrl-tests = { package = "vrl-tests", path = "/home/nathan/repos/vrl/lib/tests" } +vrl = { path = "/home/nathan/repos/vrl", features = ["test_framework"]} vector-vrl-functions = { path = "../../vector-vrl/functions" } ansi_term = "0.12" diff --git a/lib/vector-vrl/tests/src/docs.rs b/lib/vector-vrl/tests/src/docs.rs index 22fdebf7e4037..55cf8e27a2079 100644 --- a/lib/vector-vrl/tests/src/docs.rs +++ b/lib/vector-vrl/tests/src/docs.rs @@ -117,7 +117,7 @@ fn examples_to_tests( } fn test_from_cue_example(category: &'static str, name: String, example: Example) -> Test { - use ::value::Value; + use vrl::value::Value; let Example { title, diff --git a/lib/vector-vrl/tests/src/main.rs b/lib/vector-vrl/tests/src/main.rs index c0ef1dc6aa704..037b8ebedd614 100644 --- a/lib/vector-vrl/tests/src/main.rs +++ b/lib/vector-vrl/tests/src/main.rs @@ -4,7 +4,7 @@ mod docs; mod test_enrichment; -use vrl_tests::{get_tests_from_functions, run_tests, Test, TestConfig}; +use vrl::test::{get_tests_from_functions, run_tests, Test, TestConfig}; use chrono_tz::Tz; use clap::Parser; @@ -93,7 +93,7 @@ fn main() { timezone: cmd.timezone(), }; - let mut functions = stdlib::all(); + let mut functions = vrl::stdlib::all(); functions.extend(vector_vrl_functions::all()); functions.extend(enrichment::vrl_functions()); diff --git a/lib/vector-vrl/tests/src/test_enrichment.rs b/lib/vector-vrl/tests/src/test_enrichment.rs index ffa57119014e9..71ee66d87ff20 100644 --- a/lib/vector-vrl/tests/src/test_enrichment.rs +++ b/lib/vector-vrl/tests/src/test_enrichment.rs @@ -1,4 +1,5 @@ use std::collections::{BTreeMap, HashMap}; +use vrl::value::Value; #[derive(Debug, Clone)] struct TestEnrichmentTable; @@ -10,11 +11,11 @@ impl enrichment::Table for TestEnrichmentTable { _condition: &'a [enrichment::Condition<'a>], _select: Option<&[String]>, _index: Option, - ) -> Result, String> { + ) -> Result, String> { let mut result = BTreeMap::new(); - result.insert("id".to_string(), ::value::Value::from(1)); - result.insert("firstname".to_string(), ::value::Value::from("Bob")); - result.insert("surname".to_string(), ::value::Value::from("Smith")); + result.insert("id".to_string(), Value::from(1)); + result.insert("firstname".to_string(), Value::from("Bob")); + result.insert("surname".to_string(), Value::from("Smith")); Ok(result) } @@ -25,16 +26,16 @@ impl enrichment::Table for TestEnrichmentTable { _condition: &'a [enrichment::Condition<'a>], _select: Option<&[String]>, _index: Option, - ) -> Result>, String> { + ) -> Result>, String> { let mut result1 = BTreeMap::new(); - result1.insert("id".to_string(), ::value::Value::from(1)); - result1.insert("firstname".to_string(), ::value::Value::from("Bob")); - result1.insert("surname".to_string(), ::value::Value::from("Smith")); + result1.insert("id".to_string(), Value::from(1)); + result1.insert("firstname".to_string(), Value::from("Bob")); + result1.insert("surname".to_string(), Value::from("Smith")); let mut result2 = BTreeMap::new(); - result2.insert("id".to_string(), ::value::Value::from(2)); - result2.insert("firstname".to_string(), ::value::Value::from("Fred")); - result2.insert("surname".to_string(), ::value::Value::from("Smith")); + result2.insert("id".to_string(), Value::from(2)); + result2.insert("firstname".to_string(), Value::from("Fred")); + result2.insert("surname".to_string(), Value::from("Smith")); Ok(vec![result1, result2]) } From 9d5e946c84b7ef6ec7392f2ae69ea4e843156bd8 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Wed, 3 May 2023 21:48:00 -0400 Subject: [PATCH 17/23] save --- Cargo.lock | 2 -- lib/vector-vrl/web-playground/Cargo.toml | 4 +--- lib/vector-vrl/web-playground/src/lib.rs | 6 +++--- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a22d96b55b379..f315ea244b992 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9604,10 +9604,8 @@ dependencies = [ "gloo-utils", "serde", "serde-wasm-bindgen", - "value", "vector-vrl-functions", "vrl", - "vrl-stdlib", "wasm-bindgen", ] diff --git a/lib/vector-vrl/web-playground/Cargo.toml b/lib/vector-vrl/web-playground/Cargo.toml index 6fec3444556e0..aa375f2e76e00 100644 --- a/lib/vector-vrl/web-playground/Cargo.toml +++ b/lib/vector-vrl/web-playground/Cargo.toml @@ -10,9 +10,7 @@ crate-type = ["cdylib"] [dependencies] wasm-bindgen = "0.2" -vrl = { package = "vrl", path = "/home/nathan/repos/vrl", default-features = false} -value = { package = "value", path = "/home/nathan/repos/vrl/lib/value", default-features = false} -stdlib = { package = "vrl-stdlib", path = "/home/nathan/repos/vrl/lib/stdlib"} +vrl = { path = "/home/nathan/repos/vrl", default-features = false, features = ["value", "stdlib"] } serde = { version = "1.0", features = ["derive"] } serde-wasm-bindgen = "0.5" gloo-utils = { version = "0.1", features = ["serde"] } diff --git a/lib/vector-vrl/web-playground/src/lib.rs b/lib/vector-vrl/web-playground/src/lib.rs index c2a9d1775eed4..f7f8a5aa8ce60 100644 --- a/lib/vector-vrl/web-playground/src/lib.rs +++ b/lib/vector-vrl/web-playground/src/lib.rs @@ -1,13 +1,13 @@ -use ::value::Value; use gloo_utils::format::JsValueSerdeExt; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; -use value::Secrets; use vrl::compiler::runtime::{Runtime, Terminate}; use vrl::compiler::TimeZone; use vrl::compiler::{compile_with_state, CompileConfig, TargetValue, TypeState}; use vrl::diagnostic::DiagnosticList; use vrl::diagnostic::Formatter; +use vrl::value::Secrets; +use vrl::value::Value; use wasm_bindgen::prelude::*; #[derive(Serialize, Deserialize)] @@ -73,7 +73,7 @@ impl VrlDiagnosticResult { } fn compile(mut input: Input) -> Result { - let mut functions = stdlib::all(); + let mut functions = vrl::stdlib::all(); functions.extend(vector_vrl_functions::all()); functions.extend(enrichment::vrl_functions()); From f99d66200d4ce0dbfa5076ac71850e5596556213 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Wed, 3 May 2023 22:35:49 -0400 Subject: [PATCH 18/23] save --- lib/vector-core/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index ef4ea87b77d07..e2ec3093a78aa 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -99,7 +99,7 @@ vrl = { path = "/home/nathan/repos/vrl", default-features = false, features = [" [features] api = ["dep:async-graphql"] default = [] -lua = ["dep:mlua", "dep:tokio-stream"] +lua = ["dep:mlua", "dep:tokio-stream", "vrl/lua"] vrl = ["dep:enrichment"] test = ["vector-common/test", "proptest"] From ee2e2e842d7008ba94b1553d5e21041bc21b2337 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Wed, 3 May 2023 22:58:03 -0400 Subject: [PATCH 19/23] save --- Cargo.lock | 4 ++- Cargo.toml | 35 +++++++++---------- benches/enrichment_tables.rs | 2 +- benches/transform/route.rs | 2 +- lib/vector-core/Cargo.toml | 1 + src/codecs/decoding/decoder.rs | 2 +- src/codecs/encoding/transformer.rs | 2 +- src/conditions/vrl.rs | 2 +- src/config/enterprise.rs | 6 ++-- src/enrichment_tables/file.rs | 2 +- src/enrichment_tables/geoip.rs | 2 +- src/sinks/aws_cloudwatch_logs/config.rs | 2 +- src/sinks/azure_monitor_logs.rs | 2 +- src/sinks/datadog/events/config.rs | 2 +- src/sinks/datadog/logs/config.rs | 2 +- src/sinks/datadog_archives.rs | 2 +- src/sinks/elasticsearch/config.rs | 2 +- src/sinks/gcp/chronicle_unstructured.rs | 2 +- src/sinks/gcp/stackdriver_logs.rs | 2 +- src/sinks/honeycomb.rs | 2 +- src/sinks/influxdb/logs.rs | 2 +- src/sinks/kafka/config.rs | 2 +- src/sinks/loki/config.rs | 2 +- src/sinks/loki/integration_tests.rs | 2 +- src/sinks/mezmo.rs | 4 +-- src/sinks/papertrail.rs | 2 +- src/sinks/pulsar/config.rs | 2 +- src/sinks/pulsar/tests.rs | 2 +- src/sinks/pulsar/util.rs | 2 +- src/sinks/util/encoding.rs | 2 +- src/source_sender/mod.rs | 2 +- src/sources/amqp.rs | 4 +-- src/sources/aws_kinesis_firehose/mod.rs | 2 +- src/sources/aws_s3/mod.rs | 4 +-- src/sources/aws_sqs/config.rs | 2 +- src/sources/datadog_agent/mod.rs | 2 +- src/sources/datadog_agent/tests.rs | 2 +- src/sources/dnstap/mod.rs | 2 +- src/sources/dnstap/schema.rs | 4 +-- src/sources/docker_logs/mod.rs | 2 +- src/sources/exec/mod.rs | 2 +- src/sources/file.rs | 2 +- src/sources/file_descriptors/mod.rs | 2 +- src/sources/fluent/mod.rs | 6 ++-- src/sources/gcp_pubsub.rs | 4 +-- src/sources/heroku_logs.rs | 4 +-- src/sources/http_server.rs | 6 ++-- src/sources/internal_logs.rs | 4 +-- src/sources/journald.rs | 4 +-- src/sources/kafka.rs | 2 +- src/sources/kubernetes_logs/mod.rs | 4 +-- src/sources/logstash.rs | 6 ++-- src/sources/nats.rs | 4 +-- src/sources/opentelemetry/mod.rs | 2 +- src/sources/redis/mod.rs | 2 +- src/sources/socket/mod.rs | 2 +- src/sources/splunk_hec/mod.rs | 2 +- src/sources/syslog.rs | 2 +- src/sources/util/net/tcp/mod.rs | 3 +- src/sources/vector/mod.rs | 2 +- src/test_util/mock/transforms/basic.rs | 2 +- .../mock/transforms/error_definitions.rs | 2 +- src/topology/schema.rs | 2 +- src/trace.rs | 2 +- src/transforms/aws_ec2_metadata.rs | 4 +-- src/transforms/metric_to_log.rs | 6 ++-- src/transforms/reduce/mod.rs | 6 ++-- src/transforms/remap.rs | 16 ++++----- 68 files changed, 117 insertions(+), 114 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f315ea244b992..a7a45cc40ef0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9056,6 +9056,7 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" name = "value" version = "0.1.0" dependencies = [ + "async-graphql", "bytes 1.4.0", "chrono", "mlua", @@ -9067,6 +9068,7 @@ dependencies = [ "serde", "serde_json", "snafu", + "toml 0.7.3", "tracing 0.1.37", ] @@ -9306,7 +9308,6 @@ dependencies = [ "vector-lookup", "vector-vrl-functions", "vrl", - "vrl-cli", "vrl-stdlib", "warp", "windows-service", @@ -9540,6 +9541,7 @@ dependencies = [ "twox-hash", "typetag", "url", + "value", "vector-buffers", "vector-common", "vector-config", diff --git a/Cargo.toml b/Cargo.toml index e399636b973bc..ff3371a11611b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,7 +126,6 @@ portpicker = { path = "lib/portpicker" } prometheus-parser = { path = "lib/prometheus-parser", optional = true } opentelemetry-proto = { path = "lib/opentelemetry-proto", optional = true } tracing-limit = { path = "lib/tracing-limit" } -value = { package = "value", path = "/home/nathan/repos/vrl/lib/value" } vector-api-client = { path = "lib/vector-api-client", optional = true } vector-buffers = { path = "lib/vector-buffers", default-features = false } vector-common = { path = "lib/vector-common" } @@ -135,7 +134,6 @@ vector-config-common = { path = "lib/vector-config-common" } vector-config-macros = { path = "lib/vector-config-macros" } vector-core = { path = "lib/vector-core", default-features = false, features = ["vrl"] } vector-vrl-functions = { path = "lib/vector-vrl/functions" } -vrl-cli = { package = "vrl-cli", path = "/home/nathan/repos/vrl/lib/cli", optional = true } loki-logproto = { path = "lib/loki-logproto", optional = true } # Tokio / Futures @@ -223,14 +221,15 @@ number_prefix = { version = "0.4.0", default-features = false, features = ["std" tui = { version = "0.19.0", optional = true, default-features = false, features = ["crossterm"] } # Datadog Pipelines -datadog-filter = { package = "datadog-filter", path = "/home/nathan/repos/vrl/lib/datadog/filter" } -datadog-search-syntax = { package = "datadog-search-syntax", path = "/home/nathan/repos/vrl/lib/datadog/search-syntax" } + hex = { version = "0.4.3", default-features = false, optional = true } sha2 = { version = "0.10.6", default-features = false, optional = true } # VRL Lang vrl = { package = "vrl", path = "/home/nathan/repos/vrl" } vrl-stdlib = { package = "vrl-stdlib", path = "/home/nathan/repos/vrl/lib/stdlib" } +datadog-filter = { package = "datadog-filter", path = "/home/nathan/repos/vrl/lib/datadog/filter" } +datadog-search-syntax = { package = "datadog-search-syntax", path = "/home/nathan/repos/vrl/lib/datadog/search-syntax" } # External libs arc-swap = { version = "1.6", default-features = false, optional = true } @@ -378,16 +377,16 @@ ntapi = { git = "https://github.com/MSxDOS/ntapi.git", rev = "24fc1e47677fc9f6e3 [features] # Default features for *-unknown-linux-gnu and *-apple-darwin -default = ["api", "api-client", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "vrl-cli", "enterprise", "component-validation-runner"] +default = ["api", "api-client", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "enterprise", "component-validation-runner"] # Default features for `cargo docs`. The same as `default` but without `rdkafka?/gssapi-vendored` which would require installing libsasl in our doc build environment. -docs = ["api", "api-client", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "vrl-cli", "enterprise", "component-validation-runner"] +docs = ["api", "api-client", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "enterprise", "component-validation-runner"] # Default features for *-unknown-linux-* which make use of `cmake` for dependencies -default-cmake = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "vrl-cli", "enterprise"] +default-cmake = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "enterprise"] # Default features for *-pc-windows-msvc # TODO: Enable SASL https://github.com/vectordotdev/vector/pull/3081#issuecomment-659298042 -default-msvc = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "transforms", "vrl-cli", "enterprise"] -default-musl = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "vrl-cli", "enterprise"] -default-no-api-client = ["api", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "vrl-cli", "enterprise"] +default-msvc = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "transforms", "enterprise"] +default-musl = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "enterprise"] +default-no-api-client = ["api", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "enterprise"] default-no-vrl-cli = ["api", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "enterprise"] tokio-console = ["dep:console-subscriber", "tokio/tracing"] @@ -397,16 +396,16 @@ all-metrics = ["sinks-metrics", "sources-metrics", "transforms-metrics", "enterp # Target specific release features. # The `make` tasks will select this according to the appropriate triple. # Use this section to turn off or on specific features for specific triples. -target-aarch64-unknown-linux-gnu = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "vrl-cli", "enterprise"] -target-aarch64-unknown-linux-musl = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "vrl-cli", "enterprise"] -target-armv7-unknown-linux-gnueabihf = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "vrl-cli", "enterprise"] -target-armv7-unknown-linux-musleabihf = ["api", "api-client", "rdkafka?/cmake_build", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "vrl-cli", "enterprise"] -target-x86_64-unknown-linux-gnu = ["api", "api-client", "rdkafka?/cmake_build", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "vrl-cli", "enterprise"] -target-x86_64-unknown-linux-musl = ["api", "api-client", "rdkafka?/cmake_build", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "vrl-cli", "enterprise"] +target-aarch64-unknown-linux-gnu = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "enterprise"] +target-aarch64-unknown-linux-musl = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "enterprise"] +target-armv7-unknown-linux-gnueabihf = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "enterprise"] +target-armv7-unknown-linux-musleabihf = ["api", "api-client", "rdkafka?/cmake_build", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "enterprise"] +target-x86_64-unknown-linux-gnu = ["api", "api-client", "rdkafka?/cmake_build", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "enterprise"] +target-x86_64-unknown-linux-musl = ["api", "api-client", "rdkafka?/cmake_build", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "enterprise"] # Does not currently build -target-powerpc64le-unknown-linux-gnu = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "vrl-cli", "enterprise"] +target-powerpc64le-unknown-linux-gnu = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "enterprise"] # Currently doesn't build due to lack of support for 64-bit atomics -target-powerpc-unknown-linux-gnu = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "vrl-cli", "enterprise"] +target-powerpc-unknown-linux-gnu = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "enterprise"] # Enables features that work only on systems providing `cfg(unix)` unix = ["tikv-jemallocator", "allocation-tracing"] diff --git a/benches/enrichment_tables.rs b/benches/enrichment_tables.rs index 35ba9759f9b96..39be8eb92b83e 100644 --- a/benches/enrichment_tables.rs +++ b/benches/enrichment_tables.rs @@ -3,12 +3,12 @@ use std::{collections::BTreeMap, time::SystemTime}; use chrono::prelude::*; use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; use enrichment::Case; -use value::Value; use vector::enrichment_tables::{ file::File, geoip::{Geoip, GeoipConfig}, Condition, Table, }; +use vrl::value::Value; criterion_group!( name = benches; diff --git a/benches/transform/route.rs b/benches/transform/route.rs index f1ed85f163634..9e732aadcef83 100644 --- a/benches/transform/route.rs +++ b/benches/transform/route.rs @@ -6,7 +6,6 @@ use criterion::{ black_box, criterion_group, measurement::WallTime, BatchSize, BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, Throughput, }; -use value::Value; use vector::transforms::{ route::{Route, RouteConfig}, TransformOutputsBuf, @@ -16,6 +15,7 @@ use vector_core::{ event::{Event, EventContainer, EventMetadata, LogEvent}, transform::{SyncTransform, TransformContext}, }; +use vrl::value::Value; #[derive(Debug)] struct Param { diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index e2ec3093a78aa..4165ab14face6 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -95,6 +95,7 @@ rand_distr = "0.4.3" tracing-subscriber = { version = "0.3.16", default-features = false, features = ["env-filter", "fmt", "ansi", "registry"] } vector-common = { path = "../vector-common", default-features = false, features = ["test"] } vrl = { path = "/home/nathan/repos/vrl", default-features = false, features = ["value", "arbitrary", "lua"] } +value = { package = "value", path = "/home/nathan/repos/vrl/lib/value", default-features = false, features = ["lua", "toml", "json", "api", "arbitrary", "test"] } [features] api = ["dep:async-graphql"] diff --git a/src/codecs/decoding/decoder.rs b/src/codecs/decoding/decoder.rs index 76b06f5a59f70..6f212f450dc83 100644 --- a/src/codecs/decoding/decoder.rs +++ b/src/codecs/decoding/decoder.rs @@ -109,7 +109,7 @@ mod tests { }; use futures::{stream, StreamExt}; use tokio_util::{codec::FramedRead, io::StreamReader}; - use value::Value; + use vrl::value::Value; #[tokio::test] async fn framed_read_recover_from_error() { diff --git a/src/codecs/encoding/transformer.rs b/src/codecs/encoding/transformer.rs index f10bb361df1e2..79137020e6a10 100644 --- a/src/codecs/encoding/transformer.rs +++ b/src/codecs/encoding/transformer.rs @@ -10,9 +10,9 @@ use lookup::{ PathPrefix, }; use serde::{Deserialize, Deserializer}; -use value::Value; use vector_config::configurable_component; use vector_core::event::{LogEvent, MaybeAsLogMut}; +use vrl::value::Value; use crate::{event::Event, serde::skip_serializing_if_default}; diff --git a/src/conditions/vrl.rs b/src/conditions/vrl.rs index 4d44478ee1c4c..9791b84c39524 100644 --- a/src/conditions/vrl.rs +++ b/src/conditions/vrl.rs @@ -1,10 +1,10 @@ -use value::Value; use vector_common::TimeZone; use vector_config::configurable_component; use vector_core::compile_vrl; use vrl::compiler::runtime::{Runtime, RuntimeResult, Terminate}; use vrl::compiler::{CompilationResult, CompileConfig, Program, TypeState, VrlRuntime}; use vrl::diagnostic::Formatter; +use vrl::value::Value; use crate::event::TargetEvents; use crate::{ diff --git a/src/config/enterprise.rs b/src/config/enterprise.rs index e9f7e24224c66..6f9722b5b8fd5 100644 --- a/src/config/enterprise.rs +++ b/src/config/enterprise.rs @@ -832,12 +832,12 @@ mod test { use http::StatusCode; use indexmap::IndexMap; use tokio::time::sleep; - use value::btreemap; - use value::kind::Collection; - use value::Kind; use vector_core::config::proxy::ProxyConfig; use vrl::compiler::state::ExternalEnv; use vrl::compiler::{compile, compile_with_external, CompileConfig}; + use vrl::value::btreemap; + use vrl::value::kind::Collection; + use vrl::value::Kind; use wiremock::{matchers, Mock, MockServer, ResponseTemplate}; use super::{ diff --git a/src/enrichment_tables/file.rs b/src/enrichment_tables/file.rs index 3b8a4770f546f..6c92bf395f244 100644 --- a/src/enrichment_tables/file.rs +++ b/src/enrichment_tables/file.rs @@ -10,9 +10,9 @@ use std::{ use bytes::Bytes; use enrichment::{Case, Condition, IndexHandle, Table}; use tracing::trace; -use value::Value; use vector_common::{conversion::Conversion, TimeZone}; use vector_config::configurable_component; +use vrl::value::Value; use crate::config::EnrichmentTableConfig; diff --git a/src/enrichment_tables/geoip.rs b/src/enrichment_tables/geoip.rs index 4f3b608160ed8..569fc124600af 100644 --- a/src/enrichment_tables/geoip.rs +++ b/src/enrichment_tables/geoip.rs @@ -11,8 +11,8 @@ use maxminddb::{ geoip2::{City, ConnectionType, Isp}, MaxMindDBError, Reader, }; -use value::Value; use vector_config::configurable_component; +use vrl::value::Value; use crate::config::{EnrichmentTableConfig, GenerateConfig}; diff --git a/src/sinks/aws_cloudwatch_logs/config.rs b/src/sinks/aws_cloudwatch_logs/config.rs index 132365829bc8c..2197d5b96746c 100644 --- a/src/sinks/aws_cloudwatch_logs/config.rs +++ b/src/sinks/aws_cloudwatch_logs/config.rs @@ -3,9 +3,9 @@ use aws_smithy_types::retry::RetryConfig; use codecs::JsonSerializerConfig; use futures::FutureExt; use tower::ServiceBuilder; -use value::Kind; use vector_config::configurable_component; use vector_core::schema; +use vrl::value::Kind; use crate::{ aws::{ diff --git a/src/sinks/azure_monitor_logs.rs b/src/sinks/azure_monitor_logs.rs index 2f99077881b7f..bc6bc1b43a330 100644 --- a/src/sinks/azure_monitor_logs.rs +++ b/src/sinks/azure_monitor_logs.rs @@ -13,10 +13,10 @@ use openssl::{base64, hash, pkey, sign}; use regex::Regex; use serde::{Deserialize, Serialize}; use serde_json::Value as JsonValue; -use value::Kind; use vector_common::sensitive_string::SensitiveString; use vector_config::configurable_component; use vector_core::schema; +use vrl::value::Kind; use crate::{ codecs::Transformer, diff --git a/src/sinks/datadog/events/config.rs b/src/sinks/datadog/events/config.rs index 70b83f96411f7..8ca941d1cf0cb 100644 --- a/src/sinks/datadog/events/config.rs +++ b/src/sinks/datadog/events/config.rs @@ -1,9 +1,9 @@ use indoc::indoc; use tower::ServiceBuilder; -use value::Kind; use vector_config::configurable_component; use vector_core::config::proxy::ProxyConfig; use vector_core::schema; +use vrl::value::Kind; use crate::{ common::datadog::{get_base_domain_region, Region}, diff --git a/src/sinks/datadog/logs/config.rs b/src/sinks/datadog/logs/config.rs index 23c25bace98a6..5e1d6cc150962 100644 --- a/src/sinks/datadog/logs/config.rs +++ b/src/sinks/datadog/logs/config.rs @@ -2,9 +2,9 @@ use std::{convert::TryFrom, sync::Arc}; use indoc::indoc; use tower::ServiceBuilder; -use value::Kind; use vector_config::configurable_component; use vector_core::config::proxy::ProxyConfig; +use vrl::value::Kind; use super::{service::LogApiRetry, sink::LogSinkBuilder}; use crate::{ diff --git a/src/sinks/datadog_archives.rs b/src/sinks/datadog_archives.rs index c8fc0b1bce155..6be169c12a62a 100644 --- a/src/sinks/datadog_archives.rs +++ b/src/sinks/datadog_archives.rs @@ -26,7 +26,6 @@ use rand::{thread_rng, Rng}; use snafu::Snafu; use tower::ServiceBuilder; use uuid::Uuid; -use value::Kind; use vector_common::request_metadata::RequestMetadata; use vector_config::{configurable_component, NamedComponent}; use vector_core::{ @@ -34,6 +33,7 @@ use vector_core::{ event::{Event, EventFinalizers, Finalizable}, schema, ByteSizeOf, }; +use vrl::value::Kind; use crate::{ aws::{AwsAuthentication, RegionOrEndpoint}, diff --git a/src/sinks/elasticsearch/config.rs b/src/sinks/elasticsearch/config.rs index fcf861c87419a..fbfc88edcc3c8 100644 --- a/src/sinks/elasticsearch/config.rs +++ b/src/sinks/elasticsearch/config.rs @@ -33,8 +33,8 @@ use crate::{ transforms::metric_to_log::MetricToLogConfig, }; use lookup::event_path; -use value::Kind; use vector_core::schema::Requirement; +use vrl::value::Kind; /// The field name for the timestamp required by data stream mode pub const DATA_STREAM_TIMESTAMP_KEY: &str = "@timestamp"; diff --git a/src/sinks/gcp/chronicle_unstructured.rs b/src/sinks/gcp/chronicle_unstructured.rs index 6a6aaefb275d0..a9390b4335d37 100644 --- a/src/sinks/gcp/chronicle_unstructured.rs +++ b/src/sinks/gcp/chronicle_unstructured.rs @@ -12,7 +12,6 @@ use snafu::Snafu; use std::io; use tokio_util::codec::Encoder as _; use tower::{Service, ServiceBuilder}; -use value::Kind; use vector_common::request_metadata::{MetaDescriptive, RequestMetadata}; use vector_config::configurable_component; use vector_core::{ @@ -20,6 +19,7 @@ use vector_core::{ event::{Event, EventFinalizers, Finalizable}, sink::VectorSink, }; +use vrl::value::Kind; use crate::{ codecs::{self, EncodingConfig}, diff --git a/src/sinks/gcp/stackdriver_logs.rs b/src/sinks/gcp/stackdriver_logs.rs index 4752d84786289..f313bb1a38c5e 100644 --- a/src/sinks/gcp/stackdriver_logs.rs +++ b/src/sinks/gcp/stackdriver_logs.rs @@ -6,8 +6,8 @@ use http::{Request, Uri}; use hyper::Body; use serde_json::{json, map}; use snafu::Snafu; -use value::Kind; use vector_config::configurable_component; +use vrl::value::Kind; use crate::{ codecs::Transformer, diff --git a/src/sinks/honeycomb.rs b/src/sinks/honeycomb.rs index 5119ab4e5272f..c69b52515ec9e 100644 --- a/src/sinks/honeycomb.rs +++ b/src/sinks/honeycomb.rs @@ -2,9 +2,9 @@ use bytes::Bytes; use futures::{FutureExt, SinkExt}; use http::{Request, StatusCode, Uri}; use serde_json::json; -use value::Kind; use vector_common::sensitive_string::SensitiveString; use vector_config::configurable_component; +use vrl::value::Kind; use crate::{ codecs::Transformer, diff --git a/src/sinks/influxdb/logs.rs b/src/sinks/influxdb/logs.rs index 08ef59618b6f7..80781c34f863c 100644 --- a/src/sinks/influxdb/logs.rs +++ b/src/sinks/influxdb/logs.rs @@ -6,10 +6,10 @@ use http::{Request, Uri}; use indoc::indoc; use lookup::lookup_v2::{parse_value_path, OptionalValuePath}; use lookup::{OwnedValuePath, PathPrefix}; -use value::Kind; use vector_config::configurable_component; use vector_core::config::log_schema; use vector_core::schema; +use vrl::value::Kind; use crate::{ codecs::Transformer, diff --git a/src/sinks/kafka/config.rs b/src/sinks/kafka/config.rs index 6782b185badf5..4bccf58a7806c 100644 --- a/src/sinks/kafka/config.rs +++ b/src/sinks/kafka/config.rs @@ -4,9 +4,9 @@ use codecs::JsonSerializerConfig; use futures::FutureExt; use rdkafka::ClientConfig; use serde_with::serde_as; -use value::Kind; use vector_config::configurable_component; use vector_core::schema::Requirement; +use vrl::value::Kind; use crate::{ codecs::EncodingConfig, diff --git a/src/sinks/loki/config.rs b/src/sinks/loki/config.rs index 242900ca3f759..108cfa0aa3f99 100644 --- a/src/sinks/loki/config.rs +++ b/src/sinks/loki/config.rs @@ -1,8 +1,8 @@ use std::collections::HashMap; use futures::future::FutureExt; -use value::Kind; use vector_config::configurable_component; +use vrl::value::Kind; use super::{healthcheck::healthcheck, sink::LokiSink}; use crate::{ diff --git a/src/sinks/loki/integration_tests.rs b/src/sinks/loki/integration_tests.rs index f37df44aff3c0..245c8ebcb2c7b 100644 --- a/src/sinks/loki/integration_tests.rs +++ b/src/sinks/loki/integration_tests.rs @@ -5,12 +5,12 @@ use bytes::Bytes; use chrono::{DateTime, Duration, Utc}; use futures::stream; use lookup::owned_value_path; -use value::{kind::Collection, Kind}; use vector_common::encode_logfmt; use vector_core::{ config::LogNamespace, event::{BatchNotifier, BatchStatus, Event, LogEvent}, }; +use vrl::value::{kind::Collection, Kind}; use super::config::{LokiConfig, OutOfOrderAction}; use crate::{ diff --git a/src/sinks/mezmo.rs b/src/sinks/mezmo.rs index aed7049244f26..d7c055e701143 100644 --- a/src/sinks/mezmo.rs +++ b/src/sinks/mezmo.rs @@ -4,9 +4,9 @@ use bytes::Bytes; use futures::{FutureExt, SinkExt}; use http::{Request, StatusCode, Uri}; use serde_json::json; -use value::Kind; use vector_common::sensitive_string::SensitiveString; use vector_config::configurable_component; +use vrl::value::{Kind, Value}; use crate::{ codecs::Transformer, @@ -251,7 +251,7 @@ impl HttpEventEncoder> for .and_then(|path| log.remove(path.as_str())) .unwrap_or_else(|| String::from("").into()); - let timestamp: value::Value = log + let timestamp: Value = log .timestamp_path() .and_then(|path| log.remove(path.as_str())) .unwrap_or_else(|| chrono::Utc::now().into()); diff --git a/src/sinks/papertrail.rs b/src/sinks/papertrail.rs index 04897c890be3a..7dbd82459e422 100644 --- a/src/sinks/papertrail.rs +++ b/src/sinks/papertrail.rs @@ -1,7 +1,7 @@ use bytes::{BufMut, BytesMut}; use syslog::{Facility, Formatter3164, LogFormat, Severity}; -use value::Kind; use vector_config::configurable_component; +use vrl::value::Kind; use crate::{ codecs::{Encoder, EncodingConfig, Transformer}, diff --git a/src/sinks/pulsar/config.rs b/src/sinks/pulsar/config.rs index af3ad4c07b644..eba7eba4694d8 100644 --- a/src/sinks/pulsar/config.rs +++ b/src/sinks/pulsar/config.rs @@ -20,10 +20,10 @@ use pulsar::{ }; use pulsar::{error::AuthenticationError, OperationRetryOptions}; use snafu::ResultExt; -use value::Kind; use vector_common::sensitive_string::SensitiveString; use vector_config::configurable_component; use vector_core::config::DataType; +use vrl::value::Kind; /// Configuration for the `pulsar` sink. #[configurable_component(sink("pulsar"))] diff --git a/src/sinks/pulsar/tests.rs b/src/sinks/pulsar/tests.rs index 3464f194c89bb..5dae4bb67846c 100644 --- a/src/sinks/pulsar/tests.rs +++ b/src/sinks/pulsar/tests.rs @@ -2,8 +2,8 @@ use crate::event::Event; use crate::sinks::pulsar::config::PulsarSinkConfig; use lookup::lookup_v2::OptionalTargetPath; use std::collections::BTreeMap; -use value::Value; use vector_config::component::GenerateConfig; +use vrl::value::Value; use bytes::Bytes; diff --git a/src/sinks/pulsar/util.rs b/src/sinks/pulsar/util.rs index f658bd8e394dc..b14500252c8d4 100644 --- a/src/sinks/pulsar/util.rs +++ b/src/sinks/pulsar/util.rs @@ -5,8 +5,8 @@ use crate::template::Template; use bytes::Bytes; use lookup::lookup_v2::OptionalTargetPath; use std::collections::HashMap; -use value::Value; use vector_core::event::Event; +use vrl::value::Value; /// Transforms an event into a Pulsar event by rendering the required template fields. /// Returns None if there is an error whilst rendering. diff --git a/src/sinks/util/encoding.rs b/src/sinks/util/encoding.rs index daa2d8ad84f4b..c2705f5deff5a 100644 --- a/src/sinks/util/encoding.rs +++ b/src/sinks/util/encoding.rs @@ -128,8 +128,8 @@ mod tests { CharacterDelimitedEncoder, JsonSerializerConfig, NewlineDelimitedEncoder, TextSerializerConfig, }; - use value::Value; use vector_core::event::LogEvent; + use vrl::value::Value; use super::*; diff --git a/src/source_sender/mod.rs b/src/source_sender/mod.rs index 30ee2cf2b781c..a4f4eaae3b751 100644 --- a/src/source_sender/mod.rs +++ b/src/source_sender/mod.rs @@ -4,7 +4,6 @@ use std::{collections::HashMap, fmt}; use chrono::Utc; use futures::{Stream, StreamExt}; use metrics::{register_histogram, Histogram}; -use value::Value; use vector_buffers::topology::channel::{self, LimitedReceiver, LimitedSender}; #[cfg(test)] use vector_core::event::{into_event_stream, EventStatus}; @@ -16,6 +15,7 @@ use vector_core::{ }, ByteSizeOf, EstimatedJsonEncodedSizeOf, }; +use vrl::value::Value; mod errors; diff --git a/src/sources/amqp.rs b/src/sources/amqp.rs index 157742b8733e2..94fc250806cf5 100644 --- a/src/sources/amqp.rs +++ b/src/sources/amqp.rs @@ -24,7 +24,6 @@ use lookup::{lookup_v2::OptionalValuePath, metadata_path, owned_value_path, path use snafu::Snafu; use std::{io::Cursor, pin::Pin}; use tokio_util::codec::FramedRead; -use value::Kind; use vector_common::{ finalizer::UnorderedFinalizer, internal_event::{CountByteSize, EventsReceived, InternalEventHandle as _}, @@ -35,6 +34,7 @@ use vector_core::{ event::Event, EstimatedJsonEncodedSizeOf, }; +use vrl::value::Kind; #[derive(Debug, Snafu)] enum BuildError { @@ -495,8 +495,8 @@ async fn handle_ack(status: BatchStatus, entry: FinalizerEntry) { #[cfg(test)] pub mod test { use lookup::OwnedTargetPath; - use value::kind::Collection; use vector_core::schema::Definition; + use vrl::value::kind::Collection; use super::*; diff --git a/src/sources/aws_kinesis_firehose/mod.rs b/src/sources/aws_kinesis_firehose/mod.rs index 0c55614d4acac..cefb74beafc07 100644 --- a/src/sources/aws_kinesis_firehose/mod.rs +++ b/src/sources/aws_kinesis_firehose/mod.rs @@ -4,10 +4,10 @@ use codecs::decoding::{DeserializerConfig, FramingConfig}; use futures::FutureExt; use lookup::owned_value_path; use tracing::Span; -use value::Kind; use vector_common::sensitive_string::SensitiveString; use vector_config::configurable_component; use vector_core::config::{LegacyKey, LogNamespace}; +use vrl::value::Kind; use warp::Filter; use crate::{ diff --git a/src/sources/aws_s3/mod.rs b/src/sources/aws_s3/mod.rs index c0cac5559a673..49a1378c25991 100644 --- a/src/sources/aws_s3/mod.rs +++ b/src/sources/aws_s3/mod.rs @@ -8,9 +8,9 @@ use futures::{stream, stream::StreamExt, TryStreamExt}; use lookup::owned_value_path; use snafu::Snafu; use tokio_util::io::StreamReader; -use value::{kind::Collection, Kind}; use vector_config::configurable_component; use vector_core::config::{LegacyKey, LogNamespace}; +use vrl::value::{kind::Collection, Kind}; use super::util::MultilineConfig; use crate::codecs::DecodingConfig; @@ -450,7 +450,7 @@ mod integration_tests { use aws_sdk_sqs::{model::QueueAttributeName, Client as SqsClient}; use lookup::path; use similar_asserts::assert_eq; - use value::Value; + use vrl::value::Value; use super::{sqs, AwsS3Config, Compression, Strategy}; use crate::{ diff --git a/src/sources/aws_sqs/config.rs b/src/sources/aws_sqs/config.rs index b165bc16bdee2..733eab84c6c0c 100644 --- a/src/sources/aws_sqs/config.rs +++ b/src/sources/aws_sqs/config.rs @@ -2,9 +2,9 @@ use std::num::NonZeroUsize; use codecs::decoding::{DeserializerConfig, FramingConfig}; use lookup::owned_value_path; -use value::Kind; use vector_config::configurable_component; use vector_core::config::{LegacyKey, LogNamespace}; +use vrl::value::Kind; use crate::aws::create_client; use crate::codecs::DecodingConfig; diff --git a/src/sources/datadog_agent/mod.rs b/src/sources/datadog_agent/mod.rs index c92d8856e87b1..32afa640ff05e 100644 --- a/src/sources/datadog_agent/mod.rs +++ b/src/sources/datadog_agent/mod.rs @@ -30,11 +30,11 @@ use regex::Regex; use serde::{Deserialize, Serialize}; use snafu::Snafu; use tracing::Span; -use value::Kind; use vector_common::internal_event::{EventsReceived, Registered}; use vector_config::configurable_component; use vector_core::config::{LegacyKey, LogNamespace}; use vector_core::event::{BatchNotifier, BatchStatus}; +use vrl::value::Kind; use warp::{filters::BoxedFilter, reject::Rejection, reply::Response, Filter, Reply}; use crate::{ diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index 882a8f1ee3189..aa725223a8403 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -19,13 +19,13 @@ use ordered_float::NotNan; use prost::Message; use quickcheck::{Arbitrary, Gen, QuickCheck, TestResult}; use similar_asserts::assert_eq; -use value::Kind; use vector_core::{ config::LogNamespace, event::{metric::TagValue, MetricTags}, metric_tags, }; use vrl::compiler::value::Collection; +use vrl::value::Kind; use crate::schema::Definition; use crate::{ diff --git a/src/sources/dnstap/mod.rs b/src/sources/dnstap/mod.rs index cb61a2876d76b..8686cfdf8203a 100644 --- a/src/sources/dnstap/mod.rs +++ b/src/sources/dnstap/mod.rs @@ -3,11 +3,11 @@ use std::path::PathBuf; use base64::prelude::{Engine as _, BASE64_STANDARD}; use bytes::Bytes; use lookup::{owned_value_path, path, OwnedValuePath}; -use value::{kind::Collection, Kind}; use vector_common::internal_event::{ ByteSize, BytesReceived, InternalEventHandle as _, Protocol, Registered, }; use vector_config::configurable_component; +use vrl::value::{kind::Collection, Kind}; use super::util::framestream::{build_framestream_unix_source, FrameHandler}; use crate::{ diff --git a/src/sources/dnstap/schema.rs b/src/sources/dnstap/schema.rs index a928c36de1057..dcf4a60935a9b 100644 --- a/src/sources/dnstap/schema.rs +++ b/src/sources/dnstap/schema.rs @@ -1,7 +1,7 @@ use lookup::{owned_value_path, OwnedValuePath}; use std::collections::BTreeMap; -use value::btreemap; -use value::{ +use vrl::value::btreemap; +use vrl::value::{ kind::{Collection, Field}, Kind, }; diff --git a/src/sources/docker_logs/mod.rs b/src/sources/docker_logs/mod.rs index b3a1875246577..55c24e77a3c90 100644 --- a/src/sources/docker_logs/mod.rs +++ b/src/sources/docker_logs/mod.rs @@ -21,12 +21,12 @@ use once_cell::sync::Lazy; use serde_with::serde_as; use tokio::sync::mpsc; use tracing_futures::Instrument; -use value::{kind::Collection, Kind}; use vector_common::internal_event::{ ByteSize, BytesReceived, InternalEventHandle as _, Protocol, Registered, }; use vector_config::configurable_component; use vector_core::config::{LegacyKey, LogNamespace}; +use vrl::value::{kind::Collection, Kind}; use super::util::MultilineConfig; use crate::{ diff --git a/src/sources/exec/mod.rs b/src/sources/exec/mod.rs index bbd6022ed6c49..c540c8e97ae15 100644 --- a/src/sources/exec/mod.rs +++ b/src/sources/exec/mod.rs @@ -20,10 +20,10 @@ use tokio::{ }; use tokio_stream::wrappers::IntervalStream; use tokio_util::codec::FramedRead; -use value::Kind; use vector_common::internal_event::{ByteSize, BytesReceived, InternalEventHandle as _, Protocol}; use vector_config::configurable_component; use vector_core::{config::LegacyKey, EstimatedJsonEncodedSizeOf}; +use vrl::value::Kind; use crate::{ codecs::{Decoder, DecodingConfig}, diff --git a/src/sources/file.rs b/src/sources/file.rs index 4cad2b3c0bd08..8c31338b30175 100644 --- a/src/sources/file.rs +++ b/src/sources/file.rs @@ -16,10 +16,10 @@ use serde_with::serde_as; use snafu::{ResultExt, Snafu}; use tokio::{sync::oneshot, task::spawn_blocking}; use tracing::{Instrument, Span}; -use value::Kind; use vector_common::finalizer::OrderedFinalizer; use vector_config::configurable_component; use vector_core::config::{LegacyKey, LogNamespace}; +use vrl::value::Kind; use super::util::{EncodingConfig, MultilineConfig}; use crate::{ diff --git a/src/sources/file_descriptors/mod.rs b/src/sources/file_descriptors/mod.rs index 68c8df41f60db..32acfbede5e61 100644 --- a/src/sources/file_descriptors/mod.rs +++ b/src/sources/file_descriptors/mod.rs @@ -13,7 +13,6 @@ use lookup::{ owned_value_path, path, OwnedValuePath, }; use tokio_util::{codec::FramedRead, io::StreamReader}; -use value::Kind; use vector_common::internal_event::{ ByteSize, BytesReceived, CountByteSize, InternalEventHandle as _, Protocol, }; @@ -23,6 +22,7 @@ use vector_core::{ event::Event, EstimatedJsonEncodedSizeOf, }; +use vrl::value::Kind; use crate::{ codecs::{Decoder, DecodingConfig}, diff --git a/src/sources/fluent/mod.rs b/src/sources/fluent/mod.rs index eadf8abd556e3..76e51ae7a066d 100644 --- a/src/sources/fluent/mod.rs +++ b/src/sources/fluent/mod.rs @@ -13,11 +13,11 @@ use rmp_serde::{decode, Deserializer}; use serde::Deserialize; use smallvec::{smallvec, SmallVec}; use tokio_util::codec::Decoder; -use value::kind::Collection; -use value::{Kind, Value}; use vector_config::configurable_component; use vector_core::config::{LegacyKey, LogNamespace}; use vector_core::schema::Definition; +use vrl::value::kind::Collection; +use vrl::value::{Kind, Value}; use super::util::net::{SocketListenAddr, TcpSource, TcpSourceAck, TcpSourceAcker}; use crate::{ @@ -635,9 +635,9 @@ mod tests { time::{error::Elapsed, timeout, Duration}, }; use tokio_util::codec::Decoder; - use value::kind::Collection; use vector_common::assert_event_data_eq; use vector_core::{event::Value, schema::Definition}; + use vrl::value::kind::Collection; use super::{message::FluentMessageOptions, *}; use crate::{ diff --git a/src/sources/gcp_pubsub.rs b/src/sources/gcp_pubsub.rs index 3cd5836089fbe..15ae3c5db56a2 100644 --- a/src/sources/gcp_pubsub.rs +++ b/src/sources/gcp_pubsub.rs @@ -20,13 +20,13 @@ use tonic::{ transport::{Certificate, ClientTlsConfig, Endpoint, Identity}, Code, Request, Status, }; -use value::{kind::Collection, Kind}; use vector_common::internal_event::{ ByteSize, BytesReceived, EventsReceived, InternalEventHandle as _, Protocol, Registered, }; use vector_common::{byte_size_of::ByteSizeOf, finalizer::UnorderedFinalizer}; use vector_config::configurable_component; use vector_core::config::{LegacyKey, LogNamespace}; +use vrl::value::{kind::Collection, Kind}; use crate::{ codecs::{Decoder, DecodingConfig}, @@ -839,7 +839,7 @@ mod integration_tests { use once_cell::sync::Lazy; use serde_json::{json, Value}; use tokio::time::{Duration, Instant}; - use value::btreemap; + use vrl::value::btreemap; use super::*; use crate::config::{ComponentKey, ProxyConfig}; diff --git a/src/sources/heroku_logs.rs b/src/sources/heroku_logs.rs index 787b3db9db9c0..a1e3d98df4fe6 100644 --- a/src/sources/heroku_logs.rs +++ b/src/sources/heroku_logs.rs @@ -14,7 +14,7 @@ use codecs::{ use lookup::{lookup_v2::parse_value_path, owned_value_path, path}; use smallvec::SmallVec; use tokio_util::codec::Decoder as _; -use value::{kind::Collection, Kind}; +use vrl::value::{kind::Collection, Kind}; use warp::http::{HeaderMap, StatusCode}; use vector_config::configurable_component; @@ -407,12 +407,12 @@ mod tests { use futures::Stream; use lookup::{owned_value_path, OwnedTargetPath}; use similar_asserts::assert_eq; - use value::{kind::Collection, Kind}; use vector_core::{ config::LogNamespace, event::{Event, EventStatus, Value}, schema::Definition, }; + use vrl::value::{kind::Collection, Kind}; use super::{HttpSourceAuthConfig, LogplexConfig}; use crate::{ diff --git a/src/sources/http_server.rs b/src/sources/http_server.rs index eb0b7a33a41c6..e082f5e1642b8 100644 --- a/src/sources/http_server.rs +++ b/src/sources/http_server.rs @@ -11,12 +11,12 @@ use codecs::{ use http::{StatusCode, Uri}; use lookup::{lookup_v2::OptionalValuePath, owned_value_path, path}; use tokio_util::codec::Decoder as _; -use value::{kind::Collection, Kind}; use vector_config::configurable_component; use vector_core::{ config::{DataType, LegacyKey, LogNamespace}, schema::Definition, }; +use vrl::value::{kind::Collection, Kind}; use warp::http::{HeaderMap, HeaderValue}; use crate::{ @@ -463,11 +463,11 @@ mod tests { use lookup::{event_path, owned_value_path, OwnedTargetPath}; use std::str::FromStr; use std::{collections::BTreeMap, io::Write, net::SocketAddr}; - use value::kind::Collection; - use value::Kind; use vector_core::config::LogNamespace; use vector_core::event::LogEvent; use vector_core::schema::Definition; + use vrl::value::kind::Collection; + use vrl::value::Kind; use codecs::{ decoding::{DeserializerConfig, FramingConfig}, diff --git a/src/sources/internal_logs.rs b/src/sources/internal_logs.rs index bc69a34079328..bc923eef399c2 100644 --- a/src/sources/internal_logs.rs +++ b/src/sources/internal_logs.rs @@ -3,13 +3,13 @@ use codecs::BytesDeserializerConfig; use futures::{stream, StreamExt}; use lookup::lookup_v2::OptionalValuePath; use lookup::{owned_value_path, path, OwnedValuePath}; -use value::Kind; use vector_config::configurable_component; use vector_core::config::log_schema; use vector_core::{ config::{LegacyKey, LogNamespace}, schema::Definition, }; +use vrl::value::Kind; use crate::{ config::{DataType, SourceConfig, SourceContext, SourceOutput}, @@ -204,8 +204,8 @@ mod tests { use futures::Stream; use lookup::OwnedTargetPath; use tokio::time::{sleep, Duration}; - use value::kind::Collection; use vector_core::event::Value; + use vrl::value::kind::Collection; use super::*; use crate::{ diff --git a/src/sources/journald.rs b/src/sources/journald.rs index cc9507e3cc51e..33b734adaf983 100644 --- a/src/sources/journald.rs +++ b/src/sources/journald.rs @@ -28,7 +28,6 @@ use tokio::{ time::sleep, }; use tokio_util::codec::FramedRead; -use value::{kind::Collection, Kind, Value}; use vector_common::{ finalizer::OrderedFinalizer, internal_event::{ @@ -41,6 +40,7 @@ use vector_core::{ schema::Definition, EstimatedJsonEncodedSizeOf, }; +use vrl::value::{kind::Collection, Kind, Value}; use crate::{ config::{ @@ -1048,7 +1048,7 @@ mod tests { use tempfile::tempdir; use tokio::time::{sleep, timeout, Duration, Instant}; - use value::{kind::Collection, Value}; + use vrl::value::{kind::Collection, Value}; use super::*; use crate::{ diff --git a/src/sources/kafka.rs b/src/sources/kafka.rs index 94c561dffd36c..96c789cc66919 100644 --- a/src/sources/kafka.rs +++ b/src/sources/kafka.rs @@ -24,13 +24,13 @@ use serde_with::serde_as; use snafu::{ResultExt, Snafu}; use tokio_util::codec::FramedRead; -use value::{kind::Collection, Kind}; use vector_common::finalizer::OrderedFinalizer; use vector_config::configurable_component; use vector_core::{ config::{LegacyKey, LogNamespace}, EstimatedJsonEncodedSizeOf, }; +use vrl::value::{kind::Collection, Kind}; use crate::{ codecs::{Decoder, DecodingConfig}, diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 42ad81cf82041..0366c51921380 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -30,7 +30,6 @@ use kube::{ use lifecycle::Lifecycle; use lookup::{lookup_v2::OptionalTargetPath, owned_value_path, path, OwnedTargetPath}; use serde_with::serde_as; -use value::{kind::Collection, Kind}; use vector_common::{ internal_event::{ByteSize, BytesReceived, InternalEventHandle as _, Protocol}, TimeZone, @@ -39,6 +38,7 @@ use vector_config::configurable_component; use vector_core::{ config::LegacyKey, config::LogNamespace, transform::TaskTransform, EstimatedJsonEncodedSizeOf, }; +use vrl::value::{kind::Collection, Kind}; use crate::{ config::{ @@ -1037,8 +1037,8 @@ fn prepare_label_selector(selector: &str) -> String { mod tests { use lookup::{owned_value_path, OwnedTargetPath}; use similar_asserts::assert_eq; - use value::{kind::Collection, Kind}; use vector_core::{config::LogNamespace, schema::Definition}; + use vrl::value::{kind::Collection, Kind}; use crate::config::SourceConfig; diff --git a/src/sources/logstash.rs b/src/sources/logstash.rs index c8287a0d01a84..8d58f09a10c42 100644 --- a/src/sources/logstash.rs +++ b/src/sources/logstash.rs @@ -14,13 +14,13 @@ use lookup::{event_path, metadata_path, owned_value_path, path, OwnedValuePath, use smallvec::{smallvec, SmallVec}; use snafu::{ResultExt, Snafu}; use tokio_util::codec::Decoder; -use value::kind::Collection; -use value::Kind; use vector_config::configurable_component; use vector_core::{ config::{LegacyKey, LogNamespace}, schema::Definition, }; +use vrl::value::kind::Collection; +use vrl::value::Kind; use super::util::net::{SocketListenAddr, TcpSource, TcpSourceAck, TcpSourceAcker}; use crate::{ @@ -682,7 +682,7 @@ mod test { use lookup::OwnedTargetPath; use rand::{thread_rng, Rng}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; - use value::kind::Collection; + use vrl::value::kind::Collection; use super::*; use crate::{ diff --git a/src/sources/nats.rs b/src/sources/nats.rs index faef941a4c7f5..e5409836803b3 100644 --- a/src/sources/nats.rs +++ b/src/sources/nats.rs @@ -4,7 +4,6 @@ use futures::{pin_mut, stream, Stream, StreamExt}; use lookup::{lookup_v2::OptionalValuePath, owned_value_path}; use snafu::{ResultExt, Snafu}; use tokio_util::codec::FramedRead; -use value::Kind; use vector_common::internal_event::{ ByteSize, BytesReceived, CountByteSize, EventsReceived, InternalEventHandle as _, Protocol, }; @@ -13,6 +12,7 @@ use vector_core::{ config::{LegacyKey, LogNamespace}, EstimatedJsonEncodedSizeOf, }; +use vrl::value::Kind; use crate::{ codecs::{Decoder, DecodingConfig}, @@ -275,8 +275,8 @@ mod tests { #![allow(clippy::print_stdout)] //tests use lookup::{owned_value_path, OwnedTargetPath}; - use value::{kind::Collection, Kind}; use vector_core::schema::Definition; + use vrl::value::{kind::Collection, Kind}; use super::*; diff --git a/src/sources/opentelemetry/mod.rs b/src/sources/opentelemetry/mod.rs index 275721872bf1d..234144dbd1a84 100644 --- a/src/sources/opentelemetry/mod.rs +++ b/src/sources/opentelemetry/mod.rs @@ -18,13 +18,13 @@ use opentelemetry_proto::convert::{ }; use opentelemetry_proto::proto::collector::logs::v1::logs_service_server::LogsServiceServer; -use value::{kind::Collection, Kind}; use vector_common::internal_event::{BytesReceived, EventsReceived, Protocol}; use vector_config::configurable_component; use vector_core::{ config::{log_schema, LegacyKey, LogNamespace}, schema::Definition, }; +use vrl::value::{kind::Collection, Kind}; use self::{ grpc::Service, diff --git a/src/sources/redis/mod.rs b/src/sources/redis/mod.rs index b087b0f5f9b11..05723b6cc3d4a 100644 --- a/src/sources/redis/mod.rs +++ b/src/sources/redis/mod.rs @@ -8,7 +8,6 @@ use futures::StreamExt; use lookup::{lookup_v2::OptionalValuePath, owned_value_path, path, OwnedValuePath}; use snafu::{ResultExt, Snafu}; use tokio_util::codec::FramedRead; -use value::Kind; use vector_common::internal_event::{ ByteSize, BytesReceived, CountByteSize, InternalEventHandle as _, Protocol, Registered, }; @@ -17,6 +16,7 @@ use vector_core::{ config::{LegacyKey, LogNamespace}, EstimatedJsonEncodedSizeOf, }; +use vrl::value::Kind; use crate::{ codecs::{Decoder, DecodingConfig}, diff --git a/src/sources/socket/mod.rs b/src/sources/socket/mod.rs index 7f7c985b1a67a..e0d212a59f9ee 100644 --- a/src/sources/socket/mod.rs +++ b/src/sources/socket/mod.rs @@ -5,9 +5,9 @@ mod unix; use codecs::decoding::DeserializerConfig; use lookup::{lookup_v2::OptionalValuePath, owned_value_path}; -use value::{kind::Collection, Kind}; use vector_config::configurable_component; use vector_core::config::{log_schema, LegacyKey, LogNamespace}; +use vrl::value::{kind::Collection, Kind}; #[cfg(unix)] use crate::serde::default_framing_message_based; diff --git a/src/sources/splunk_hec/mod.rs b/src/sources/splunk_hec/mod.rs index 8cc9461efdc22..34808fac0c046 100644 --- a/src/sources/splunk_hec/mod.rs +++ b/src/sources/splunk_hec/mod.rs @@ -15,7 +15,6 @@ use serde::Serialize; use serde_json::{de::Read as JsonRead, Deserializer, Value as JsonValue}; use snafu::Snafu; use tracing::Span; -use value::{kind::Collection, Kind}; use vector_common::internal_event::{CountByteSize, InternalEventHandle as _, Registered}; use vector_common::sensitive_string::SensitiveString; use vector_config::configurable_component; @@ -24,6 +23,7 @@ use vector_core::{ event::BatchNotifier, EstimatedJsonEncodedSizeOf, }; +use vrl::value::{kind::Collection, Kind}; use warp::{filters::BoxedFilter, path, reject::Rejection, reply::Response, Filter, Reply}; use self::{ diff --git a/src/sources/syslog.rs b/src/sources/syslog.rs index f6bfe34c5e34b..6bd7fd3c6df7d 100644 --- a/src/sources/syslog.rs +++ b/src/sources/syslog.rs @@ -452,9 +452,9 @@ mod test { use serde::Deserialize; use tokio::time::{sleep, Duration, Instant}; use tokio_util::codec::BytesCodec; - use value::{kind::Collection, Kind, Value}; use vector_common::assert_event_data_eq; use vector_core::{config::ComponentKey, schema::Definition}; + use vrl::value::{kind::Collection, Kind, Value}; use super::*; use crate::{ diff --git a/src/sources/util/net/tcp/mod.rs b/src/sources/util/net/tcp/mod.rs index e260e67872eb3..e3b68e539d8aa 100644 --- a/src/sources/util/net/tcp/mod.rs +++ b/src/sources/util/net/tcp/mod.rs @@ -22,6 +22,7 @@ use vector_core::{ config::{LegacyKey, LogNamespace, SourceAcknowledgementsConfig}, EstimatedJsonEncodedSizeOf, }; +use vrl::value::Value; use self::request_limiter::RequestLimiter; use super::SocketListenAddr; @@ -363,7 +364,7 @@ async fn handle_stream( if let Some(certificate_metadata) = &certificate_metadata { - let mut metadata: BTreeMap = BTreeMap::new(); + let mut metadata: BTreeMap = BTreeMap::new(); metadata.insert("subject".to_string(), certificate_metadata.subject().into()); for event in &mut events { let log = event.as_mut_log(); diff --git a/src/sources/vector/mod.rs b/src/sources/vector/mod.rs index 7f148fff81c47..7cdaa4633a9e9 100644 --- a/src/sources/vector/mod.rs +++ b/src/sources/vector/mod.rs @@ -213,8 +213,8 @@ impl SourceConfig for VectorConfig { #[cfg(test)] mod test { use lookup::owned_value_path; - use value::{kind::Collection, Kind}; use vector_core::{config::LogNamespace, schema::Definition}; + use vrl::value::{kind::Collection, Kind}; use crate::config::SourceConfig; diff --git a/src/test_util/mock/transforms/basic.rs b/src/test_util/mock/transforms/basic.rs index f1de0483da2a5..94f1860df83fe 100644 --- a/src/test_util/mock/transforms/basic.rs +++ b/src/test_util/mock/transforms/basic.rs @@ -1,7 +1,6 @@ use std::collections::BTreeSet; use async_trait::async_trait; -use value::Value; use vector_config::configurable_component; use vector_core::config::LogNamespace; use vector_core::{ @@ -13,6 +12,7 @@ use vector_core::{ schema, transform::{FunctionTransform, OutputBuffer, Transform}, }; +use vrl::value::Value; use crate::config::{OutputId, TransformConfig, TransformContext}; diff --git a/src/test_util/mock/transforms/error_definitions.rs b/src/test_util/mock/transforms/error_definitions.rs index 3d13402957dda..00c2a40223e3a 100644 --- a/src/test_util/mock/transforms/error_definitions.rs +++ b/src/test_util/mock/transforms/error_definitions.rs @@ -1,12 +1,12 @@ use async_trait::async_trait; use snafu::Snafu; -use value::Kind; use vector_config::configurable_component; use vector_core::{ config::{DataType, Input, LogNamespace, TransformOutput}, schema::Definition, transform::Transform, }; +use vrl::value::Kind; use crate::config::{OutputId, TransformConfig, TransformContext}; diff --git a/src/topology/schema.rs b/src/topology/schema.rs index 9d00ff11544cc..b0256a22178b0 100644 --- a/src/topology/schema.rs +++ b/src/topology/schema.rs @@ -444,8 +444,8 @@ mod tests { use indexmap::IndexMap; use lookup::owned_value_path; use similar_asserts::assert_eq; - use value::Kind; use vector_core::config::{DataType, SourceOutput, TransformOutput}; + use vrl::value::Kind; use super::*; diff --git a/src/trace.rs b/src/trace.rs index e5291a4d9d1e3..936d20c16a9c2 100644 --- a/src/trace.rs +++ b/src/trace.rs @@ -28,7 +28,7 @@ use tracing_subscriber::{ Layer, }; pub use tracing_tower::{InstrumentableService, InstrumentedService}; -use value::Value; +use vrl::value::Value; use crate::event::LogEvent; diff --git a/src/transforms/aws_ec2_metadata.rs b/src/transforms/aws_ec2_metadata.rs index 68c81b6b30bad..f80975bcaef5d 100644 --- a/src/transforms/aws_ec2_metadata.rs +++ b/src/transforms/aws_ec2_metadata.rs @@ -14,9 +14,9 @@ use serde_with::serde_as; use snafu::ResultExt as _; use tokio::time::{sleep, Duration, Instant}; use tracing::Instrument; -use value::Kind; use vector_config::configurable_component; use vector_core::config::LogNamespace; +use vrl::value::Kind; use crate::config::OutputId; use crate::{ @@ -723,7 +723,7 @@ mod integration_tests { transforms::test::create_topology, }; use std::collections::BTreeMap; - use value::Value; + use vrl::value::Value; use warp::Filter; fn ec2_metadata_address() -> String { diff --git a/src/transforms/metric_to_log.rs b/src/transforms/metric_to_log.rs index 34b412c325506..f3408fdf44ee6 100644 --- a/src/transforms/metric_to_log.rs +++ b/src/transforms/metric_to_log.rs @@ -4,11 +4,11 @@ use lookup::lookup_v2::parse_value_path; use lookup::{event_path, owned_value_path, path, PathPrefix}; use serde_json::Value; use std::collections::{BTreeMap, BTreeSet}; -use value::kind::Collection; -use value::Kind; use vector_common::TimeZone; use vector_config::configurable_component; use vector_core::config::LogNamespace; +use vrl::value::kind::Collection; +use vrl::value::Kind; use crate::config::OutputId; use crate::{ @@ -311,7 +311,7 @@ impl MetricToLog { // This can be removed once metrics support namespacing. log.insert( (PathPrefix::Metadata, path!("vector")), - value::Value::Object(BTreeMap::new()), + vrl::value::Value::Object(BTreeMap::new()), ); } Some(log) diff --git a/src/transforms/reduce/mod.rs b/src/transforms/reduce/mod.rs index 40d8964a9ee27..658abb1b1383e 100644 --- a/src/transforms/reduce/mod.rs +++ b/src/transforms/reduce/mod.rs @@ -28,9 +28,9 @@ mod merge_strategy; use crate::event::Value; pub use merge_strategy::*; -use value::kind::Collection; -use value::Kind; use vector_core::config::LogNamespace; +use vrl::value::kind::Collection; +use vrl::value::Kind; /// Configuration for the `reduce` transform. #[serde_as] @@ -475,7 +475,7 @@ mod test { use serde_json::json; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; - use value::Kind; + use vrl::value::Kind; use super::*; use crate::event::{LogEvent, Value}; diff --git a/src/transforms/remap.rs b/src/transforms/remap.rs index 3661969d379b0..41be9c43f9cb6 100644 --- a/src/transforms/remap.rs +++ b/src/transforms/remap.rs @@ -11,9 +11,6 @@ use codecs::MetricTagValues; use lookup::lookup_v2::{parse_value_path, ValuePath}; use lookup::{metadata_path, owned_value_path, path, OwnedTargetPath, PathPrefix}; use snafu::{ResultExt, Snafu}; -use value::kind::merge::{CollisionStrategy, Strategy}; -use value::kind::Collection; -use value::Kind; use vector_common::TimeZone; use vector_config::configurable_component; use vector_core::compile_vrl; @@ -24,6 +21,9 @@ use vrl::compiler::runtime::{Runtime, Terminate}; use vrl::compiler::state::ExternalEnv; use vrl::compiler::{CompileConfig, ExpressionError, Function, Program, TypeState, VrlRuntime}; use vrl::diagnostic::{DiagnosticMessage, Formatter, Note}; +use vrl::value::kind::merge::{CollisionStrategy, Strategy}; +use vrl::value::kind::Collection; +use vrl::value::{Kind, Value}; use crate::config::OutputId; use crate::{ @@ -385,7 +385,7 @@ pub trait VrlRunner { target: &mut VrlTarget, program: &Program, timezone: &TimeZone, - ) -> std::result::Result; + ) -> std::result::Result; } #[derive(Debug)] @@ -407,7 +407,7 @@ impl VrlRunner for AstRunner { target: &mut VrlTarget, program: &Program, timezone: &TimeZone, - ) -> std::result::Result { + ) -> std::result::Result { let result = self.runtime.resolve(target, program, timezone); self.runtime.clear(); result @@ -542,7 +542,7 @@ where } } - fn run_vrl(&mut self, target: &mut VrlTarget) -> std::result::Result { + fn run_vrl(&mut self, target: &mut VrlTarget) -> std::result::Result { self.runner.run(target, &self.program, &self.timezone) } } @@ -720,11 +720,11 @@ mod tests { use std::collections::{HashMap, HashSet}; use indoc::{formatdoc, indoc}; - use value::{ + use vector_core::{config::GlobalOptions, event::EventMetadata, metric_tags}; + use vrl::value::{ btreemap, kind::{Collection, Index}, }; - use vector_core::{config::GlobalOptions, event::EventMetadata, metric_tags}; use super::*; use crate::{ From 75a3a4ae12f62e22dee0b54c7bd81adeb5c6a450 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Wed, 3 May 2023 23:04:38 -0400 Subject: [PATCH 20/23] save --- Cargo.lock | 1 - Cargo.toml | 1 - src/cli.rs | 2 +- src/conditions/vrl.rs | 2 +- src/config/enterprise.rs | 4 ++-- src/transforms/remap.rs | 2 +- 6 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a7a45cc40ef0d..37a96acec3d8c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9308,7 +9308,6 @@ dependencies = [ "vector-lookup", "vector-vrl-functions", "vrl", - "vrl-stdlib", "warp", "windows-service", "wiremock", diff --git a/Cargo.toml b/Cargo.toml index ff3371a11611b..b14a30edcf1c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -227,7 +227,6 @@ sha2 = { version = "0.10.6", default-features = false, optional = true } # VRL Lang vrl = { package = "vrl", path = "/home/nathan/repos/vrl" } -vrl-stdlib = { package = "vrl-stdlib", path = "/home/nathan/repos/vrl/lib/stdlib" } datadog-filter = { package = "datadog-filter", path = "/home/nathan/repos/vrl/lib/datadog/filter" } datadog-search-syntax = { package = "datadog-search-syntax", path = "/home/nathan/repos/vrl/lib/datadog/search-syntax" } diff --git a/src/cli.rs b/src/cli.rs index 14be7c476e0d1..e64510ed70dd6 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -263,7 +263,7 @@ impl SubCommand { Self::Validate(v) => validate::validate(v, color).await, #[cfg(feature = "vrl-cli")] Self::Vrl(s) => { - let mut functions = vrl_stdlib::all(); + let mut functions = vrl::stdlib::all(); functions.extend(vector_vrl_functions::all()); vrl_cli::cmd::cmd(s, functions) } diff --git a/src/conditions/vrl.rs b/src/conditions/vrl.rs index 9791b84c39524..0c132ccc485aa 100644 --- a/src/conditions/vrl.rs +++ b/src/conditions/vrl.rs @@ -40,7 +40,7 @@ impl ConditionalConfig for VrlConfig { // }, // }; - let functions = vrl_stdlib::all() + let functions = vrl::stdlib::all() .into_iter() .chain(enrichment::vrl_functions().into_iter()) .chain(vector_vrl_functions::all()) diff --git a/src/config/enterprise.rs b/src/config/enterprise.rs index 6f9722b5b8fd5..97075716b3b96 100644 --- a/src/config/enterprise.rs +++ b/src/config/enterprise.rs @@ -1012,7 +1012,7 @@ mod test { ); assert!(compile_with_external( vrl.as_str(), - vrl_stdlib::all().as_ref(), + vrl::stdlib::all().as_ref(), &state, CompileConfig::default() ) @@ -1032,6 +1032,6 @@ mod test { vrl, r#". = merge(., {"pull_request":"1234","replica":"abcd","variant":"baseline"}, deep: true)"# ); - assert!(compile(vrl.as_str(), vrl_stdlib::all().as_ref()).is_ok()); + assert!(compile(vrl.as_str(), vrl::stdlib::all().as_ref()).is_ok()); } } diff --git a/src/transforms/remap.rs b/src/transforms/remap.rs index 41be9c43f9cb6..428cfe0c9cadd 100644 --- a/src/transforms/remap.rs +++ b/src/transforms/remap.rs @@ -157,7 +157,7 @@ impl RemapConfig { _ => return Err(Box::new(BuildError::SourceAndOrFile)), }; - let mut functions = vrl_stdlib::all(); + let mut functions = vrl::stdlib::all(); functions.append(&mut enrichment::vrl_functions()); functions.append(&mut vector_vrl_functions::all()); From b4d569bdb38a35bb5dd838b7e18c0b266ab2f424 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Wed, 3 May 2023 23:27:25 -0400 Subject: [PATCH 21/23] save --- Cargo.lock | 6 ------ Cargo.toml | 3 --- lib/vector-core/Cargo.toml | 1 - src/conditions/datadog_search.rs | 14 +++++++------- src/sinks/splunk_hec/logs/tests.rs | 8 +++++--- src/sources/datadog_agent/tests.rs | 8 ++++---- src/sources/dnstap/mod.rs | 2 +- src/sources/journald.rs | 2 +- src/transforms/remap.rs | 4 ++-- 9 files changed, 20 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 37a96acec3d8c..7e33ad1d257ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9056,7 +9056,6 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" name = "value" version = "0.1.0" dependencies = [ - "async-graphql", "bytes 1.4.0", "chrono", "mlua", @@ -9068,7 +9067,6 @@ dependencies = [ "serde", "serde_json", "snafu", - "toml 0.7.3", "tracing 0.1.37", ] @@ -9167,8 +9165,6 @@ dependencies = [ "criterion", "crossterm 0.26.1", "csv", - "datadog-filter", - "datadog-search-syntax", "derivative", "dirs-next", "dnsmsg-parser", @@ -9297,7 +9293,6 @@ dependencies = [ "typetag", "url", "uuid", - "value", "vector-api-client", "vector-buffers", "vector-common", @@ -9540,7 +9535,6 @@ dependencies = [ "twox-hash", "typetag", "url", - "value", "vector-buffers", "vector-common", "vector-config", diff --git a/Cargo.toml b/Cargo.toml index b14a30edcf1c3..48636b5c3afb4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -227,8 +227,6 @@ sha2 = { version = "0.10.6", default-features = false, optional = true } # VRL Lang vrl = { package = "vrl", path = "/home/nathan/repos/vrl" } -datadog-filter = { package = "datadog-filter", path = "/home/nathan/repos/vrl/lib/datadog/filter" } -datadog-search-syntax = { package = "datadog-search-syntax", path = "/home/nathan/repos/vrl/lib/datadog/search-syntax" } # External libs arc-swap = { version = "1.6", default-features = false, optional = true } @@ -358,7 +356,6 @@ test-generator = "0.3.1" tokio-test = "0.4.2" tokio = { version = "1.28.0", features = ["test-util"] } tower-test = "0.4.0" -value = { package = "value", path = "/home/nathan/repos/vrl/lib/value", features = ["test"] } vector-core = { path = "lib/vector-core", default-features = false, features = ["vrl", "test"] } wiremock = "0.5.18" zstd = { version = "0.12.3", default-features = false } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 4165ab14face6..e2ec3093a78aa 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -95,7 +95,6 @@ rand_distr = "0.4.3" tracing-subscriber = { version = "0.3.16", default-features = false, features = ["env-filter", "fmt", "ansi", "registry"] } vector-common = { path = "../vector-common", default-features = false, features = ["test"] } vrl = { path = "/home/nathan/repos/vrl", default-features = false, features = ["value", "arbitrary", "lua"] } -value = { package = "value", path = "/home/nathan/repos/vrl/lib/value", default-features = false, features = ["lua", "toml", "json", "api", "arbitrary", "test"] } [features] api = ["dep:async-graphql"] diff --git a/src/conditions/datadog_search.rs b/src/conditions/datadog_search.rs index 6be28c8f7c24b..27dcdfa57348a 100644 --- a/src/conditions/datadog_search.rs +++ b/src/conditions/datadog_search.rs @@ -1,15 +1,15 @@ use std::borrow::Cow; use bytes::Bytes; -use datadog_filter::{ +use vector_config::configurable_component; +use vector_core::event::{Event, LogEvent, Value}; +use vrl::datadog_filter::{ build_matcher, regex::{wildcard_regex, word_regex}, Filter, Matcher, Resolver, Run, }; -use datadog_search_syntax::parse; -use datadog_search_syntax::{Comparison, ComparisonValue, Field}; -use vector_config::configurable_component; -use vector_core::event::{Event, LogEvent, Value}; +use vrl::datadog_search_syntax::parse; +use vrl::datadog_search_syntax::{Comparison, ComparisonValue, Field}; use crate::conditions::{Condition, Conditional, ConditionalConfig}; @@ -312,10 +312,10 @@ where #[cfg(test)] mod test { - use datadog_filter::{build_matcher, Filter, Resolver}; - use datadog_search_syntax::parse; use serde_json::json; use vector_core::event::Event; + use vrl::datadog_filter::{build_matcher, Filter, Resolver}; + use vrl::datadog_search_syntax::parse; use super::*; use crate::log_event; diff --git a/src/sinks/splunk_hec/logs/tests.rs b/src/sinks/splunk_hec/logs/tests.rs index 9013a242c3280..5f6bf777d0d40 100644 --- a/src/sinks/splunk_hec/logs/tests.rs +++ b/src/sinks/splunk_hec/logs/tests.rs @@ -100,7 +100,7 @@ fn get_processed_event_timestamp( fn get_processed_event() -> HecProcessedEvent { get_processed_event_timestamp( - Some(value::Value::Timestamp( + Some(vrl::value::Value::Timestamp( Utc.timestamp_nanos(1638366107111456123), )), config_timestamp_key().path, @@ -274,8 +274,10 @@ fn splunk_encode_log_event_json_timestamps() { assert_eq!(hec_data.time, None); // timestamp_key is provided but timestamp is not valid type - hec_data = - get_hec_data_for_timestamp_test(Some(value::Value::Integer(0)), Some(timestamp.clone())); + hec_data = get_hec_data_for_timestamp_test( + Some(vrl::value::Value::Integer(0)), + Some(timestamp.clone()), + ); assert_eq!(hec_data.time, None); // timestamp_key is provided but no timestamp in the event diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index aa725223a8403..107f8f10aced1 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -1722,9 +1722,9 @@ fn test_config_outputs() { Kind::bytes().or_object(Collection::from_unknown(Kind::bytes())), None, ) - .unknown_fields(Kind::object(value::kind::Collection::from_unknown( - Kind::bytes(), - ))), + .unknown_fields(Kind::object( + vrl::value::kind::Collection::from_unknown(Kind::bytes()), + )), ), )]), }, @@ -1804,7 +1804,7 @@ fn test_config_outputs() { None, ) .unknown_fields(Kind::object( - value::kind::Collection::from_unknown(Kind::bytes()), + vrl::value::kind::Collection::from_unknown(Kind::bytes()), )), ), ), diff --git a/src/sources/dnstap/mod.rs b/src/sources/dnstap/mod.rs index 8686cfdf8203a..f2871689f1087 100644 --- a/src/sources/dnstap/mod.rs +++ b/src/sources/dnstap/mod.rs @@ -404,7 +404,7 @@ mod tests { }"#; let json: serde_json::Value = serde_json::from_str(record).unwrap(); - let mut event = Event::from(LogEvent::from(value::Value::from(json))); + let mut event = Event::from(LogEvent::from(vrl::value::Value::from(json))); event.as_mut_log().insert("timestamp", chrono::Utc::now()); let definition = DnstapConfig::event_schema(Some(&owned_value_path!("timestamp"))); diff --git a/src/sources/journald.rs b/src/sources/journald.rs index 33b734adaf983..461e15f0e06bb 100644 --- a/src/sources/journald.rs +++ b/src/sources/journald.rs @@ -1555,7 +1555,7 @@ mod tests { }"#; let json: serde_json::Value = serde_json::from_str(record).unwrap(); - let mut event = Event::from(LogEvent::from(value::Value::from(json))); + let mut event = Event::from(LogEvent::from(vrl::value::Value::from(json))); event.as_mut_log().insert("timestamp", chrono::Utc::now()); diff --git a/src/transforms/remap.rs b/src/transforms/remap.rs index 428cfe0c9cadd..5107a32f3bf91 100644 --- a/src/transforms/remap.rs +++ b/src/transforms/remap.rs @@ -1683,7 +1683,7 @@ mod tests { fn test_merged_array_definitions_simple() { // Test merging the array definitions where the schema definition // is simple, containing only one possible type in the array. - let object: BTreeMap = [ + let object: BTreeMap = [ ("carrot".into(), Kind::bytes()), ("potato".into(), Kind::integer()), ] @@ -1709,7 +1709,7 @@ mod tests { fn test_merged_array_definitions_complex() { // Test merging the array definitions where the schema definition // is fairly complex containing multiple different possible types. - let object: BTreeMap = [ + let object: BTreeMap = [ ("carrot".into(), Kind::bytes()), ("potato".into(), Kind::integer()), ] From 8b2254ea1a13b99862afa1f4ebfe39cee25a8fe1 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Fri, 5 May 2023 10:17:35 -0400 Subject: [PATCH 22/23] fix VRL cli --- Cargo.toml | 2 +- src/cli.rs | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c817ec232a34f..b6f8983864d1c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -226,7 +226,7 @@ hex = { version = "0.4.3", default-features = false, optional = true } sha2 = { version = "0.10.6", default-features = false, optional = true } # VRL Lang -vrl = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "v0.3.0" } +vrl = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "v0.3.0", features = ["cli"] } # External libs arc-swap = { version = "1.6", default-features = false, optional = true } diff --git a/src/cli.rs b/src/cli.rs index e64510ed70dd6..f9688a86cc133 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -237,8 +237,7 @@ pub enum SubCommand { Service(service::Opts), /// Vector Remap Language CLI - #[cfg(feature = "vrl-cli")] - Vrl(vrl_cli::Opts), + Vrl(vrl::cli::Opts), } impl SubCommand { @@ -261,11 +260,10 @@ impl SubCommand { #[cfg(feature = "api-client")] Self::Top(t) => top::cmd(t).await, Self::Validate(v) => validate::validate(v, color).await, - #[cfg(feature = "vrl-cli")] Self::Vrl(s) => { let mut functions = vrl::stdlib::all(); functions.extend(vector_vrl_functions::all()); - vrl_cli::cmd::cmd(s, functions) + vrl::cli::cmd::cmd(s, functions) } } } From 8e46bccd6deb0a092ca046ac2f36717edae78bb3 Mon Sep 17 00:00:00 2001 From: Nathan Fox Date: Fri, 5 May 2023 11:54:39 -0400 Subject: [PATCH 23/23] fix WASM compilation --- lib/enrichment/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/enrichment/Cargo.toml b/lib/enrichment/Cargo.toml index 14bdf10665428..ef118022f36d8 100644 --- a/lib/enrichment/Cargo.toml +++ b/lib/enrichment/Cargo.toml @@ -10,4 +10,4 @@ arc-swap = { version = "1.6.0", default-features = false } dyn-clone = { version = "1.0.11", default-features = false } chrono = { version = "0.4.19", default-features = false } vector-common = { path = "../vector-common", default-features = false, features = [ "btreemap", "conversion", "serde" ] } -vrl = { git = "https://github.com/vectordotdev/vrl", rev = "v0.3.0" } +vrl = { git = "https://github.com/vectordotdev/vrl", rev = "v0.3.0", default-features = false, features = ["diagnostic"] }