From 1e858c42fe72a3b383cc5e5bd902d67c76feebaa Mon Sep 17 00:00:00 2001 From: neuronull Date: Mon, 17 Apr 2023 11:53:32 -0600 Subject: [PATCH 1/4] remove max_length setting --- .../src/decoding/framing/newline_delimited.rs | 2 +- .../src/decoding/framing/octet_counting.rs | 2 +- src/sources/socket/mod.rs | 68 +++++++------------ src/sources/socket/tcp.rs | 26 +------ src/sources/socket/unix.rs | 29 ++++---- 5 files changed, 42 insertions(+), 85 deletions(-) diff --git a/lib/codecs/src/decoding/framing/newline_delimited.rs b/lib/codecs/src/decoding/framing/newline_delimited.rs index 1ca8e68815c60..cc96d08d7ab40 100644 --- a/lib/codecs/src/decoding/framing/newline_delimited.rs +++ b/lib/codecs/src/decoding/framing/newline_delimited.rs @@ -34,7 +34,7 @@ pub struct NewlineDelimitedDecoderOptions { /// consider setting the maximum length to a reasonably large value as a safety net. This /// ensures that processing is not actually unbounded. #[serde(skip_serializing_if = "vector_core::serde::skip_serializing_if_default")] - max_length: Option, + pub max_length: Option, } impl NewlineDelimitedDecoderOptions { diff --git a/lib/codecs/src/decoding/framing/octet_counting.rs b/lib/codecs/src/decoding/framing/octet_counting.rs index 60a6408a50c7d..98fdfbc78fdf0 100644 --- a/lib/codecs/src/decoding/framing/octet_counting.rs +++ b/lib/codecs/src/decoding/framing/octet_counting.rs @@ -38,7 +38,7 @@ impl OctetCountingDecoderConfig { pub struct OctetCountingDecoderOptions { /// The maximum length of the byte buffer. #[serde(skip_serializing_if = "vector_core::serde::skip_serializing_if_default")] - max_length: Option, + pub max_length: Option, } /// Codec using the `Octet Counting` format as specified in diff --git a/src/sources/socket/mod.rs b/src/sources/socket/mod.rs index c21a84b30da7a..3aeea5d342a90 100644 --- a/src/sources/socket/mod.rs +++ b/src/sources/socket/mod.rs @@ -3,7 +3,7 @@ pub mod udp; #[cfg(unix)] mod unix; -use codecs::{decoding::DeserializerConfig, NewlineDelimitedDecoderConfig}; +use codecs::decoding::DeserializerConfig; use lookup::{lookup_v2::OptionalValuePath, owned_value_path}; use value::{kind::Collection, Kind}; use vector_config::configurable_component; @@ -113,26 +113,18 @@ impl SourceConfig for SocketConfig { async fn build(&self, cx: SourceContext) -> crate::Result { match self.mode.clone() { Mode::Tcp(config) => { - let decoding = config.decoding().clone(); - // TODO: in v0.30.0 , remove the `max_length` setting from - // the UnixConfig, and all of the below mess and replace - // it with the configured framing / - // decoding.default_stream_framing(). - let framing = match (config.framing().clone(), config.max_length()) { - (Some(framing), Some(_)) => { - warn!(message = "DEPRECATION: The `max_length` setting is deprecated and will be removed in an upcoming release. Since a `framing` setting was provided, the `max_length` setting has no effect."); - framing - } - (Some(framing), None) => framing, - (None, Some(max_length)) => { - warn!(message = "DEPRECATION: The `max_length` setting is deprecated and will be removed in an upcoming release. Please configure the `max_length` from the `framing` setting instead."); - NewlineDelimitedDecoderConfig::new_with_max_length(max_length).into() - } - (None, None) => decoding.default_stream_framing(), - }; - let log_namespace = cx.log_namespace(config.log_namespace); - let decoder = DecodingConfig::new(framing, decoding, log_namespace).build(); + + let decoding = config.decoding().clone(); + let decoder = DecodingConfig::new( + config + .framing + .clone() + .unwrap_or_else(|| decoding.default_stream_framing()), + decoding, + log_namespace, + ) + .build(); let tcp = tcp::RawTcpSource::new(config.clone(), decoder, log_namespace); let tls_config = config.tls().as_ref().map(|tls| tls.tls_config.clone()); @@ -190,27 +182,18 @@ impl SourceConfig for SocketConfig { } #[cfg(unix)] Mode::UnixStream(config) => { - let decoding = config.decoding.clone(); - - // TODO: in v0.30.0 , remove the `max_length` setting from - // the UnixConfig, and all of the below mess and replace - // it with the configured framing / - // decoding.default_stream_framing(). - let framing = match (config.framing.clone(), config.max_length) { - (Some(framing), Some(_)) => { - warn!(message = "DEPRECATION: The `max_length` setting is deprecated and will be removed in an upcoming release. Since a `framing` setting was provided, the `max_length` setting has no effect."); - framing - } - (Some(framing), None) => framing, - (None, Some(max_length)) => { - warn!(message = "DEPRECATION: The `max_length` setting is deprecated and will be removed in an upcoming release. Please configure the `max_length` from the `framing` setting instead."); - NewlineDelimitedDecoderConfig::new_with_max_length(max_length).into() - } - (None, None) => decoding.default_stream_framing(), - }; - let log_namespace = cx.log_namespace(config.log_namespace); - let decoder = DecodingConfig::new(framing, decoding, log_namespace).build(); + + let decoding = config.decoding().clone(); + let decoder = DecodingConfig::new( + config + .framing + .clone() + .unwrap_or_else(|| decoding.default_stream_framing()), + decoding, + log_namespace, + ) + .build(); unix::unix_stream(config, decoder, cx.shutdown, cx.out, log_namespace) } @@ -335,10 +318,6 @@ pub(crate) fn default_host_key() -> OptionalValuePath { OptionalValuePath::from(owned_value_path!(log_schema().host_key())) } -fn default_max_length() -> Option { - Some(crate::serde::default_max_length()) -} - #[cfg(test)] mod test { use approx::assert_relative_eq; @@ -529,7 +508,6 @@ mod test { let addr = next_addr(); let mut config = TcpConfig::from_address(addr.into()); - config.set_max_length(None); config.set_framing(Some( NewlineDelimitedDecoderConfig::new_with_max_length(10).into(), )); diff --git a/src/sources/socket/tcp.rs b/src/sources/socket/tcp.rs index 3345722fae859..6135ca0e3cf68 100644 --- a/src/sources/socket/tcp.rs +++ b/src/sources/socket/tcp.rs @@ -17,7 +17,7 @@ use crate::{ tls::TlsSourceConfig, }; -use super::{default_host_key, default_max_length, SocketConfig}; +use super::{default_host_key, SocketConfig}; /// TCP configuration for the `socket` source. #[serde_as] @@ -30,16 +30,6 @@ pub struct TcpConfig { #[configurable(derived)] keepalive: Option, - /// The maximum buffer size of incoming messages. - /// - /// Messages larger than this are truncated. - // TODO: communicated as deprecated in v0.29.0, can be removed in v0.30.0 - #[configurable( - deprecated = "This option has been deprecated. Configure `max_length` on the framing config instead." - )] - #[configurable(metadata(docs::type_unit = "bytes"))] - max_length: Option, - /// The timeout before a connection is forcefully closed during shutdown. #[serde(default = "default_shutdown_timeout_secs")] #[serde_as(as = "serde_with::DurationSeconds")] @@ -85,11 +75,11 @@ pub struct TcpConfig { pub connection_limit: Option, #[configurable(derived)] - framing: Option, + pub(super) framing: Option, #[configurable(derived)] #[serde(default = "default_decoding")] - decoding: DeserializerConfig, + pub(super) decoding: DeserializerConfig, /// The namespace to use for logs. This overrides the global setting. #[serde(default)] @@ -110,7 +100,6 @@ impl TcpConfig { Self { address, keepalive: None, - max_length: default_max_length(), shutdown_timeout_secs: default_shutdown_timeout_secs(), host_key: default_host_key(), port_key: default_port_key(), @@ -152,10 +141,6 @@ impl TcpConfig { self.keepalive } - pub const fn max_length(&self) -> Option { - self.max_length - } - pub const fn shutdown_timeout_secs(&self) -> Duration { self.shutdown_timeout_secs } @@ -173,11 +158,6 @@ impl TcpConfig { self } - pub fn set_max_length(&mut self, val: Option) -> &mut Self { - self.max_length = val; - self - } - pub fn set_shutdown_timeout_secs(&mut self, val: u64) -> &mut Self { self.shutdown_timeout_secs = Duration::from_secs(val); self diff --git a/src/sources/socket/unix.rs b/src/sources/socket/unix.rs index 942af6f5cc5f6..2144a8c8e0278 100644 --- a/src/sources/socket/unix.rs +++ b/src/sources/socket/unix.rs @@ -19,7 +19,7 @@ use crate::{ SourceSender, }; -use super::{default_host_key, default_max_length, SocketConfig}; +use super::{default_host_key, SocketConfig}; /// Unix domain socket configuration for the `socket` source. #[configurable_component] @@ -41,16 +41,6 @@ pub struct UnixConfig { #[configurable(metadata(docs::examples = 508))] pub socket_file_mode: Option, - /// The maximum buffer size of incoming messages. - /// - /// Messages larger than this are truncated. - // TODO: communicated as deprecated in v0.29.0, can be removed in v0.30.0 - #[configurable( - deprecated = "This option has been deprecated. Configure `max_length` on the framing config instead." - )] - #[configurable(metadata(docs::type_unit = "bytes"))] - pub max_length: Option, - /// Overrides the name of the log field used to add the peer host to each event. /// /// The value will be the peer host's address, including the port i.e. `1.2.3.4:9000`. @@ -82,7 +72,6 @@ impl UnixConfig { Self { path, socket_file_mode: None, - max_length: default_max_length(), host_key: default_host_key(), framing: None, decoding: default_decoding(), @@ -135,12 +124,22 @@ pub(super) fn unix_datagram( out: SourceSender, log_namespace: LogNamespace, ) -> crate::Result { + let max_length = config + .framing + .and_then(|framing| match framing { + FramingConfig::CharacterDelimited { + character_delimited, + } => character_delimited.max_length, + FramingConfig::NewlineDelimited { newline_delimited } => newline_delimited.max_length, + FramingConfig::OctetCounting { octet_counting } => octet_counting.max_length, + _ => None, + }) + .unwrap_or_else(crate::serde::default_max_length); + build_unix_datagram_source( config.path, config.socket_file_mode, - config - .max_length - .unwrap_or_else(crate::serde::default_max_length), + max_length, decoder, move |events, received_from| { handle_events(events, &config.host_key, received_from, log_namespace) From a1b00a962a14a7bc08c0058506a46330d3e81ef6 Mon Sep 17 00:00:00 2001 From: neuronull Date: Mon, 17 Apr 2023 12:04:43 -0600 Subject: [PATCH 2/4] add upgrade guide --- .../2023-05-23-0-30-0-upgrade-guide.md | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 website/content/en/highlights/2023-05-23-0-30-0-upgrade-guide.md diff --git a/website/content/en/highlights/2023-05-23-0-30-0-upgrade-guide.md b/website/content/en/highlights/2023-05-23-0-30-0-upgrade-guide.md new file mode 100644 index 0000000000000..725134016f6bc --- /dev/null +++ b/website/content/en/highlights/2023-05-23-0-30-0-upgrade-guide.md @@ -0,0 +1,32 @@ +--- +date: "2023-05-23" +title: "0.30 Upgrade Guide" +description: "An upgrade guide that addresses breaking changes in 0.30.0" +authors: ["neuronull"] +release: "0.30.0" +hide_on_release_notes: false +badges: + type: breaking change +--- + +Vector's 0.30.0 release includes **breaking changes**: + +1. [Removal of the `socket` source's `tcp` and `unix` mode `max_length` setting](#socket-source-max-length) + +We cover them below to help you upgrade quickly: + +## Upgrade guide + +### Breaking changes + +#### Removal of the `socket` source's `tcp` and `unix` mode `max_length` setting {#socket-source-max-length} + +In v0.29.0 the `socket` source modes `tcp` and `unix` setting `max_length` +was marked as deprecated. + +That setting was replaced by the `max_length` setting within the `framing` +setting. + +Any explicit usages of `max_length` will no longer work and configurations +will need to instead use a `framing` setting in order to set a maximum +length. From 11d77a415cb03c70f5d9b758f328de4a0b43d2ff Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 18 Apr 2023 11:32:14 -0600 Subject: [PATCH 3/4] update gen docs + remove unecessary Option --- src/sources/socket/mod.rs | 4 ++-- src/sources/socket/udp.rs | 10 ++++------ .../cue/reference/components/sources/base/socket.cue | 10 ++++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/sources/socket/mod.rs b/src/sources/socket/mod.rs index 3aeea5d342a90..888df782f08bd 100644 --- a/src/sources/socket/mod.rs +++ b/src/sources/socket/mod.rs @@ -1004,7 +1004,7 @@ mod test { let (tx, rx) = SourceSender::new_test(); let address = next_addr(); let mut config = UdpConfig::from_address(address.into()); - config.max_length = Some(11); + config.max_length = 11; let address = init_udp_with_config(tx, config).await; send_lines_udp( @@ -1040,7 +1040,7 @@ mod test { let (tx, rx) = SourceSender::new_test(); let address = next_addr(); let mut config = UdpConfig::from_address(address.into()); - config.max_length = Some(10); + config.max_length = 10; config.framing = CharacterDelimitedDecoderConfig { character_delimited: CharacterDelimitedDecoderOptions::new(b',', None), } diff --git a/src/sources/socket/udp.rs b/src/sources/socket/udp.rs index f7400d539a09f..d6bbee7f45dc6 100644 --- a/src/sources/socket/udp.rs +++ b/src/sources/socket/udp.rs @@ -45,7 +45,7 @@ pub struct UdpConfig { /// Messages larger than this are truncated. #[serde(default = "default_max_length")] #[configurable(metadata(docs::type_unit = "bytes"))] - pub(super) max_length: Option, + pub(super) max_length: usize, /// Overrides the name of the log field used to add the peer host to each event. /// @@ -95,8 +95,8 @@ fn default_port_key() -> OptionalValuePath { OptionalValuePath::from(owned_value_path!("port")) } -fn default_max_length() -> Option { - Some(crate::serde::default_max_length()) +fn default_max_length() -> usize { + crate::serde::default_max_length() } impl UdpConfig { @@ -163,9 +163,7 @@ pub(super) fn udp( } } - let mut max_length = config - .max_length - .unwrap_or_else(|| default_max_length().unwrap()); + let mut max_length = config.max_length; if let Some(receive_buffer_bytes) = config.receive_buffer_bytes { max_length = std::cmp::min(max_length, receive_buffer_bytes); diff --git a/website/cue/reference/components/sources/base/socket.cue b/website/cue/reference/components/sources/base/socket.cue index ff4764d8d3837..f6236c0d1c94e 100644 --- a/website/cue/reference/components/sources/base/socket.cue +++ b/website/cue/reference/components/sources/base/socket.cue @@ -191,15 +191,17 @@ base: components: sources: socket: configuration: { type: uint: unit: "seconds" } max_length: { - deprecated: true - deprecated_message: "This option has been deprecated. Configure `max_length` on the framing config instead." description: """ The maximum buffer size of incoming messages. Messages larger than this are truncated. """ - required: false - type: uint: unit: "bytes" + relevant_when: "mode = \"udp\"" + required: false + type: uint: { + default: 102400 + unit: "bytes" + } } mode: { description: "The type of socket to use." From d24c9124179916c8502d35d8659f6c992ea899ef Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 18 Apr 2023 17:14:54 -0600 Subject: [PATCH 4/4] feedback tobz --- .../en/highlights/2023-05-23-0-30-0-upgrade-guide.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/website/content/en/highlights/2023-05-23-0-30-0-upgrade-guide.md b/website/content/en/highlights/2023-05-23-0-30-0-upgrade-guide.md index 725134016f6bc..4f8e7f7294e34 100644 --- a/website/content/en/highlights/2023-05-23-0-30-0-upgrade-guide.md +++ b/website/content/en/highlights/2023-05-23-0-30-0-upgrade-guide.md @@ -21,12 +21,12 @@ We cover them below to help you upgrade quickly: #### Removal of the `socket` source's `tcp` and `unix` mode `max_length` setting {#socket-source-max-length} -In v0.29.0 the `socket` source modes `tcp` and `unix` setting `max_length` -was marked as deprecated. +In v0.29.0 the `max_length` setting, used by the `tcp` and `unix` modes +of the `socket` source, was marked as deprecated. That setting was replaced by the `max_length` setting within the `framing` setting. -Any explicit usages of `max_length` will no longer work and configurations -will need to instead use a `framing` setting in order to set a maximum -length. +Any explicit usages of `max_length` for those modes of the `socket` +source will no longer work and configurations will need to instead use +a `framing` setting in order to set a maximum length.