Skip to content

fix(loki sink): update to use the global list of compression algorithms #19104

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
StephenWakely merged 3 commits into from
Nov 9, 2023
Merged

fix(loki sink): update to use the global list of compression algorithms #19104

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0bc6bb5
Update to use the global list of compression algorithms
StephenWakely Nov 9, 2023
9b44c2d
Mention snappy compression using protocol buffers
StephenWakely Nov 9, 2023
8529438
Snappy is compressed separately
StephenWakely Nov 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 6 additions & 39 deletions src/sinks/loki/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,42 +9,8 @@ use crate::{
sinks::{prelude::*, util::UriSerde},
};

/// Loki-specific compression.
#[configurable_component]
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum ExtendedCompression {
/// Snappy compression.
///
/// This implies sending push requests as Protocol Buffers.
#[serde(rename = "snappy")]
Snappy,
}

/// Compression configuration.
#[configurable_component]
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
#[serde(untagged)]
pub enum CompressionConfigAdapter {
/// Basic compression.
Original(Compression),

/// Loki-specific compression.
Extended(ExtendedCompression),
}

impl CompressionConfigAdapter {
pub const fn content_encoding(self) -> Option<&'static str> {
match self {
CompressionConfigAdapter::Original(compression) => compression.content_encoding(),
CompressionConfigAdapter::Extended(_) => Some("snappy"),
}
}
}

impl Default for CompressionConfigAdapter {
fn default() -> Self {
CompressionConfigAdapter::Extended(ExtendedCompression::Snappy)
}
const fn default_compression() -> Compression {
Compression::Snappy
}

fn default_loki_path() -> String {
Expand Down Expand Up @@ -106,9 +72,10 @@ pub struct LokiConfig {
#[serde(default = "crate::serde::default_true")]
pub remove_timestamp: bool,

#[configurable(derived)]
#[serde(default)]
pub compression: CompressionConfigAdapter,
/// Compression configuration.
/// Snappy compression implies sending push requests as Protocol Buffers.
#[serde(default = "default_compression")]
pub compression: Compression,

#[configurable(derived)]
#[serde(default)]
Expand Down
9 changes: 3 additions & 6 deletions src/sinks/loki/service.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ use http::StatusCode;
use snafu::Snafu;
use tracing::Instrument;

use crate::sinks::loki::config::{CompressionConfigAdapter, ExtendedCompression};
use crate::{
http::{Auth, HttpClient},
sinks::{prelude::*, util::UriSerde},
Expand Down Expand Up @@ -60,7 +59,7 @@ impl DriverResponse for LokiResponse {

#[derive(Clone)]
pub struct LokiRequest {
pub compression: CompressionConfigAdapter,
pub compression: Compression,
pub finalizers: EventFinalizers,
pub payload: Bytes,
pub tenant_id: Option<String>,
Expand Down Expand Up @@ -113,10 +112,8 @@ impl Service<LokiRequest> for LokiService {

fn call(&mut self, request: LokiRequest) -> Self::Future {
let content_type = match request.compression {
CompressionConfigAdapter::Original(_) => "application/json",
CompressionConfigAdapter::Extended(ExtendedCompression::Snappy) => {
"application/x-protobuf"
}
Compression::Snappy => "application/x-protobuf",
_ => "application/json",
};
let mut req = http::Request::post(&self.endpoint.uri).header("Content-Type", content_type);

Expand Down
18 changes: 9 additions & 9 deletions src/sinks/loki/sink.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ use super::{
event::{LokiBatchEncoder, LokiEvent, LokiRecord, PartitionKey},
service::{LokiRequest, LokiRetryLogic, LokiService},
};
use crate::sinks::loki::config::{CompressionConfigAdapter, ExtendedCompression};
use crate::sinks::loki::event::LokiBatchEncoding;
use crate::{
http::{get_http_scheme_from_uri, HttpClient},
Expand Down Expand Up @@ -65,7 +64,7 @@ impl Partitioner for RecordPartitioner {

#[derive(Clone)]
pub struct LokiRequestBuilder {
compression: CompressionConfigAdapter,
compression: Compression,
encoder: LokiBatchEncoder,
}

Expand All @@ -92,9 +91,12 @@ impl RequestBuilder<(PartitionKey, Vec<LokiRecord>)> for LokiRequestBuilder {
type Error = RequestBuildError;

fn compression(&self) -> Compression {
match self.compression {
CompressionConfigAdapter::Original(compression) => compression,
CompressionConfigAdapter::Extended(_) => Compression::None,
if self.compression == Compression::Snappy {
// Snappy compression is applied after converting the batch to protobuf so
// we need to handle this separately.
Compression::None
} else {
self.compression
}
Comment on lines +94 to 100
Copy link
Contributor

@dsmith3197 dsmith3197 Nov 9, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does it make more sense to remove the snap encoding here and defer to the request builder here?

Copy link
Contributor Author

@StephenWakely StephenWakely Nov 9, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm.. potentially. I'll look into that for merging into master.

}

Expand Down Expand Up @@ -415,10 +417,8 @@ impl LokiSink {
let serializer = config.encoding.build()?;
let encoder = Encoder::<()>::new(serializer);
let batch_encoder = match config.compression {
CompressionConfigAdapter::Original(_) => LokiBatchEncoder(LokiBatchEncoding::Json),
CompressionConfigAdapter::Extended(ExtendedCompression::Snappy) => {
LokiBatchEncoder(LokiBatchEncoding::Protobuf)
}
Compression::Snappy => LokiBatchEncoder(LokiBatchEncoding::Protobuf),
_ => LokiBatchEncoder(LokiBatchEncoding::Json),
};

Ok(Self {
Expand Down
11 changes: 7 additions & 4 deletions website/cue/reference/components/sinks/base/loki.cue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,11 @@ base: components: sinks: loki: configuration: {
}
}
compression: {
description: "Compression configuration."
required: false
description: """
Compression configuration.
Snappy compression implies sending push requests as Protocol Buffers.
"""
required: false
type: string: {
default: "snappy"
enum: {
Expand All @@ -122,9 +125,9 @@ base: components: sinks: loki: configuration: {
"""
none: "No compression."
snappy: """
Snappy compression.
[Snappy][snappy] compression.

This implies sending push requests as Protocol Buffers.
[snappy]: https://github.com/google/snappy/blob/main/docs/README.md
"""
zlib: """
[Zlib][zlib] compression.
Expand Down