Skip to content
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion components/api-server/src/bin/api_server.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use anyhow::Context;
use clap::Parser;
use clp_rust_utils::{clp_config::package, serde::yaml};
use clp_rust_utils::clp_config::package;
use clp_rust_utils::serde::yaml;

#[derive(Parser)]
#[command(version, about = "API Server for CLP.")]
Expand Down
32 changes: 19 additions & 13 deletions components/api-server/src/client.rs
Original file line number Diff line number Diff line change
@@ -1,22 +1,28 @@
use std::pin::Pin;

use async_stream::stream;
use chrono::{DateTime, TimeZone, Utc};
use chrono::DateTime;
use chrono::TimeZone;
use chrono::Utc;
use clp_rust_utils::aws::AWS_DEFAULT_REGION;
use clp_rust_utils::clp_config::package::config::Config;
use clp_rust_utils::clp_config::package::config::StorageEngine;
use clp_rust_utils::clp_config::package::config::StreamOutputStorage;
use clp_rust_utils::clp_config::package::credentials::Credentials;
use clp_rust_utils::database::mysql::create_clp_db_mysql_pool;
pub use clp_rust_utils::job_config::CompressionJobStatus;
use clp_rust_utils::{
aws::AWS_DEFAULT_REGION,
clp_config::package::{
config::{Config, StorageEngine, StreamOutputStorage},
credentials::Credentials,
},
database::mysql::create_clp_db_mysql_pool,
job_config::{QUERY_JOBS_TABLE_NAME, QueryJobStatus, QueryJobType, SearchJobConfig},
};
use futures::{Stream, StreamExt};
use clp_rust_utils::job_config::QUERY_JOBS_TABLE_NAME;
use clp_rust_utils::job_config::QueryJobStatus;
use clp_rust_utils::job_config::QueryJobType;
use clp_rust_utils::job_config::SearchJobConfig;
use futures::Stream;
use futures::StreamExt;
use pin_project_lite::pin_project;
use serde::{Deserialize, Serialize};
use serde::Deserialize;
use serde::Serialize;
use sqlx::Row;
use utoipa::{IntoParams, ToSchema};
use utoipa::IntoParams;
use utoipa::ToSchema;

pub use crate::error::ClientError;

Expand Down
3 changes: 2 additions & 1 deletion components/api-server/src/error.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use aws_sdk_s3::{error::SdkError, primitives::ByteStreamError};
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::primitives::ByteStreamError;
use num_enum::TryFromPrimitive;
use thiserror::Error;

Expand Down
75 changes: 37 additions & 38 deletions components/api-server/src/routes.rs
Original file line number Diff line number Diff line change
@@ -1,29 +1,31 @@
use axum::{
Json,
extract::{Path, Query, State},
http::StatusCode,
response::{
IntoResponse,
Sse,
sse::{Event, KeepAlive},
},
routing::get,
};
use futures::{Stream, StreamExt};
use serde::{Deserialize, Serialize};
use axum::Json;
use axum::extract::Path;
use axum::extract::Query;
use axum::extract::State;
use axum::http::StatusCode;
use axum::response::IntoResponse;
use axum::response::Sse;
use axum::response::sse::Event;
use axum::response::sse::KeepAlive;
use axum::routing::get;
use futures::Stream;
use futures::StreamExt;
use serde::Deserialize;
use serde::Serialize;
use thiserror::Error;
use tower_http::cors::{Any, CorsLayer};
use utoipa::{OpenApi, ToSchema};
use utoipa_axum::{router::OpenApiRouter, routes};

use crate::client::{
Client,
ClientError,
CompressionUsage,
CompressionUsageParams,
QueryConfig,
ValidatedCompressionUsageParams,
};
use tower_http::cors::Any;
use tower_http::cors::CorsLayer;
use utoipa::OpenApi;
use utoipa::ToSchema;
use utoipa_axum::router::OpenApiRouter;
use utoipa_axum::routes;

use crate::client::Client;
use crate::client::ClientError;
use crate::client::CompressionUsage;
use crate::client::CompressionUsageParams;
use crate::client::QueryConfig;
use crate::client::ValidatedCompressionUsageParams;

/// Factory method to create an Axum router configured with all API routes.
///
Expand Down Expand Up @@ -65,14 +67,12 @@ pub fn from_client(client: Client) -> Result<axum::Router, serde_json::Error> {
mod api_doc {
// Using `super::...` can cause `super` to appear as a tag in the generated OpenAPI
// documentation. Importing the paths directly prevents this issue.
use super::{
__path_cancel_query,
__path_compression_usage,
__path_health,
__path_query,
__path_query_results,
CompressionUsage,
};
use super::__path_cancel_query;
use super::__path_compression_usage;
use super::__path_health;
use super::__path_query;
use super::__path_query_results;
use super::CompressionUsage;
use crate::client::CompressionJobStatus;

#[derive(utoipa::OpenApi)]
Expand Down Expand Up @@ -337,11 +337,10 @@ impl IntoResponse for HandlerError {

#[cfg(test)]
mod tests {
use axum::{
body::Body,
http::{Request, StatusCode},
routing::get,
};
use axum::body::Body;
use axum::http::Request;
use axum::http::StatusCode;
use axum::routing::get;
use http_body_util::BodyExt;
use tower::ServiceExt;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use serde::Deserialize;

use crate::clp_config::{AwsAuthentication, S3Config};
use crate::clp_config::AwsAuthentication;
use crate::clp_config::S3Config;

/// Mirror of `clp_py_utils.clp_config.ClpConfig`.
///
Expand Down
3 changes: 2 additions & 1 deletion components/clp-rust-utils/src/clp_config/s3_config.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use non_empty_string::NonEmptyString;
use serde::{Deserialize, Serialize};
use serde::Deserialize;
use serde::Serialize;

/// Represents the configuration for connecting to an S3 bucket.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
Expand Down
6 changes: 2 additions & 4 deletions components/clp-rust-utils/src/database/mysql.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
use secrecy::ExposeSecret;
use strum::IntoEnumIterator;

use crate::clp_config::package::{
config::Database as DatabaseConfig,
credentials::Database as DatabaseCredentials,
};
use crate::clp_config::package::config::Database as DatabaseConfig;
use crate::clp_config::package::credentials::Database as DatabaseCredentials;

/// Implements [`sqlx::Type<sqlx::MySql>`] for `$ty` by delegating to `$delegate`.
///
Expand Down
8 changes: 3 additions & 5 deletions components/clp-rust-utils/src/job_config/clp_io_config.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
use non_empty_string::NonEmptyString;
use serde::Serialize;

use crate::{
clp_config::S3Config,
job_config::ingestion::JobId as IngestionJobId,
s3::S3ObjectMetadataId,
};
use crate::clp_config::S3Config;
use crate::job_config::ingestion::JobId as IngestionJobId;
use crate::s3::S3ObjectMetadataId;

/// Represents CLP IO config.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
Expand Down
6 changes: 4 additions & 2 deletions components/clp-rust-utils/src/job_config/compression.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
use num_enum::{IntoPrimitive, TryFromPrimitive};
use serde::{Deserialize, Serialize};
use num_enum::IntoPrimitive;
use num_enum::TryFromPrimitive;
use serde::Deserialize;
use serde::Serialize;
use strum::EnumString;
use utoipa::ToSchema;

Expand Down
3 changes: 2 additions & 1 deletion components/clp-rust-utils/src/job_config/ingestion.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
pub mod s3 {
use non_empty_string::NonEmptyString;
use serde::{Deserialize, Serialize};
use serde::Deserialize;
use serde::Serialize;
use thiserror::Error;
use utoipa::ToSchema;

Expand Down
6 changes: 4 additions & 2 deletions components/clp-rust-utils/src/job_config/search.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
use num_enum::{IntoPrimitive, TryFromPrimitive};
use serde::{Deserialize, Serialize};
use num_enum::IntoPrimitive;
use num_enum::TryFromPrimitive;
use serde::Deserialize;
use serde::Serialize;

pub const QUERY_JOBS_TABLE_NAME: &str = "query_jobs";

Expand Down
11 changes: 6 additions & 5 deletions components/clp-rust-utils/src/logging.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
use tracing_appender::{
non_blocking::{NonBlockingBuilder, WorkerGuard},
rolling::{RollingFileAppender, Rotation},
};
use tracing_subscriber::{self, fmt::writer::MakeWriterExt};
use tracing_appender::non_blocking::NonBlockingBuilder;
use tracing_appender::non_blocking::WorkerGuard;
use tracing_appender::rolling::RollingFileAppender;
use tracing_appender::rolling::Rotation;
use tracing_subscriber::fmt::writer::MakeWriterExt;
use tracing_subscriber::{self};

/// Opaque struct to hold the worker guards for the background log writers.
/// These guards must be held for the lifetime of the program to ensure logs are flushed.
Expand Down
8 changes: 4 additions & 4 deletions components/clp-rust-utils/src/s3/client.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
use aws_config::BehaviorVersion;
use aws_sdk_s3::{
Client,
config::{Builder, Credentials, Region},
};
use aws_sdk_s3::Client;
use aws_sdk_s3::config::Builder;
use aws_sdk_s3::config::Credentials;
use aws_sdk_s3::config::Region;
use non_empty_string::NonEmptyString;

use crate::clp_config::AwsAuthentication;
Expand Down
8 changes: 4 additions & 4 deletions components/clp-rust-utils/src/sqs/client.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
use aws_config::BehaviorVersion;
use aws_sdk_sqs::{
Client,
config::{Builder, Credentials, Region},
};
use aws_sdk_sqs::Client;
use aws_sdk_sqs::config::Builder;
use aws_sdk_sqs::config::Credentials;
use aws_sdk_sqs::config::Region;
use non_empty_string::NonEmptyString;

use crate::clp_config::AwsAuthentication;
Expand Down
3 changes: 2 additions & 1 deletion components/clp-rust-utils/src/task_io/compression.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
//! Protocol types exchanged with the Spider (Huntsman) tasks that run CLP S3 compression jobs.

use non_empty_string::NonEmptyString;
use serde::{Deserialize, Serialize};
use serde::Deserialize;
use serde::Serialize;

use crate::clp_config::AwsAuthentication;

Expand Down
6 changes: 4 additions & 2 deletions components/clp-rust-utils/src/telemetry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@

use std::env;

use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider};
use opentelemetry_sdk::metrics::PeriodicReader;
use opentelemetry_sdk::metrics::SdkMeterProvider;

use crate::{Error, clp_config::package::config::Telemetry};
use crate::Error;
use crate::clp_config::package::config::Telemetry;

/// RAII guard that shuts down the meter provider and flushes pending metric exports when dropped.
pub struct TelemetryGuard {
Expand Down
15 changes: 9 additions & 6 deletions components/clp-rust-utils/tests/clp_config_test.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
use clp_rust_utils::{
clp_config::{AwsAuthentication, AwsCredentials, S3Config},
job_config::{ClpIoConfig, InputConfig, OutputConfig, S3ObjectMetadataInputConfig},
serde::BrotliMsgpack,
types::non_empty_string::ExpectedNonEmpty,
};
use clp_rust_utils::clp_config::AwsAuthentication;
use clp_rust_utils::clp_config::AwsCredentials;
use clp_rust_utils::clp_config::S3Config;
use clp_rust_utils::job_config::ClpIoConfig;
use clp_rust_utils::job_config::InputConfig;
use clp_rust_utils::job_config::OutputConfig;
use clp_rust_utils::job_config::S3ObjectMetadataInputConfig;
use clp_rust_utils::serde::BrotliMsgpack;
use clp_rust_utils::types::non_empty_string::ExpectedNonEmpty;
use non_empty_string::NonEmptyString;
use serde_json::Value;

Expand Down
7 changes: 3 additions & 4 deletions components/clp-tdl-package/src/common.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
//! Process-global state shared by this package's tasks: the Tokio runtime, the Spider task
//! executor config, and this cdylib's `tracing` subscriber.

use std::{
path::{Path, PathBuf},
sync::OnceLock,
};
use std::path::Path;
use std::path::PathBuf;
use std::sync::OnceLock;

use anyhow::Context;
use clp_rust_utils::clp_config::package::config::SpiderTaskExecutorConfig;
Expand Down
12 changes: 6 additions & 6 deletions components/clp-tdl-package/src/task/compression/mod.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
//! The compression tasks: the `#[task]` wrappers Spider invokes and their implementations.

use clp_rust_utils::task_io::compression::{
ClpSCompressionOption,
CompressionTaskOutput,
S3InputSource,
};
use spider_tdl::{TaskContext, TdlError, task};
use clp_rust_utils::task_io::compression::ClpSCompressionOption;
use clp_rust_utils::task_io::compression::CompressionTaskOutput;
use clp_rust_utils::task_io::compression::S3InputSource;
use spider_tdl::TaskContext;
use spider_tdl::TdlError;
use spider_tdl::task;

mod commit;
mod compress;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,14 @@ mod spider;
use std::time::Duration;

use async_trait::async_trait;
use clp_rust_utils::{
job_config::CompressionJobId,
task_io::compression::{ClpSCompressionOption, S3InputSource},
};
use serde::{Deserialize, Serialize};
use spider_core::{
task::ExecutionPolicy,
types::id::{JobId, ResourceGroupId},
};
use clp_rust_utils::job_config::CompressionJobId;
use clp_rust_utils::task_io::compression::ClpSCompressionOption;
use clp_rust_utils::task_io::compression::S3InputSource;
use serde::Deserialize;
use serde::Serialize;
use spider_core::task::ExecutionPolicy;
use spider_core::types::id::JobId;
use spider_core::types::id::ResourceGroupId;

use crate::error::Error;

Expand Down
Loading
Loading