Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions components/spider-scheduler/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@
//! order* and *with what throttling* ready tasks are offered to the fleet.
//!
//! The crate defines three trait seams wired into a single pipeline — a storage client that polls
//! the ready queue, a core that makes serial decisions, and a dispatching queue that fans those
//! the inbound queue, a core that makes serial decisions, and a dispatching queue that fans those
//! decisions out to execution managers:
//!
//! ```text
//! storage ── authoritative ready queue (owned by the storage layer, not this crate)
//! storage ── authoritative inbound queue (owned by the storage layer, not this crate)
//! │
//! │ poll_ready / poll_commit_ready / poll_cleanup_ready (SchedulerStorageClient)
//! ▼
Expand Down
8 changes: 4 additions & 4 deletions components/spider-storage/src/cache/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,17 +79,17 @@ pub enum InternalError {
#[error("task instance pool corrupted: {0}")]
TaskInstancePoolCorrupted(String),

#[error("invalid ready-queue config: {0}")]
ReadyQueueInvalidConfig(&'static str),
#[error("invalid inbound-queue config: {0}")]
InboundQueueInvalidConfig(&'static str),

#[error("invalid task instance pool config: {0}")]
TaskInstancePoolInvalidConfig(&'static str),

#[error("invalid job cache GC config: {0}")]
JobCacheGcInvalidConfig(&'static str),

#[error("ready queue channel is closed")]
ReadyQueueChannelClosed,
#[error("inbound queue channel is closed")]
InboundQueueChannelClosed,

#[error("invalid recoverable job context: {0}")]
InvalidRecoverableJobContext(String),
Expand Down
127 changes: 71 additions & 56 deletions components/spider-storage/src/cache/job.rs

Large diffs are not rendered by default.

54 changes: 27 additions & 27 deletions components/spider-storage/src/grpc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ use tonic::Status;
use crate::cache::error::CacheError;
use crate::db::DbError;
use crate::db::DbStorage;
use crate::ready_queue::ReadyQueueEntry;
use crate::ready_queue::ReadyQueueSender;
use crate::inbound_queue::InboundQueueEntry;
use crate::inbound_queue::InboundQueueSender;
use crate::state::ServiceState;
use crate::state::StorageServerError;
use crate::task_instance_pool::TaskInstancePoolConnector;
Expand All @@ -32,24 +32,24 @@ use crate::task_instance_pool::TaskInstancePoolConnector;
///
/// # Type Parameters
///
/// * `ReadyQueueSenderType` - The ready queue sender type.
/// * `InboundQueueSenderType` - The inbound queue sender type.
/// * `DbConnectorType` - The database connector type.
/// * `TaskInstancePoolConnectorType` - The task instance pool connector type.
#[derive(Clone)]
pub struct GrpcServiceState<
ReadyQueueSenderType: ReadyQueueSender + 'static,
InboundQueueSenderType: InboundQueueSender + 'static,
DbConnectorType: DbStorage + 'static,
TaskInstancePoolConnectorType: TaskInstancePoolConnector + 'static,
> {
inner: ServiceState<ReadyQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>,
inner: ServiceState<InboundQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>,
cancellation_token: CancellationToken,
}

impl<
ReadyQueueSenderType: ReadyQueueSender + 'static,
InboundQueueSenderType: InboundQueueSender + 'static,
DbConnectorType: DbStorage + 'static,
TaskInstancePoolConnectorType: TaskInstancePoolConnector + 'static,
> GrpcServiceState<ReadyQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>
> GrpcServiceState<InboundQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>
{
/// Factory function.
///
Expand All @@ -58,7 +58,7 @@ impl<
/// A new [`GrpcServiceState`] wrapping [`ServiceState`].
#[must_use]
pub const fn new(
inner: ServiceState<ReadyQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>,
inner: ServiceState<InboundQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>,
cancellation_token: CancellationToken,
) -> Self {
Self {
Expand Down Expand Up @@ -420,21 +420,21 @@ impl<
Status::internal("storage service internal error")
}

/// Builds a [`storage::ReadyTasks`] message from a batch of ready-queue entries.
/// Builds a [`storage::ReadyTasks`] message from a batch of inbound-queue entries.
///
/// # Type Parameters
///
/// * `TaskKindType` - The kind of ready-queue task carried by each entry:
/// * `TaskKindType` - The kind of inbound-queue task carried by each entry:
/// * [`spider_core::task::TaskIndex`] for the regular lane.
/// * [`crate::ready_queue::CommitTaskMarker`] for the commit lane.
/// * [`crate::ready_queue::CleanupTaskMarker`] for the cleanup lane.
/// * [`crate::inbound_queue::CommitTaskMarker`] for the commit lane.
/// * [`crate::inbound_queue::CleanupTaskMarker`] for the cleanup lane.
///
/// # Returns
///
/// A [`storage::ReadyTasks`] carrying the storage session and the flattened ready tasks.
fn build_ready_tasks<TaskKindType>(
&self,
entries: Vec<ReadyQueueEntry<TaskKindType>>,
entries: Vec<InboundQueueEntry<TaskKindType>>,
to_task_id: impl Fn(TaskKindType) -> common::TaskId,
) -> storage::ReadyTasks {
let tasks = entries
Expand Down Expand Up @@ -463,11 +463,11 @@ impl<
/// [`GrpcServiceState::job_orchestration_service_error_handler`].
#[async_trait]
impl<
ReadyQueueSenderType: ReadyQueueSender + 'static,
InboundQueueSenderType: InboundQueueSender + 'static,
DbConnectorType: DbStorage + 'static,
TaskInstancePoolConnectorType: TaskInstancePoolConnector + 'static,
> JobOrchestrationService
for GrpcServiceState<ReadyQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>
for GrpcServiceState<InboundQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>
{
async fn register_job(
&self,
Expand Down Expand Up @@ -569,11 +569,11 @@ impl<

#[async_trait]
impl<
ReadyQueueSenderType: ReadyQueueSender + 'static,
InboundQueueSenderType: InboundQueueSender + 'static,
DbConnectorType: DbStorage + 'static,
TaskInstancePoolConnectorType: TaskInstancePoolConnector + 'static,
> TaskInstanceManagementService
for GrpcServiceState<ReadyQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>
for GrpcServiceState<InboundQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>
{
async fn register_task_instance(
&self,
Expand Down Expand Up @@ -683,11 +683,11 @@ impl<

#[async_trait]
impl<
ReadyQueueSenderType: ReadyQueueSender + 'static,
InboundQueueSenderType: InboundQueueSender + 'static,
DbConnectorType: DbStorage + 'static,
TaskInstancePoolConnectorType: TaskInstancePoolConnector + 'static,
> InboundQueueService
for GrpcServiceState<ReadyQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>
for GrpcServiceState<InboundQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>
{
async fn poll_ready_tasks(
&self,
Expand Down Expand Up @@ -746,11 +746,11 @@ impl<

#[async_trait]
impl<
ReadyQueueSenderType: ReadyQueueSender + 'static,
InboundQueueSenderType: InboundQueueSender + 'static,
DbConnectorType: DbStorage + 'static,
TaskInstancePoolConnectorType: TaskInstancePoolConnector + 'static,
> ResourceGroupManagementService
for GrpcServiceState<ReadyQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>
for GrpcServiceState<InboundQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>
{
async fn add_resource_group(
&self,
Expand Down Expand Up @@ -791,11 +791,11 @@ impl<

#[async_trait]
impl<
ReadyQueueSenderType: ReadyQueueSender + 'static,
InboundQueueSenderType: InboundQueueSender + 'static,
DbConnectorType: DbStorage + 'static,
TaskInstancePoolConnectorType: TaskInstancePoolConnector + 'static,
> ExecutionManagerLivenessService
for GrpcServiceState<ReadyQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>
for GrpcServiceState<InboundQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>
{
async fn register_execution_manager(
&self,
Expand Down Expand Up @@ -849,11 +849,11 @@ impl<

#[async_trait]
impl<
ReadyQueueSenderType: ReadyQueueSender + 'static,
InboundQueueSenderType: InboundQueueSender + 'static,
DbConnectorType: DbStorage + 'static,
TaskInstancePoolConnectorType: TaskInstancePoolConnector + 'static,
> SchedulerRegistrationService
for GrpcServiceState<ReadyQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>
for GrpcServiceState<InboundQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>
{
async fn register_scheduler(
&self,
Expand Down Expand Up @@ -886,11 +886,11 @@ impl<

#[async_trait]
impl<
ReadyQueueSenderType: ReadyQueueSender + 'static,
InboundQueueSenderType: InboundQueueSender + 'static,
DbConnectorType: DbStorage + 'static,
TaskInstancePoolConnectorType: TaskInstancePoolConnector + 'static,
> SessionManagementService
for GrpcServiceState<ReadyQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>
for GrpcServiceState<InboundQueueSenderType, DbConnectorType, TaskInstancePoolConnectorType>
{
async fn get_session(
&self,
Expand Down
Loading
Loading