Skip to content

Perform volume and local storage delete in tasks #11027

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jmpesp wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Perform volume and local storage delete in tasks #11027

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dbf6de8
Perform volume and local storage delete in tasks
jmpesp Jul 28, 2026
c50958c
doctest exposed some stale comments
jmpesp Aug 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions dev-tools/omdb/src/bin/omdb/db.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2645,11 +2645,11 @@ async fn cmd_db_disk_info(
datastore: &DataStore,
args: &DiskInfoArgs,
) -> Result<(), anyhow::Error> {
let conn = datastore.pool_connection_for_tests().await?;

let disk = {
use nexus_db_schema::schema::disk::dsl;

let conn = datastore.pool_connection_for_tests().await?;

dsl::disk
.filter(dsl::id.eq(args.uuid))
.select(nexus_db_model::Disk::as_select())
Expand All @@ -2658,7 +2658,7 @@ async fn cmd_db_disk_info(
.context("failed to find disk")?
};

match datastore.disk_get_with_model(opctx, disk).await? {
match datastore.disk_get_with_model(&conn, disk).await? {
Disk::Crucible(disk) => {
crucible_disk_info(opctx, datastore, disk).await
}
Expand Down
83 changes: 83 additions & 0 deletions dev-tools/omdb/src/bin/omdb/nexus.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ use nexus_types::internal_api::background::IncompleteBootstoreConfigReport;
use nexus_types::internal_api::background::InstanceReincarnationStatus;
use nexus_types::internal_api::background::InstanceUpdaterStatus;
use nexus_types::internal_api::background::InventoryLoadStatus;
use nexus_types::internal_api::background::LocalStorageDeleteStatus;
use nexus_types::internal_api::background::LookupRegionPortStatus;
use nexus_types::internal_api::background::PhysicalDiskAdoptionStatus;
use nexus_types::internal_api::background::ProbeDistributorStatus;
Expand All @@ -90,6 +91,7 @@ use nexus_types::internal_api::background::TufArtifactReplicationCounters;
use nexus_types::internal_api::background::TufArtifactReplicationRequest;
use nexus_types::internal_api::background::TufArtifactReplicationStatus;
use nexus_types::internal_api::background::TufRepoPrunerStatus;
use nexus_types::internal_api::background::VolumeDeleteStatus;
use nexus_types::internal_api::background::fm_rendezvous;
use omicron_uuid_kinds::BlueprintUuid;
use omicron_uuid_kinds::CollectionUuid;
Expand Down Expand Up @@ -1400,6 +1402,12 @@ fn print_task_details(
"switch_port_config_manager" => {
print_task_switch_port_settings_manager(details);
}
"volume_delete" => {
print_task_volume_delete(details);
}
"local_storage_delete" => {
print_task_local_storage_delete(details);
}
_ => {
println!(
"warning: unknown background task: {:?} \
Expand Down Expand Up @@ -4198,6 +4206,81 @@ fn print_task_physical_disk_adoption(details: &serde_json::Value) {
}
}

fn print_task_volume_delete(details: &serde_json::Value) {
match serde_json::from_value::<VolumeDeleteStatus>(details.clone()) {
Err(error) => eprintln!(
"warning: failed to interpret task details: {:?}: {:?}",
error, details
),

Ok(status) => {
let VolumeDeleteStatus {
region_results,
running_snapshot_results,
snapshot_results,
volumes_deleted,
errors,
} = &status;

println!(" result of deleting regions:");
for result in region_results {
println!(" > {result}");
}

println!(" result of deleting running snapshots:");
for result in running_snapshot_results {
println!(" > {result}");
}

println!(" result of deleting snapshots:");
for result in snapshot_results {
println!(" > {result}");
}

println!(" volumes deleted:");
for id in volumes_deleted {
println!(" > {id}");
}

println!(" errors: {}", errors.len());
for error in errors {
println!(" > {error}");
}
}
}
}

fn print_task_local_storage_delete(details: &serde_json::Value) {
match serde_json::from_value::<LocalStorageDeleteStatus>(details.clone()) {
Err(error) => eprintln!(
"warning: failed to interpret task details: {:?}: {:?}",
error, details
),

Ok(status) => {
let LocalStorageDeleteStatus {
delete_results,
deallocate_results,
errors,
} = &status;

println!(" result of deleting local storage:");
for result in delete_results {
println!(" > {result}");
}

println!(" result of deallocating local storage:");
for result in deallocate_results {
println!(" > {result}");
}

println!(" errors: {}", errors.len());
for error in errors {
println!(" > {error}");
}
}
}
}
const ERRICON: &str = "/!\\";

fn warn_if_nonzero(n: usize) -> &'static str {
Expand Down
24 changes: 24 additions & 0 deletions dev-tools/omdb/tests/env.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,10 @@ task: "inventory_loader"
loads the latest inventory collection from the DB


task: "local_storage_delete"
delete resources for disks backed by local storage


task: "lookup_region_port"
fill in missing ports for region records

Expand Down Expand Up @@ -268,6 +272,10 @@ task: "v2p_manager"
manages opte v2p mappings for vpc networking


task: "volume_delete"
delete resources from soft-deleted volumes


task: "vpc_route_manager"
propagates updated VPC routes to all OPTE ports

Expand Down Expand Up @@ -428,6 +436,10 @@ task: "inventory_loader"
loads the latest inventory collection from the DB


task: "local_storage_delete"
delete resources for disks backed by local storage


task: "lookup_region_port"
fill in missing ports for region records

Expand Down Expand Up @@ -541,6 +553,10 @@ task: "v2p_manager"
manages opte v2p mappings for vpc networking


task: "volume_delete"
delete resources from soft-deleted volumes


task: "vpc_route_manager"
propagates updated VPC routes to all OPTE ports

Expand Down Expand Up @@ -688,6 +704,10 @@ task: "inventory_loader"
loads the latest inventory collection from the DB


task: "local_storage_delete"
delete resources for disks backed by local storage


task: "lookup_region_port"
fill in missing ports for region records

Expand Down Expand Up @@ -801,6 +821,10 @@ task: "v2p_manager"
manages opte v2p mappings for vpc networking


task: "volume_delete"
delete resources from soft-deleted volumes


task: "vpc_route_manager"
propagates updated VPC routes to all OPTE ports

Expand Down
44 changes: 44 additions & 0 deletions dev-tools/omdb/tests/successes.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,10 @@ task: "inventory_loader"
loads the latest inventory collection from the DB


task: "local_storage_delete"
delete resources for disks backed by local storage


task: "lookup_region_port"
fill in missing ports for region records

Expand Down Expand Up @@ -503,6 +507,10 @@ task: "v2p_manager"
manages opte v2p mappings for vpc networking


task: "volume_delete"
delete resources from soft-deleted volumes


task: "vpc_route_manager"
propagates updated VPC routes to all OPTE ports

Expand Down Expand Up @@ -863,6 +871,14 @@ task: "inventory_loader"
loaded latest inventory collection as of <REDACTED_TIMESTAMP>:
collection ..........<REDACTED_UUID>..........., taken at <REDACTED_TIMESTAMP>

task: "local_storage_delete"
configured period: every <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
result of deleting local storage:
result of deallocating local storage:
errors: 0

task: "lookup_region_port"
configured period: every <REDACTED_DURATION>m
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
Expand Down Expand Up @@ -1096,6 +1112,16 @@ task: "v2p_manager"
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
warning: unknown background task: "v2p_manager" (don't know how to interpret details: Object {})

task: "volume_delete"
configured period: every <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
result of deleting regions:
result of deleting running snapshots:
result of deleting snapshots:
volumes deleted:
errors: 0

task: "vpc_route_manager"
configured period: every <REDACTED_DURATION>s
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
Expand Down Expand Up @@ -1585,6 +1611,14 @@ task: "inventory_loader"
loaded latest inventory collection as of <REDACTED_TIMESTAMP>:
collection ..........<REDACTED_UUID>..........., taken at <REDACTED_TIMESTAMP>

task: "local_storage_delete"
configured period: every <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
result of deleting local storage:
result of deallocating local storage:
errors: 0

task: "lookup_region_port"
configured period: every <REDACTED_DURATION>m
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
Expand Down Expand Up @@ -1818,6 +1852,16 @@ task: "v2p_manager"
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
warning: unknown background task: "v2p_manager" (don't know how to interpret details: Object {})

task: "volume_delete"
configured period: every <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
result of deleting regions:
result of deleting running snapshots:
result of deleting snapshots:
volumes deleted:
errors: 0

task: "vpc_route_manager"
configured period: every <REDACTED_DURATION>s
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
Expand Down
32 changes: 32 additions & 0 deletions nexus-config/src/nexus_config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,10 @@ pub struct BackgroundTaskConfig {
pub audit_log_cleanup: AuditLogCleanupConfig,
/// configuration for populate switch ports task
pub populate_switch_ports: PopulateSwitchPortsConfig,
/// configuration for volume delete task
pub volume_delete: VolumeDeleteConfig,
/// configuration for local storage delete task
pub local_storage_delete: LocalStorageDeleteConfig,
}

#[serde_as]
Expand Down Expand Up @@ -1108,6 +1112,22 @@ pub struct TrustQuorumConfig {
pub period_secs: Duration,
}

#[serde_as]
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct VolumeDeleteConfig {
/// period (in seconds) for periodic activations of this background task
#[serde_as(as = "DurationSeconds<u64>")]
pub period_secs: Duration,
}

#[serde_as]
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct LocalStorageDeleteConfig {
/// period (in seconds) for periodic activations of this background task
#[serde_as(as = "DurationSeconds<u64>")]
pub period_secs: Duration,
}

/// Configuration for a nexus server
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
pub struct PackageConfig {
Expand Down Expand Up @@ -1399,6 +1419,8 @@ mod test {
audit_log_cleanup.retention_days = 90
audit_log_cleanup.max_deleted_per_activation = 10000
populate_switch_ports.period_secs = 31
volume_delete.period_secs = 30
local_storage_delete.period_secs = 30
[default_region_allocation_strategy]
type = "random"
seed = 0
Expand Down Expand Up @@ -1685,6 +1707,14 @@ mod test {
populate_switch_ports: PopulateSwitchPortsConfig {
period_secs: Duration::from_secs(31),
},
volume_delete:
VolumeDeleteConfig {
period_secs: Duration::from_secs(30),
},
local_storage_delete:
LocalStorageDeleteConfig {
period_secs: Duration::from_secs(30),
},
},
multicast: MulticastConfig { enabled: false },
default_region_allocation_strategy:
Expand Down Expand Up @@ -1802,6 +1832,8 @@ mod test {
audit_log_cleanup.retention_days = 90
audit_log_cleanup.max_deleted_per_activation = 10000
populate_switch_ports.period_secs = 31
volume_delete.period_secs = 30
local_storage_delete.period_secs = 30

[default_region_allocation_strategy]
type = "random"
Expand Down
2 changes: 2 additions & 0 deletions nexus/background-task-interface/src/init.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ pub struct BackgroundTasks {
pub task_attached_subnet_manager: Activator,
pub task_session_cleanup: Activator,
pub task_populate_switch_ports: Activator,
pub task_volume_delete: Activator,
pub task_local_storage_delete: Activator,

// Handles to activate background tasks that do not get used by Nexus
// at-large. These background tasks are implementation details as far as
Expand Down
Loading
Loading