Skip to content

Nexus background task for distributing attached subnets #9780

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bnaecker merged 4 commits into from
Feb 5, 2026
Merged

Nexus background task for distributing attached subnets #9780

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5851b43
Add sled-agent API for managing attached subnets
bnaecker Jan 21, 2026
2b032b6
Add Nexus sagas for external subnet attach / detach
bnaecker Jan 27, 2026
8e62a71
Nexus background task for distributing attached subnets
bnaecker Feb 2, 2026
78d0736
Merge branch 'main' into nexus-attached-subnet-bg-task
bnaecker Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions common/src/api/internal/shared/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,13 @@ use crate::{
zpool_name::ZpoolName,
};
use daft::Diffable;
use omicron_uuid_kinds::{
DatasetUuid, ExternalSubnetUuid, InstanceUuid, RackUuid, SledUuid,
};
use omicron_uuid_kinds::{ExternalZpoolUuid, PropolisUuid};
use omicron_uuid_kinds::DatasetUuid;
use omicron_uuid_kinds::ExternalSubnetUuid;
use omicron_uuid_kinds::ExternalZpoolUuid;
use omicron_uuid_kinds::InstanceUuid;
use omicron_uuid_kinds::PropolisUuid;
use omicron_uuid_kinds::RackUuid;
use omicron_uuid_kinds::SledUuid;
use oxnet::{IpNet, Ipv4Net, Ipv6Net};
use schemars::JsonSchema;
use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
Expand Down Expand Up @@ -1106,7 +1109,7 @@ pub enum AttachedSubnetId {
}

/// All details about an attached subnet and the Instance it's attached to.
#[derive(Debug)]
#[derive(Clone, Copy, Debug)]
pub struct AttachedSubnet {
/// ID of the rack hosting this instance.
pub rack_id: RackUuid,
Expand All @@ -1118,7 +1121,7 @@ pub struct AttachedSubnet {
pub vmm_id: PropolisUuid,
/// ID of the instance
pub instance_id: InstanceUuid,
/// ID of the subnet itself.
/// ID of the attached subnet itself.
pub subnet_id: AttachedSubnetId,
/// The IP subnet that's attached.
pub subnet: IpNet,
Expand Down
49 changes: 49 additions & 0 deletions dev-tools/omdb/src/bin/omdb/nexus.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ use nexus_types::deployment::OximeterReadMode;
use nexus_types::deployment::OximeterReadPolicy;
use nexus_types::fm;
use nexus_types::internal_api::background::AbandonedVmmReaperStatus;
use nexus_types::internal_api::background::AttachedSubnetManagerStatus;
use nexus_types::internal_api::background::BlueprintPlannerStatus;
use nexus_types::internal_api::background::BlueprintRendezvousStats;
use nexus_types::internal_api::background::BlueprintRendezvousStatus;
Expand Down Expand Up @@ -1204,6 +1205,9 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) {
"abandoned_vmm_reaper" => {
print_task_abandoned_vmm_reaper(details);
}
"attached_subnet_manager" => {
print_task_attached_subnet_manager_status(details);
}
"blueprint_planner" => {
print_task_blueprint_planner(details);
}
Expand Down Expand Up @@ -2228,6 +2232,51 @@ fn print_task_probe_distributor(details: &serde_json::Value) {
};
}

fn print_task_attached_subnet_manager_status(details: &serde_json::Value) {
match serde_json::from_value::<AttachedSubnetManagerStatus>(details.clone())
{
Err(error) => eprintln!(
"warning: failed to interpret task details: {:?}: {:?}",
error, details
),
Ok(AttachedSubnetManagerStatus { db_error, dendrite, sled }) => {
if let Some(err) = db_error {
println!(
" error accessing database to list attached subnets:"
);
println!(" {err}");
}
if dendrite.is_empty() {
println!(" no dendrite instances found");
} else {
for (loc, details) in dendrite.iter() {
println!(" dendrite instance on switch {loc}");
println!(
" n_subnets_removed={}",
details.n_subnets_removed
);
println!(
" n_subnets_added={}",
details.n_subnets_added
);
println!(
" n_subnets_total={}",
details.n_total_subnets
);
}
}
if sled.is_empty() {
println!(" no sleds found");
} else {
for (sled_id, details) in sled.iter() {
println!(" sled {sled_id}");
println!(" n_subnets={}", details.n_subnets);
}
}
}
};
}

fn print_task_read_only_region_replacement_start(details: &serde_json::Value) {
match serde_json::from_value::<ReadOnlyRegionReplacementStartStatus>(
details.clone(),
Expand Down
12 changes: 12 additions & 0 deletions dev-tools/omdb/tests/env.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ task: "alert_dispatcher"
dispatches queued alerts to receivers


task: "attached_subnet_manager"
distributes attached subnets to sleds and switch


task: "bfd_manager"
Manages bidirectional fowarding detection (BFD) configuration on rack
switches
Expand Down Expand Up @@ -271,6 +275,10 @@ task: "alert_dispatcher"
dispatches queued alerts to receivers


task: "attached_subnet_manager"
distributes attached subnets to sleds and switch


task: "bfd_manager"
Manages bidirectional fowarding detection (BFD) configuration on rack
switches
Expand Down Expand Up @@ -495,6 +503,10 @@ task: "alert_dispatcher"
dispatches queued alerts to receivers


task: "attached_subnet_manager"
distributes attached subnets to sleds and switch


task: "bfd_manager"
Manages bidirectional fowarding detection (BFD) configuration on rack
switches
Expand Down
18 changes: 18 additions & 0 deletions dev-tools/omdb/tests/successes.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,10 @@ task: "alert_dispatcher"
dispatches queued alerts to receivers


task: "attached_subnet_manager"
distributes attached subnets to sleds and switch


task: "bfd_manager"
Manages bidirectional fowarding detection (BFD) configuration on rack
switches
Expand Down Expand Up @@ -573,6 +577,13 @@ task: "alert_dispatcher"
alerts dispatched: 0
alerts with no receivers subscribed: 0

task: "attached_subnet_manager"
configured period: every <REDACTED_DURATION>m
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
no dendrite instances found
no sleds found

task: "bfd_manager"
configured period: every <REDACTED_DURATION>s
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
Expand Down Expand Up @@ -1153,6 +1164,13 @@ task: "alert_dispatcher"
alerts dispatched: 0
alerts with no receivers subscribed: 0

task: "attached_subnet_manager"
configured period: every <REDACTED_DURATION>m
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
no dendrite instances found
no sleds found

task: "bfd_manager"
configured period: every <REDACTED_DURATION>s
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
Expand Down
16 changes: 16 additions & 0 deletions nexus-config/src/nexus_config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,8 @@ pub struct BackgroundTaskConfig {
pub multicast_reconciler: MulticastGroupReconcilerConfig,
/// configuration for trust quorum manager task
pub trust_quorum: TrustQuorumConfig,
/// configuration for the attached subnet manager
pub attached_subnet_manager: AttachedSubnetManagerConfig,
}

#[serde_as]
Expand Down Expand Up @@ -1016,6 +1018,15 @@ pub struct PackageConfig {
pub default_region_allocation_strategy: RegionAllocationStrategy,
}

#[serde_as]
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct AttachedSubnetManagerConfig {
/// period (in seconds) for periodic activations of the background task that
/// pushes attached subnets to the switches and sleds.
#[serde_as(as = "DurationSeconds<u64>")]
pub period_secs: Duration,
}

// Re-export SchemeName from nexus-types for use in config parsing.
pub use nexus_types::authn::SchemeName;

Expand Down Expand Up @@ -1241,6 +1252,7 @@ mod test {
probe_distributor.period_secs = 50
multicast_reconciler.period_secs = 60
trust_quorum.period_secs = 60
attached_subnet_manager.period_secs = 60
[default_region_allocation_strategy]
type = "random"
seed = 0
Expand Down Expand Up @@ -1501,6 +1513,9 @@ mod test {
trust_quorum: TrustQuorumConfig {
period_secs: Duration::from_secs(60),
},
attached_subnet_manager: AttachedSubnetManagerConfig {
period_secs: Duration::from_secs(60),
},
},
multicast: MulticastConfig { enabled: false },
default_region_allocation_strategy:
Expand Down Expand Up @@ -1605,6 +1620,7 @@ mod test {
probe_distributor.period_secs = 47
multicast_reconciler.period_secs = 60
trust_quorum.period_secs = 60
attached_subnet_manager.period_secs = 60

[default_region_allocation_strategy]
type = "random"
Expand Down
1 change: 1 addition & 0 deletions nexus/background-task-interface/src/init.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ pub struct BackgroundTasks {
pub task_probe_distributor: Activator,
pub task_multicast_reconciler: Activator,
pub task_trust_quorum_manager: Activator,
pub task_attached_subnet_manager: Activator,

// Handles to activate background tasks that do not get used by Nexus
// at-large. These background tasks are implementation details as far as
Expand Down
2 changes: 1 addition & 1 deletion nexus/db-model/src/external_subnet.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ use nexus_db_schema::schema::subnet_pool_member;
use nexus_db_schema::schema::subnet_pool_silo_link;
use nexus_types::external_api::params;
use nexus_types::external_api::views;
use nexus_types::identity::Resource;
use nexus_types::identity::Resource as _;
Copy link
Contributor

@rcgoodfellow rcgoodfellow Feb 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hello there. We meet again.

Copy link
Collaborator Author

@bnaecker bnaecker Feb 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, maybe I fucked up the last merge with main before I integrated #9779. This is on main, in any case.

use omicron_common::api::external;
use omicron_common::api::external::Error;
use omicron_uuid_kinds::GenericUuid as _;
Expand Down
1 change: 1 addition & 0 deletions nexus/examples/config-second.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,7 @@ multicast_reconciler.period_secs = 60
# Default: 86400 seconds (24 hours) - refreshed on-demand when validation fails
# multicast_reconciler.backplane_cache_ttl_secs = 86400
trust_quorum.period_secs = 60
attached_subnet_manager.period_secs = 60

[default_region_allocation_strategy]
# allocate region on 3 random distinct zpools, on 3 random distinct sleds.
Expand Down
1 change: 1 addition & 0 deletions nexus/examples/config.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@ multicast_reconciler.period_secs = 60
# Default: 86400 seconds (24 hours) - refreshed on-demand when validation fails
# multicast_reconciler.backplane_cache_ttl_secs = 86400
trust_quorum.period_secs = 60
attached_subnet_manager.period_secs = 60

[default_region_allocation_strategy]
# allocate region on 3 random distinct zpools, on 3 random distinct sleds.
Expand Down
19 changes: 17 additions & 2 deletions nexus/src/app/background/init.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ use super::Driver;
use super::driver::TaskDefinition;
use super::tasks::abandoned_vmm_reaper;
use super::tasks::alert_dispatcher::AlertDispatcher;
use super::tasks::attached_subnets;
use super::tasks::bfd;
use super::tasks::blueprint_execution;
use super::tasks::blueprint_load;
Expand Down Expand Up @@ -264,6 +265,7 @@ impl BackgroundTasksInitializer {
task_probe_distributor: Activator::new(),
task_multicast_reconciler: Activator::new(),
task_trust_quorum_manager: Activator::new(),
task_attached_subnet_manager: Activator::new(),

// Handles to activate background tasks that do not get used by Nexus
// at-large. These background tasks are implementation details as far as
Expand Down Expand Up @@ -353,6 +355,7 @@ impl BackgroundTasksInitializer {
task_probe_distributor,
task_multicast_reconciler,
task_trust_quorum_manager,
task_attached_subnet_manager,
// Add new background tasks here. Be sure to use this binding in a
// call to `Driver::register()` below. That's what actually wires
// up the Activator to the corresponding background task.
Expand Down Expand Up @@ -1091,7 +1094,7 @@ impl BackgroundTasksInitializer {
period: config.sp_ereport_ingester.period_secs,
task_impl: Box::new(ereport_ingester::SpEreportIngester::new(
datastore.clone(),
resolver,
resolver.clone(),
nexus_id,
config.sp_ereport_ingester.disable,
)),
Expand Down Expand Up @@ -1145,13 +1148,25 @@ impl BackgroundTasksInitializer {
description: "Drive trust quorum reconfigurations to completion",
period: config.trust_quorum.period_secs,
task_impl: Box::new(trust_quorum::TrustQuorumManager::new(
datastore,
datastore.clone(),
)),
opctx: opctx.child(BTreeMap::new()),
watchers: vec![],
activator: task_trust_quorum_manager,
});

driver.register(TaskDefinition {
name: "attached_subnet_manager",
description: "distributes attached subnets to sleds and switch",
period: config.attached_subnet_manager.period_secs,
task_impl: Box::new(attached_subnets::Manager::new(
resolver, datastore,
)),
opctx: opctx.child(BTreeMap::new()),
watchers: vec![],
activator: task_attached_subnet_manager,
});

driver
}
}
Expand Down
Loading
Loading