From 5851b4300b1be89005cfbe6ef0aad648101fa722 Mon Sep 17 00:00:00 2001 From: Benjamin Naecker Date: Wed, 21 Jan 2026 15:41:34 -0800 Subject: [PATCH 1/3] Add sled-agent API for managing attached subnets - Add APIs to the sled agent for attaching and detaching either a single subnet on an instance, or setting / clearing the entire set for an instance. - Add list of attached subnets in the instance-creation request body, and fill that in from Nexus with the (currently-empty) set of attached subnets for the target instnace. - Plumb attachment requests all the way through the sled-agent internals to the new APIs in OPTE. - Add mapping of attached subnets per-instance to the simulated sled agent for testing. - Fixes #9702 --- common/src/api/internal/shared/mod.rs | 37 +- illumos-utils/src/opte/illumos.rs | 5 + illumos-utils/src/opte/mod.rs | 2 +- illumos-utils/src/opte/non_illumos.rs | 25 + illumos-utils/src/opte/port_manager.rs | 137 + .../src/db/datastore/switch_port.rs | 8 +- .../src/test_util/host_phase_2_test_state.rs | 34 + nexus/src/app/instance.rs | 14 + .../sled-agent/sled-agent-18.0.0-ee9451.json | 10515 ++++++++++++++++ openapi/sled-agent/sled-agent-latest.json | 2 +- sled-agent/api/src/lib.rs | 70 +- sled-agent/src/http_entrypoints.rs | 55 + sled-agent/src/instance.rs | 223 +- sled-agent/src/instance_manager.rs | 155 + sled-agent/src/sim/http_entrypoints.rs | 56 + sled-agent/src/sim/sled_agent.rs | 77 +- sled-agent/src/sled_agent.rs | 62 + sled-agent/types/src/attached_subnet.rs | 5 + sled-agent/types/src/lib.rs | 1 + .../add_attached_subnets/attached_subnet.rs | 44 + .../src/add_attached_subnets/instance.rs | 96 + .../versions/src/add_attached_subnets/mod.rs | 8 + sled-agent/types/versions/src/latest.rs | 10 +- sled-agent/types/versions/src/lib.rs | 2 + 24 files changed, 11623 insertions(+), 20 deletions(-) create mode 100644 openapi/sled-agent/sled-agent-18.0.0-ee9451.json create mode 100644 sled-agent/types/src/attached_subnet.rs create mode 100644 sled-agent/types/versions/src/add_attached_subnets/attached_subnet.rs create mode 100644 sled-agent/types/versions/src/add_attached_subnets/instance.rs create mode 100644 sled-agent/types/versions/src/add_attached_subnets/mod.rs diff --git a/common/src/api/internal/shared/mod.rs b/common/src/api/internal/shared/mod.rs index 039c05a8b77..efa0892fb7a 100644 --- a/common/src/api/internal/shared/mod.rs +++ b/common/src/api/internal/shared/mod.rs @@ -6,13 +6,15 @@ use super::nexus::HostIdentifier; use crate::{ - api::external::{self, BfdMode, ImportExportPolicy, Name, Vni}, + api::external::{self, BfdMode, ImportExportPolicy, MacAddr, Name, Vni}, disk::DatasetName, zpool_name::ZpoolName, }; use daft::Diffable; -use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::ExternalZpoolUuid; +use omicron_uuid_kinds::{ + DatasetUuid, ExternalSubnetUuid, InstanceUuid, RackUuid, SledUuid, +}; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; use schemars::JsonSchema; use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; @@ -1097,6 +1099,37 @@ impl DelegatedZvol { } } +#[derive(Clone, Copy, Debug)] +pub enum AttachedSubnetId { + External(ExternalSubnetUuid), + Vpc(Uuid), +} + +/// All details about an attached subnet and the Instance it's attached to. +pub struct AttachedExternalSubnet { + /// ID of the subnet itself. + pub subnet_id: AttachedSubnetId, + /// ID of the instance + pub instance_id: InstanceUuid, + /// ID of the rack hosting this instance. + // + // NOTE: We do not use this today. It's here for the future when we've + // resolved https://github.com/oxidecomputer/omicron/issues/5201, and so + // we can map this entry to the Dendrite instances only on the rack this + // instance lives on. + pub _rack_id: RackUuid, + /// ID of the sled hosting the instance. + pub sled_id: SledUuid, + /// Underlay IP address of the sled hosting the instance. + pub sled_ip: Ipv6Addr, + /// The IP subnet that's attached. + pub subnet: IpNet, + /// The MAC address of the primary network interface. + pub mac: MacAddr, + /// The VNI of the VPC the instance is in. + pub vni: Vni, +} + #[cfg(test)] mod tests { use super::*; diff --git a/illumos-utils/src/opte/illumos.rs b/illumos-utils/src/opte/illumos.rs index 2cef857393d..1b22311c070 100644 --- a/illumos-utils/src/opte/illumos.rs +++ b/illumos-utils/src/opte/illumos.rs @@ -65,6 +65,11 @@ pub enum Error { #[error("No matching NIC found for port {0} at slot {1}.")] NoNicforPort(String, u32), + + #[error( + "Tried to update attached subnets on non-existent port ({0}, {1:?})" + )] + AttachedSubnetUpdateMissingPort(uuid::Uuid, NetworkInterfaceKind), } /// Delete all xde devices on the system. diff --git a/illumos-utils/src/opte/mod.rs b/illumos-utils/src/opte/mod.rs index e5a8c7058df..34de9e5dfa9 100644 --- a/illumos-utils/src/opte/mod.rs +++ b/illumos-utils/src/opte/mod.rs @@ -112,7 +112,7 @@ impl Gateway { } /// Convert a nexus [IpNet] to an OPTE [IpCidr]. -fn net_to_cidr(net: IpNet) -> IpCidr { +pub fn net_to_cidr(net: IpNet) -> IpCidr { match net { IpNet::V4(net) => IpCidr::Ip4(Ipv4Cidr::new( net.addr().into(), diff --git a/illumos-utils/src/opte/non_illumos.rs b/illumos-utils/src/opte/non_illumos.rs index 00b4e69a048..7724d9497ce 100644 --- a/illumos-utils/src/opte/non_illumos.rs +++ b/illumos-utils/src/opte/non_illumos.rs @@ -9,6 +9,7 @@ use omicron_common::api::internal::shared::NetworkInterfaceKind; use oxide_vpc::api::AddRouterEntryReq; use oxide_vpc::api::ClearVirt2PhysReq; use oxide_vpc::api::DelRouterEntryReq; +use oxide_vpc::api::DetachSubnetResp; use oxide_vpc::api::Direction; use oxide_vpc::api::DumpVirt2PhysResp; use oxide_vpc::api::IpCfg; @@ -58,6 +59,11 @@ pub enum Error { #[error("No matching NIC found for port {0} at slot {1}.")] NoNicforPort(String, u32), + + #[error( + "Tried to update attached subnets on non-existent port ({0}, {1:?})" + )] + AttachedSubnetUpdateMissingPort(uuid::Uuid, NetworkInterfaceKind), } pub fn initialize_xde_driver( @@ -347,4 +353,23 @@ impl Handle { state.underlay_initialized = true; Ok(NO_RESPONSE) } + + /// Attach a subnet. + pub(crate) fn attach_subnet( + &self, + _name: &str, + _subnet: IpCidr, + _is_external: bool, + ) -> Result { + unimplemented!("Not yet used in tests"); + } + + /// Detach a subnet. + pub(crate) fn detach_subnet( + &self, + _name: &str, + _subnet: IpCidr, + ) -> Result { + unimplemented!("Not yet used in tests"); + } } diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index c9f6b10b3d7..a2a62914c2e 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -41,6 +41,7 @@ use omicron_common::api::internal::shared::VirtualNetworkInterfaceHost; use oxide_vpc::api::AddRouterEntryReq; use oxide_vpc::api::AttachedSubnetConfig; use oxide_vpc::api::DelRouterEntryReq; +use oxide_vpc::api::DetachSubnetResp; use oxide_vpc::api::DhcpCfg; use oxide_vpc::api::ExternalIpCfg; use oxide_vpc::api::IpCfg; @@ -63,6 +64,7 @@ use slog::Logger; use slog::debug; use slog::error; use slog::info; +use slog::warn; use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; @@ -993,6 +995,141 @@ impl PortManager { Ok(()) } + + pub fn attached_subnets_ensure( + &self, + nic_id: Uuid, + nic_kind: NetworkInterfaceKind, + ensure_removed: Vec, + ensure_added: Vec, + ) -> Result<(), Error> { + let ports = self.inner.ports.lock().unwrap(); + let port = ports.get(&(nic_id, nic_kind)).ok_or_else(|| { + Error::AttachedSubnetUpdateMissingPort(nic_id, nic_kind) + })?; + self.attached_subnets_ensure_port(port, ensure_removed, ensure_added) + } + + fn attached_subnets_ensure_port( + &self, + port: &Port, + ensure_removed: Vec, + ensure_added: Vec, + ) -> Result<(), Error> { + debug!( + self.inner.log, + "ensuring attached subnets for port"; + "port_name" => %port.name(), + ); + let hdl = Handle::new()?; + for cidr in ensure_removed.into_iter() { + hdl.detach_subnet(port.name(), cidr)?; + } + for subnet in ensure_added.into_iter() { + self.attach_subnet_port(port, subnet)? + } + Ok(()) + } + + pub fn attach_subnet( + &self, + nic_id: Uuid, + nic_kind: NetworkInterfaceKind, + subnet: AttachedSubnet, + ) -> Result<(), Error> { + let ports = self.inner.ports.lock().unwrap(); + let port = ports.get(&(nic_id, nic_kind)).ok_or_else(|| { + Error::AttachedSubnetUpdateMissingPort(nic_id, nic_kind) + })?; + self.attach_subnet_port(port, subnet) + } + + fn attach_subnet_port( + &self, + port: &Port, + subnet: AttachedSubnet, + ) -> Result<(), Error> { + let hdl = Handle::new()?; + let AttachedSubnet { cidr, is_external } = subnet; + match hdl.attach_subnet(port.name(), cidr, is_external) { + Ok(_) => { + debug!( + self.inner.log, + "attached subnet"; + "port_name" => %port.name(), + "subnet" => %cidr, + "is_external" => is_external, + ); + Ok(()) + } + Err(e) => { + error!( + self.inner.log, + "failed to attach subnet"; + "port_name" => %port.name(), + "subnet" => %cidr, + "is_external" => is_external, + "error" => ?e, + ); + Err(Error::from(e)) + } + } + } + + pub fn detach_subnet( + &self, + nic_id: Uuid, + nic_kind: NetworkInterfaceKind, + subnet: IpCidr, + ) -> Result<(), Error> { + let ports = self.inner.ports.lock().unwrap(); + let port = ports.get(&(nic_id, nic_kind)).ok_or_else(|| { + Error::AttachedSubnetUpdateMissingPort(nic_id, nic_kind) + })?; + self.detach_subnet_port(port, subnet) + } + + fn detach_subnet_port( + &self, + port: &Port, + subnet: IpCidr, + ) -> Result<(), Error> { + let hdl = Handle::new()?; + // This returns an Error if the actual request failed. The + // `DetachSubnetResp` it returns in the Ok(_) variant is either + // `NotFound` or `Ok(IpCidr)`, so in both cases we've "detached" it. We + // return success either way. + match hdl.detach_subnet(port.name(), subnet) { + Ok(DetachSubnetResp::Ok(_)) => { + debug!( + self.inner.log, + "detached subnet"; + "port_name" => %port.name(), + "subnet" => %subnet, + ); + Ok(()) + } + Ok(DetachSubnetResp::NotFound) => { + warn!( + self.inner.log, + "subnet is already detached"; + "port_name" => %port.name(), + "subnet" => %subnet, + ); + Ok(()) + } + Err(e) => { + error!( + self.inner.log, + "failed to detach subnet"; + "port_name" => %port.name(), + "subnet" => %subnet, + "error" => ?e, + ); + Err(Error::from(e)) + } + } + } } pub struct PortTicket { diff --git a/nexus/db-queries/src/db/datastore/switch_port.rs b/nexus/db-queries/src/db/datastore/switch_port.rs index f1080f87497..5c0aa3f7901 100644 --- a/nexus/db-queries/src/db/datastore/switch_port.rs +++ b/nexus/db-queries/src/db/datastore/switch_port.rs @@ -977,13 +977,13 @@ impl DataStore { let msg = "failed to check if bgp peer exists in switch port settings"; match e { diesel::result::Error::NotFound => { - debug!(opctx.log, "{msg}"; "error" => ?e); + debug!(opctx.log, "{msg}"; "error" => ?e); Ok(None) }, _ => { - error!(opctx.log, "{msg}"; "error" => ?e); - Err(err.bail(Error::internal_error(msg))) - } + error!(opctx.log, "{msg}"; "error" => ?e); + Err(err.bail(Error::internal_error(msg))) + } } } }?; diff --git a/nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs b/nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs index 6046fb64337..fab9f9efa64 100644 --- a/nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs +++ b/nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs @@ -1053,5 +1053,39 @@ mod api_impl { ) -> Result { unimplemented!() } + + async fn vmm_put_attached_subnets( + _request_context: RequestContext, + _path_params: Path, + _body: TypedBody< + sled_agent_types::attached_subnet::AttachedSubnets, + >, + ) -> Result { + unimplemented!() + } + + async fn vmm_delete_attached_subnets( + _request_context: RequestContext, + _path_params: Path, + ) -> Result { + unimplemented!() + } + + async fn vmm_post_attached_subnet( + _request_context: RequestContext, + _path_params: Path, + _body: TypedBody, + ) -> Result { + unimplemented!() + } + + async fn vmm_delete_attached_subnet( + _request_context: RequestContext, + _path_params: Path< + sled_agent_types::attached_subnet::VmmSubnetPathParam, + >, + ) -> Result { + unimplemented!() + } } } diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index 9f95ed1aaaf..5b2575d63e7 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -1597,6 +1597,19 @@ impl super::Nexus { } } + // TODO-completeness: We need to handle VPC subnets too, see + // https://github.com/oxidecomputer/omicron/issues/9580. + let attached_subnets = self + .datastore() + .instance_lookup_external_subnets(opctx, authz_instance) + .await? + .into_iter() + .map(|ext| sled_agent_client::types::AttachedSubnet { + subnet: ext.subnet.into(), + is_external: true, + }) + .collect(); + let local_config = sled_agent_client::types::InstanceSledLocalConfig { hostname, nics, @@ -1610,6 +1623,7 @@ impl super::Nexus { search_domains: Vec::new(), }, delegated_zvols, + attached_subnets, }; let instance_id = InstanceUuid::from_untyped_uuid(db_instance.id()); diff --git a/openapi/sled-agent/sled-agent-18.0.0-ee9451.json b/openapi/sled-agent/sled-agent-18.0.0-ee9451.json new file mode 100644 index 00000000000..4aea76020c7 --- /dev/null +++ b/openapi/sled-agent/sled-agent-18.0.0-ee9451.json @@ -0,0 +1,10515 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "Oxide Sled Agent API", + "description": "API for interacting with individual sleds", + "contact": { + "url": "https://oxide.computer", + "email": "api@oxide.computer" + }, + "version": "18.0.0" + }, + "paths": { + "/artifacts": { + "get": { + "operationId": "artifact_list", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ArtifactListResponse" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/artifacts/{sha256}": { + "put": { + "operationId": "artifact_put", + "parameters": [ + { + "in": "path", + "name": "sha256", + "required": true, + "schema": { + "type": "string", + "format": "hex string (32 bytes)" + } + }, + { + "in": "query", + "name": "generation", + "required": true, + "schema": { + "$ref": "#/components/schemas/Generation" + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ArtifactPutResponse" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/artifacts/{sha256}/copy-from-depot": { + "post": { + "operationId": "artifact_copy_from_depot", + "parameters": [ + { + "in": "path", + "name": "sha256", + "required": true, + "schema": { + "type": "string", + "format": "hex string (32 bytes)" + } + }, + { + "in": "query", + "name": "generation", + "required": true, + "schema": { + "$ref": "#/components/schemas/Generation" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ArtifactCopyFromDepotBody" + } + } + }, + "required": true + }, + "responses": { + "202": { + "description": "successfully enqueued operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ArtifactCopyFromDepotResponse" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/artifacts-config": { + "get": { + "operationId": "artifact_config_get", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ArtifactConfig" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "operationId": "artifact_config_put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ArtifactConfig" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/bootstore/status": { + "get": { + "summary": "Get the internal state of the local bootstore node", + "operationId": "bootstore_status", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BootstoreStatus" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/debug/switch-zone-policy": { + "get": { + "summary": "A debugging endpoint only used by `omdb` that allows us to test", + "description": "restarting the switch zone without restarting sled-agent. See for context.", + "operationId": "debug_operator_switch_zone_policy_get", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/OperatorSwitchZonePolicy" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "summary": "A debugging endpoint only used by `omdb` that allows us to test", + "description": "restarting the switch zone without restarting sled-agent. See for context.\n\nSetting the switch zone policy is asynchronous and inherently racy with the standard process of starting the switch zone. If the switch zone is in the process of being started or stopped when this policy is changed, the new policy may not take effect until that transition completes.", + "operationId": "debug_operator_switch_zone_policy_put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/OperatorSwitchZonePolicy" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/disks/{disk_id}": { + "put": { + "operationId": "disk_put", + "parameters": [ + { + "in": "path", + "name": "disk_id", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DiskEnsureBody" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DiskRuntimeState" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/eip-gateways": { + "put": { + "summary": "Update per-NIC IP address <-> internet gateway mappings.", + "operationId": "set_eip_gateways", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExternalIpGatewayMap" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/inventory": { + "get": { + "summary": "Fetch basic information about this sled", + "operationId": "inventory", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Inventory" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/local-storage": { + "post": { + "summary": "Create a local storage dataset", + "operationId": "local_storage_dataset_ensure", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LocalStorageDatasetEnsureRequest" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "summary": "Delete a local storage dataset", + "operationId": "local_storage_dataset_delete", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LocalStorageDatasetDeleteRequest" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/network-bootstore-config": { + "get": { + "summary": "This API endpoint is only reading the local sled agent's view of the", + "description": "bootstore. The boostore is a distributed data store that is eventually consistent. Reads from individual nodes may not represent the latest state.", + "operationId": "read_network_bootstore_config_cache", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EarlyNetworkConfig" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "operationId": "write_network_bootstore_config", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EarlyNetworkConfig" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/omicron-config": { + "put": { + "operationId": "omicron_config_put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/OmicronSledConfig" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/probes": { + "put": { + "summary": "Update the entire set of probe zones on this sled.", + "description": "Probe zones are used to debug networking configuration. They look similar to instances, in that they have an OPTE port on a VPC subnet and external addresses, but no actual VM.", + "operationId": "probes_put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProbeSet" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/sled-identifiers": { + "get": { + "summary": "Fetch sled identifiers", + "operationId": "sled_identifiers", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledIdentifiers" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/sleds": { + "put": { + "summary": "Add a sled to a rack that was already initialized via RSS", + "operationId": "sled_add", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AddSledRequest" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/dladm-info": { + "get": { + "operationId": "support_dladm_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_SledDiagnosticsQueryOutput", + "type": "array", + "items": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/health-check": { + "get": { + "operationId": "support_health_check", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_SledDiagnosticsQueryOutput", + "type": "array", + "items": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/ipadm-info": { + "get": { + "operationId": "support_ipadm_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_SledDiagnosticsQueryOutput", + "type": "array", + "items": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/logs/download/{zone}": { + "get": { + "summary": "This endpoint returns a zip file of a zone's logs organized by service.", + "operationId": "support_logs_download", + "parameters": [ + { + "in": "path", + "name": "zone", + "description": "The zone for which one would like to collect logs for", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "max_rotated", + "description": "The max number of rotated logs to include in the final support bundle", + "required": true, + "schema": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + } + ], + "responses": { + "default": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + } + } + } + }, + "/support/logs/zones": { + "get": { + "summary": "This endpoint returns a list of known zones on a sled that have service", + "description": "logs that can be collected into a support bundle.", + "operationId": "support_logs", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_String", + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/nvmeadm-info": { + "get": { + "operationId": "support_nvmeadm_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/pargs-info": { + "get": { + "operationId": "support_pargs_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_SledDiagnosticsQueryOutput", + "type": "array", + "items": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/pfiles-info": { + "get": { + "operationId": "support_pfiles_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_SledDiagnosticsQueryOutput", + "type": "array", + "items": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/pstack-info": { + "get": { + "operationId": "support_pstack_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_SledDiagnosticsQueryOutput", + "type": "array", + "items": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/zfs-info": { + "get": { + "operationId": "support_zfs_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/zoneadm-info": { + "get": { + "operationId": "support_zoneadm_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/zpool-info": { + "get": { + "operationId": "support_zpool_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support-bundles/{zpool_id}/{dataset_id}": { + "get": { + "summary": "List all support bundles within a particular dataset", + "operationId": "support_bundle_list", + "parameters": [ + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_SupportBundleMetadata", + "type": "array", + "items": { + "$ref": "#/components/schemas/SupportBundleMetadata" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}": { + "post": { + "summary": "Starts creation of a support bundle within a particular dataset", + "description": "Callers should transfer chunks of the bundle with \"support_bundle_transfer\", and then call \"support_bundle_finalize\" once the bundle has finished transferring.\n\nIf a support bundle was previously created without being finalized successfully, this endpoint will reset the state.\n\nIf a support bundle was previously created and finalized successfully, this endpoint will return metadata indicating that it already exists.", + "operationId": "support_bundle_start_creation", + "parameters": [ + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SupportBundleMetadata" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "summary": "Delete a support bundle from a particular dataset", + "operationId": "support_bundle_delete", + "parameters": [ + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/download": { + "get": { + "summary": "Fetch a support bundle from a particular dataset", + "operationId": "support_bundle_download", + "parameters": [ + { + "in": "header", + "name": "range", + "description": "A request to access a portion of the resource, such as `bytes=0-499`\n\nSee: ", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "default": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + } + } + }, + "head": { + "summary": "Fetch metadata about a support bundle from a particular dataset", + "operationId": "support_bundle_head", + "parameters": [ + { + "in": "header", + "name": "range", + "description": "A request to access a portion of the resource, such as `bytes=0-499`\n\nSee: ", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "default": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + } + } + } + }, + "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/download/{file}": { + "get": { + "summary": "Fetch a file within a support bundle from a particular dataset", + "operationId": "support_bundle_download_file", + "parameters": [ + { + "in": "header", + "name": "range", + "description": "A request to access a portion of the resource, such as `bytes=0-499`\n\nSee: ", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "file", + "description": "The path of the file within the support bundle to query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "default": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + } + } + }, + "head": { + "summary": "Fetch metadata about a file within a support bundle from a particular dataset", + "operationId": "support_bundle_head_file", + "parameters": [ + { + "in": "header", + "name": "range", + "description": "A request to access a portion of the resource, such as `bytes=0-499`\n\nSee: ", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "file", + "description": "The path of the file within the support bundle to query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "default": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + } + } + } + }, + "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/finalize": { + "post": { + "summary": "Finalizes the creation of a support bundle", + "description": "If the requested hash matched the bundle, the bundle is created. Otherwise, an error is returned.", + "operationId": "support_bundle_finalize", + "parameters": [ + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + }, + { + "in": "query", + "name": "hash", + "required": true, + "schema": { + "type": "string", + "format": "hex string (32 bytes)" + } + } + ], + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SupportBundleMetadata" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/index": { + "get": { + "summary": "Fetch the index (list of files within a support bundle)", + "operationId": "support_bundle_index", + "parameters": [ + { + "in": "header", + "name": "range", + "description": "A request to access a portion of the resource, such as `bytes=0-499`\n\nSee: ", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "default": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + } + } + }, + "head": { + "summary": "Fetch metadata about the list of files within a support bundle", + "operationId": "support_bundle_head_index", + "parameters": [ + { + "in": "header", + "name": "range", + "description": "A request to access a portion of the resource, such as `bytes=0-499`\n\nSee: ", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "default": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + } + } + } + }, + "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/transfer": { + "put": { + "summary": "Transfers a chunk of a support bundle within a particular dataset", + "operationId": "support_bundle_transfer", + "parameters": [ + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + }, + { + "in": "query", + "name": "offset", + "required": true, + "schema": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SupportBundleMetadata" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/switch-ports": { + "post": { + "operationId": "uplink_ensure", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SwitchPorts" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/trust-quorum/commit": { + "put": { + "summary": "Commit a trust quorum configuration", + "operationId": "trust_quorum_commit", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CommitRequest" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/trust-quorum/configuration": { + "post": { + "summary": "Initiate a trust quorum reconfiguration", + "operationId": "trust_quorum_reconfigure", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ReconfigureMsg" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/trust-quorum/coordinator-status": { + "get": { + "summary": "Get the coordinator status if this node is coordinating a reconfiguration", + "operationId": "trust_quorum_coordinator_status", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CoordinatorStatus" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/trust-quorum/network-config": { + "get": { + "summary": "Get the current network config from trust quorum", + "operationId": "trust_quorum_network_config_get", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TrustQuorumNetworkConfig" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "summary": "Update the network config in trust quorum", + "operationId": "trust_quorum_network_config_put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TrustQuorumNetworkConfig" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/trust-quorum/prepare-and-commit": { + "put": { + "summary": "Attempt to prepare and commit a trust quorum configuration", + "operationId": "trust_quorum_prepare_and_commit", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PrepareAndCommitRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CommitStatus" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/trust-quorum/proxy/commit": { + "put": { + "summary": "Proxy a commit operation to another trust quorum node", + "operationId": "trust_quorum_proxy_commit", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProxyCommitRequest" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/trust-quorum/proxy/prepare-and-commit": { + "put": { + "summary": "Proxy a prepare-and-commit operation to another trust quorum node", + "operationId": "trust_quorum_proxy_prepare_and_commit", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProxyPrepareAndCommitRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CommitStatus" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/trust-quorum/proxy/status": { + "get": { + "summary": "Proxy a status request to another trust quorum node", + "operationId": "trust_quorum_proxy_status", + "parameters": [ + { + "in": "query", + "name": "part_number", + "description": "Oxide Part Number", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "serial_number", + "description": "Serial number (unique for a given part number)", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NodeStatus" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/trust-quorum/status": { + "get": { + "summary": "Get the status of this trust quorum node", + "operationId": "trust_quorum_status", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NodeStatus" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/trust-quorum/upgrade": { + "post": { + "summary": "Initiate an upgrade from LRTQ", + "operationId": "trust_quorum_upgrade_from_lrtq", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LrtqUpgradeMsg" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v2p": { + "get": { + "summary": "List v2p mappings present on sled", + "operationId": "list_v2p", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_VirtualNetworkInterfaceHost", + "type": "array", + "items": { + "$ref": "#/components/schemas/VirtualNetworkInterfaceHost" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "summary": "Create a mapping from a virtual NIC to a physical host", + "operationId": "set_v2p", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VirtualNetworkInterfaceHost" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "summary": "Delete a mapping from a virtual NIC to a physical host", + "operationId": "del_v2p", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VirtualNetworkInterfaceHost" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vmms/{propolis_id}": { + "put": { + "operationId": "vmm_register", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceEnsureBody" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledVmmState" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "operationId": "vmm_unregister", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VmmUnregisterResponse" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vmms/{propolis_id}/attached-subnets": { + "put": { + "summary": "Update the subnets attached to an instance.", + "operationId": "vmm_put_attached_subnets", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AttachedSubnets" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "summary": "Attach a subnet to an instance.", + "operationId": "vmm_post_attached_subnet", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AttachedSubnet" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "summary": "Delete all subnets attached to an instance.", + "operationId": "vmm_delete_attached_subnets", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vmms/{propolis_id}/attached-subnets/{subnet}": { + "delete": { + "summary": "Detach a subnet from an instance.", + "operationId": "vmm_delete_attached_subnet", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + }, + { + "in": "path", + "name": "subnet", + "required": true, + "schema": { + "$ref": "#/components/schemas/IpNet" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vmms/{propolis_id}/disks/{disk_id}/snapshot": { + "post": { + "summary": "Take a snapshot of a disk that is attached to an instance", + "operationId": "vmm_issue_disk_snapshot_request", + "parameters": [ + { + "in": "path", + "name": "disk_id", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VmmIssueDiskSnapshotRequestBody" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VmmIssueDiskSnapshotRequestResponse" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vmms/{propolis_id}/external-ip": { + "put": { + "operationId": "vmm_put_external_ip", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceExternalIpBody" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "operationId": "vmm_delete_external_ip", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceExternalIpBody" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vmms/{propolis_id}/multicast-group": { + "put": { + "operationId": "vmm_join_multicast_group", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceMulticastBody" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "operationId": "vmm_leave_multicast_group", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceMulticastBody" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vmms/{propolis_id}/state": { + "get": { + "operationId": "vmm_get_state", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledVmmState" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "operationId": "vmm_put_state", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VmmPutStateBody" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VmmPutStateResponse" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vpc/{vpc_id}/firewall/rules": { + "put": { + "operationId": "vpc_firewall_rules_put", + "parameters": [ + { + "in": "path", + "name": "vpc_id", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcFirewallRulesEnsureBody" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vpc-routes": { + "get": { + "summary": "Get the current versions of VPC routing rules.", + "operationId": "list_vpc_routes", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_ResolvedVpcRouteState", + "type": "array", + "items": { + "$ref": "#/components/schemas/ResolvedVpcRouteState" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "summary": "Update VPC routing rules.", + "operationId": "set_vpc_routes", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Array_of_ResolvedVpcRouteSet", + "type": "array", + "items": { + "$ref": "#/components/schemas/ResolvedVpcRouteSet" + } + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/zones": { + "get": { + "summary": "List the zones that are currently managed by the sled agent.", + "operationId": "zones_list", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_String", + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/zones/bundle-cleanup": { + "post": { + "summary": "Trigger a zone bundle cleanup.", + "operationId": "zone_bundle_cleanup", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Map_of_CleanupCount", + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/CleanupCount" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/zones/bundle-cleanup/context": { + "get": { + "summary": "Return context used by the zone-bundle cleanup task.", + "operationId": "zone_bundle_cleanup_context", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CleanupContext" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "summary": "Update context used by the zone-bundle cleanup task.", + "operationId": "zone_bundle_cleanup_context_update", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CleanupContextUpdate" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/zones/bundle-cleanup/utilization": { + "get": { + "summary": "Return utilization information about all zone bundles.", + "operationId": "zone_bundle_utilization", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Map_of_BundleUtilization", + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/BundleUtilization" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/zones/bundles": { + "get": { + "summary": "List all zone bundles that exist, even for now-deleted zones.", + "operationId": "zone_bundle_list_all", + "parameters": [ + { + "in": "query", + "name": "filter", + "description": "An optional substring used to filter zone bundles.", + "schema": { + "nullable": true, + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_ZoneBundleMetadata", + "type": "array", + "items": { + "$ref": "#/components/schemas/ZoneBundleMetadata" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/zones/bundles/{zone_name}": { + "get": { + "summary": "List the zone bundles that are available for a running zone.", + "operationId": "zone_bundle_list", + "parameters": [ + { + "in": "path", + "name": "zone_name", + "description": "The name of the zone.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_ZoneBundleMetadata", + "type": "array", + "items": { + "$ref": "#/components/schemas/ZoneBundleMetadata" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/zones/bundles/{zone_name}/{bundle_id}": { + "get": { + "summary": "Fetch the binary content of a single zone bundle.", + "operationId": "zone_bundle_get", + "parameters": [ + { + "in": "path", + "name": "bundle_id", + "description": "The ID for this bundle itself.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "path", + "name": "zone_name", + "description": "The name of the zone this bundle is derived from.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "summary": "Delete a zone bundle.", + "operationId": "zone_bundle_delete", + "parameters": [ + { + "in": "path", + "name": "bundle_id", + "description": "The ID for this bundle itself.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "path", + "name": "zone_name", + "description": "The name of the zone this bundle is derived from.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + } + }, + "components": { + "schemas": { + "AddSledRequest": { + "description": "A request to Add a given sled after rack initialization has occurred", + "type": "object", + "properties": { + "sled_id": { + "$ref": "#/components/schemas/BaseboardId" + }, + "start_request": { + "$ref": "#/components/schemas/StartSledAgentRequest" + } + }, + "required": [ + "sled_id", + "start_request" + ] + }, + "Alarm": { + "description": "An alarm indicating a protocol invariant violation.", + "oneOf": [ + { + "description": "Different configurations found for the same epoch.\n\nReason: Nexus creates configurations and stores them in CRDB before sending them to a coordinator of its choosing. Nexus will not send the same reconfiguration request to different coordinators. If it does those coordinators will generate different key shares. However, since Nexus will not tell different nodes to coordinate the same configuration, this state should be impossible to reach.", + "type": "object", + "properties": { + "mismatched_configurations": { + "type": "object", + "properties": { + "config1": { + "$ref": "#/components/schemas/Configuration" + }, + "config2": { + "$ref": "#/components/schemas/Configuration" + }, + "from": { + "description": "Either a stringified `BaseboardId` or \"Nexus\".", + "type": "string" + } + }, + "required": [ + "config1", + "config2", + "from" + ] + } + }, + "required": [ + "mismatched_configurations" + ], + "additionalProperties": false + }, + { + "description": "The `keyShareComputer` could not compute this node's share.\n\nReason: A threshold of valid key shares were received based on the the share digests in the Configuration. However, computation of the share still failed. This should be impossible.", + "type": "object", + "properties": { + "share_computation_failed": { + "type": "object", + "properties": { + "epoch": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "err": { + "$ref": "#/components/schemas/CombineError" + } + }, + "required": [ + "epoch", + "err" + ] + } + }, + "required": [ + "share_computation_failed" + ], + "additionalProperties": false + }, + { + "description": "We started collecting shares for a committed configuration, but we no longer have that configuration in our persistent state.", + "type": "object", + "properties": { + "committed_configuration_lost": { + "type": "object", + "properties": { + "collecting_epoch": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "latest_committed_epoch": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "collecting_epoch", + "latest_committed_epoch" + ] + } + }, + "required": [ + "committed_configuration_lost" + ], + "additionalProperties": false + }, + { + "description": "Decrypting the encrypted rack secrets failed when presented with a `valid` RackSecret.\n\n`Configuration` membership contains the hashes of each valid share. All shares utilized to reconstruct the rack secret were validated against these hashes, and the rack secret was reconstructed. However, using the rack secret to derive encryption keys and decrypt the secrets from old configurations still failed. This should never be possible, and therefore we raise an alarm.", + "type": "object", + "properties": { + "rack_secret_decryption_failed": { + "type": "object", + "properties": { + "epoch": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "err": { + "$ref": "#/components/schemas/DecryptionError" + } + }, + "required": [ + "epoch", + "err" + ] + } + }, + "required": [ + "rack_secret_decryption_failed" + ], + "additionalProperties": false + }, + { + "description": "Reconstructing the rack secret failed when presented with `valid` shares.\n\n`Configuration` membership contains the hashes of each valid share. All shares utilized to reconstruct the rack secret were validated against these hashes, and yet, the reconstruction still failed. This indicates either a bit flip in a share after validation, or, more likely, an invalid hash.", + "type": "object", + "properties": { + "rack_secret_reconstruction_failed": { + "type": "object", + "properties": { + "epoch": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "err": { + "$ref": "#/components/schemas/RackSecretReconstructError" + } + }, + "required": [ + "epoch", + "err" + ] + } + }, + "required": [ + "rack_secret_reconstruction_failed" + ], + "additionalProperties": false + } + ] + }, + "ArtifactConfig": { + "description": "Artifact configuration.\n\nThis type is used in both GET (response) and PUT (request) operations.", + "type": "object", + "properties": { + "artifacts": { + "type": "array", + "items": { + "type": "string", + "format": "hex string (32 bytes)" + }, + "uniqueItems": true + }, + "generation": { + "$ref": "#/components/schemas/Generation" + } + }, + "required": [ + "artifacts", + "generation" + ] + }, + "ArtifactCopyFromDepotBody": { + "description": "Request body for copying artifacts from a depot.", + "type": "object", + "properties": { + "depot_base_url": { + "type": "string" + } + }, + "required": [ + "depot_base_url" + ] + }, + "ArtifactCopyFromDepotResponse": { + "description": "Response for copying artifacts from a depot.", + "type": "object" + }, + "ArtifactListResponse": { + "description": "Response for listing artifacts.", + "type": "object", + "properties": { + "generation": { + "$ref": "#/components/schemas/Generation" + }, + "list": { + "type": "object", + "additionalProperties": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + } + }, + "required": [ + "generation", + "list" + ] + }, + "ArtifactPutResponse": { + "description": "Response for putting an artifact.", + "type": "object", + "properties": { + "datasets": { + "description": "The number of valid M.2 artifact datasets we found on the sled. There is typically one of these datasets for each functional M.2.", + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "successful_writes": { + "description": "The number of valid writes to the M.2 artifact datasets. This should be less than or equal to the number of artifact datasets.", + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "datasets", + "successful_writes" + ] + }, + "AttachedSubnet": { + "description": "A subnet attached to a single instance.", + "type": "object", + "properties": { + "is_external": { + "description": "Is this is a subnet in the external customer network.\n\nIf false, this is a VPC Subnet attached to the instance.", + "type": "boolean" + }, + "subnet": { + "description": "The IP subnet.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + } + }, + "required": [ + "is_external", + "subnet" + ] + }, + "AttachedSubnets": { + "description": "Subnets attached to a single instance.", + "type": "object", + "properties": { + "subnets": { + "title": "IdOrdMap", + "x-rust-type": { + "crate": "iddqd", + "parameters": [ + { + "$ref": "#/components/schemas/AttachedSubnet" + } + ], + "path": "iddqd::IdOrdMap", + "version": "*" + }, + "type": "array", + "items": { + "$ref": "#/components/schemas/AttachedSubnet" + }, + "uniqueItems": true + } + }, + "required": [ + "subnets" + ] + }, + "Baseboard": { + "description": "Describes properties that should uniquely identify a Gimlet.", + "oneOf": [ + { + "type": "object", + "properties": { + "identifier": { + "type": "string" + }, + "model": { + "type": "string" + }, + "revision": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "gimlet" + ] + } + }, + "required": [ + "identifier", + "model", + "revision", + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "unknown" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "identifier": { + "type": "string" + }, + "model": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "pc" + ] + } + }, + "required": [ + "identifier", + "model", + "type" + ] + } + ] + }, + "BaseboardId": { + "description": "A representation of a Baseboard ID as used in the inventory subsystem.\n\nThis type is essentially the same as a `Baseboard` except it doesn't have a revision or HW type (Gimlet, PC, Unknown).", + "type": "object", + "properties": { + "part_number": { + "description": "Oxide Part Number", + "type": "string" + }, + "serial_number": { + "description": "Serial number (unique for a given part number)", + "type": "string" + } + }, + "required": [ + "part_number", + "serial_number" + ] + }, + "BfdMode": { + "description": "BFD connection mode.", + "type": "string", + "enum": [ + "single_hop", + "multi_hop" + ] + }, + "BfdPeerConfig": { + "type": "object", + "properties": { + "detection_threshold": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "local": { + "nullable": true, + "type": "string", + "format": "ip" + }, + "mode": { + "$ref": "#/components/schemas/BfdMode" + }, + "remote": { + "type": "string", + "format": "ip" + }, + "required_rx": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "switch": { + "$ref": "#/components/schemas/SwitchLocation" + } + }, + "required": [ + "detection_threshold", + "mode", + "remote", + "required_rx", + "switch" + ] + }, + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "checker": { + "nullable": true, + "description": "Checker to apply to incoming messages.", + "default": null, + "type": "string" + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Net" + } + }, + "shaper": { + "nullable": true, + "description": "Shaper to apply to outgoing messages.", + "default": null, + "type": "string" + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "allowed_export": { + "description": "Define export policy for a peer.", + "default": { + "type": "no_filtering" + }, + "allOf": [ + { + "$ref": "#/components/schemas/ImportExportPolicy" + } + ] + }, + "allowed_import": { + "description": "Define import policy for a peer.", + "default": { + "type": "no_filtering" + }, + "allOf": [ + { + "$ref": "#/components/schemas/ImportExportPolicy" + } + ] + }, + "asn": { + "description": "The autonomous system number of the router the peer belongs to.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "communities": { + "description": "Include the provided communities in updates sent to the peer.", + "default": [], + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "connect_retry": { + "nullable": true, + "description": "The interval in seconds between peer connection retry attempts.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "delay_open": { + "nullable": true, + "description": "How long to delay sending open messages to a peer. In seconds.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "enforce_first_as": { + "description": "Enforce that the first AS in paths received from this peer is the peer's AS.", + "default": false, + "type": "boolean" + }, + "hold_time": { + "nullable": true, + "description": "How long to keep a session alive without a keepalive in seconds. Defaults to 6.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "idle_hold_time": { + "nullable": true, + "description": "How long to keep a peer in idle after a state machine reset in seconds.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "keepalive": { + "nullable": true, + "description": "The interval to send keepalive messages at.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "local_pref": { + "nullable": true, + "description": "Apply a local preference to routes received from this peer.", + "default": null, + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "md5_auth_key": { + "nullable": true, + "description": "Use the given key for TCP-MD5 authentication with the peer.", + "default": null, + "type": "string" + }, + "min_ttl": { + "nullable": true, + "description": "Require messages from a peer have a minimum IP time to live field.", + "default": null, + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "multi_exit_discriminator": { + "nullable": true, + "description": "Apply the provided multi-exit discriminator (MED) updates sent to the peer.", + "default": null, + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + }, + "remote_asn": { + "nullable": true, + "description": "Require that a peer has a specified ASN.", + "default": null, + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "vlan_id": { + "nullable": true, + "description": "Associate a VLAN ID with a BGP peer session.", + "default": null, + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, + "BlobStorageBackend": { + "description": "A storage backend for a disk whose initial contents are given explicitly by the specification.", + "type": "object", + "properties": { + "base64": { + "description": "The disk's initial contents, encoded as a base64 string.", + "type": "string" + }, + "readonly": { + "description": "Indicates whether the storage is read-only.", + "type": "boolean" + } + }, + "required": [ + "base64", + "readonly" + ], + "additionalProperties": false + }, + "Board": { + "description": "A VM's mainboard.", + "type": "object", + "properties": { + "chipset": { + "description": "The chipset to expose to guest software.", + "allOf": [ + { + "$ref": "#/components/schemas/Chipset" + } + ] + }, + "cpuid": { + "nullable": true, + "description": "The CPUID values to expose to the guest. If `None`, bhyve will derive default values from the host's CPUID values.", + "allOf": [ + { + "$ref": "#/components/schemas/Cpuid" + } + ] + }, + "cpus": { + "description": "The number of virtual logical processors attached to this VM.", + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "guest_hv_interface": { + "description": "The hypervisor platform to expose to the guest. The default is a bhyve-compatible interface with no additional features.\n\nFor compatibility with older versions of Propolis, this field is only serialized if it specifies a non-default interface.", + "allOf": [ + { + "$ref": "#/components/schemas/GuestHypervisorInterface" + } + ] + }, + "memory_mb": { + "description": "The amount of guest RAM attached to this VM.", + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "chipset", + "cpus", + "memory_mb" + ], + "additionalProperties": false + }, + "BootImageHeader": { + "type": "object", + "properties": { + "data_size": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "flags": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "image_name": { + "type": "string" + }, + "image_size": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "sha256": { + "type": "array", + "items": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "minItems": 32, + "maxItems": 32 + }, + "target_size": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "data_size", + "flags", + "image_name", + "image_size", + "sha256", + "target_size" + ] + }, + "BootOrderEntry": { + "description": "An entry in the boot order stored in a [`BootSettings`] component.", + "type": "object", + "properties": { + "id": { + "description": "The ID of another component in the spec that Propolis should try to boot from.\n\nCurrently, only disk device components are supported.", + "allOf": [ + { + "$ref": "#/components/schemas/SpecKey" + } + ] + } + }, + "required": [ + "id" + ] + }, + "BootPartitionContents": { + "type": "object", + "properties": { + "boot_disk": { + "x-rust-type": { + "crate": "std", + "parameters": [ + { + "$ref": "#/components/schemas/M2Slot" + }, + { + "type": "string" + } + ], + "path": "::std::result::Result", + "version": "*" + }, + "oneOf": [ + { + "type": "object", + "properties": { + "ok": { + "$ref": "#/components/schemas/M2Slot" + } + }, + "required": [ + "ok" + ] + }, + { + "type": "object", + "properties": { + "err": { + "type": "string" + } + }, + "required": [ + "err" + ] + } + ] + }, + "slot_a": { + "x-rust-type": { + "crate": "std", + "parameters": [ + { + "$ref": "#/components/schemas/BootPartitionDetails" + }, + { + "type": "string" + } + ], + "path": "::std::result::Result", + "version": "*" + }, + "oneOf": [ + { + "type": "object", + "properties": { + "ok": { + "$ref": "#/components/schemas/BootPartitionDetails" + } + }, + "required": [ + "ok" + ] + }, + { + "type": "object", + "properties": { + "err": { + "type": "string" + } + }, + "required": [ + "err" + ] + } + ] + }, + "slot_b": { + "x-rust-type": { + "crate": "std", + "parameters": [ + { + "$ref": "#/components/schemas/BootPartitionDetails" + }, + { + "type": "string" + } + ], + "path": "::std::result::Result", + "version": "*" + }, + "oneOf": [ + { + "type": "object", + "properties": { + "ok": { + "$ref": "#/components/schemas/BootPartitionDetails" + } + }, + "required": [ + "ok" + ] + }, + { + "type": "object", + "properties": { + "err": { + "type": "string" + } + }, + "required": [ + "err" + ] + } + ] + } + }, + "required": [ + "boot_disk", + "slot_a", + "slot_b" + ] + }, + "BootPartitionDetails": { + "type": "object", + "properties": { + "artifact_hash": { + "type": "string", + "format": "hex string (32 bytes)" + }, + "artifact_size": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "header": { + "$ref": "#/components/schemas/BootImageHeader" + } + }, + "required": [ + "artifact_hash", + "artifact_size", + "header" + ] + }, + "BootSettings": { + "description": "Settings supplied to the guest's firmware image that specify the order in which it should consider its options when selecting a device to try to boot from.", + "type": "object", + "properties": { + "order": { + "description": "An ordered list of components to attempt to boot from.", + "type": "array", + "items": { + "$ref": "#/components/schemas/BootOrderEntry" + } + } + }, + "required": [ + "order" + ], + "additionalProperties": false + }, + "BootstoreStatus": { + "description": "Status of the local bootstore node.", + "type": "object", + "properties": { + "accepted_connections": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "established_connections": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EstablishedConnection" + } + }, + "fsm_ledger_generation": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "fsm_state": { + "type": "string" + }, + "negotiating_connections": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "network_config_ledger_generation": { + "nullable": true, + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "peers": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "required": [ + "accepted_connections", + "established_connections", + "fsm_ledger_generation", + "fsm_state", + "negotiating_connections", + "peers" + ] + }, + "BundleUtilization": { + "description": "The portion of a debug dataset used for zone bundles.", + "type": "object", + "properties": { + "bytes_available": { + "description": "The total number of bytes available for zone bundles.\n\nThis is `dataset_quota` multiplied by the context's storage limit.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "bytes_used": { + "description": "Total bundle usage, in bytes.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "dataset_quota": { + "description": "The total dataset quota, in bytes.", + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "bytes_available", + "bytes_used", + "dataset_quota" + ] + }, + "ByteCount": { + "description": "Byte count to express memory or storage capacity.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "Chipset": { + "description": "A kind of virtual chipset.", + "oneOf": [ + { + "description": "An Intel 440FX-compatible chipset.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "i440_fx" + ] + }, + "value": { + "$ref": "#/components/schemas/I440Fx" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + } + ] + }, + "CleanupContext": { + "description": "Context provided for the zone bundle cleanup task.", + "type": "object", + "properties": { + "period": { + "description": "The period on which automatic checks and cleanup is performed.", + "allOf": [ + { + "$ref": "#/components/schemas/CleanupPeriod" + } + ] + }, + "priority": { + "description": "The priority ordering for keeping old bundles.", + "allOf": [ + { + "$ref": "#/components/schemas/PriorityOrder" + } + ] + }, + "storage_limit": { + "description": "The limit on the dataset quota available for zone bundles.", + "allOf": [ + { + "$ref": "#/components/schemas/StorageLimit" + } + ] + } + }, + "required": [ + "period", + "priority", + "storage_limit" + ] + }, + "CleanupContextUpdate": { + "description": "Parameters used to update the zone bundle cleanup context.", + "type": "object", + "properties": { + "period": { + "nullable": true, + "description": "The new period on which automatic cleanups are run.", + "allOf": [ + { + "$ref": "#/components/schemas/Duration" + } + ] + }, + "priority": { + "nullable": true, + "description": "The priority ordering for preserving old zone bundles.", + "allOf": [ + { + "$ref": "#/components/schemas/PriorityOrder" + } + ] + }, + "storage_limit": { + "nullable": true, + "description": "The new limit on the underlying dataset quota allowed for bundles.", + "type": "integer", + "format": "uint8", + "minimum": 0 + } + } + }, + "CleanupCount": { + "description": "The count of bundles / bytes removed during a cleanup operation.", + "type": "object", + "properties": { + "bundles": { + "description": "The number of bundles removed.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "bytes": { + "description": "The number of bytes removed.", + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "bundles", + "bytes" + ] + }, + "CleanupPeriod": { + "description": "A period on which bundles are automatically cleaned up.", + "allOf": [ + { + "$ref": "#/components/schemas/Duration" + } + ] + }, + "CombineError": { + "type": "string", + "enum": [ + "too_few_shares", + "duplicate_x_coordinates", + "invalid_share_lengths", + "invalid_share_id" + ] + }, + "CommitRequest": { + "description": "Request to commit a trust quorum configuration at a given epoch.", + "type": "object", + "properties": { + "epoch": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "rack_id": { + "$ref": "#/components/schemas/RackUuid" + } + }, + "required": [ + "epoch", + "rack_id" + ] + }, + "CommitStatus": { + "description": "Whether or not a configuration has been committed or is still underway.", + "type": "string", + "enum": [ + "committed", + "pending" + ] + }, + "ComponentV0": { + "oneOf": [ + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/VirtioDisk" + }, + "type": { + "type": "string", + "enum": [ + "virtio_disk" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/NvmeDisk" + }, + "type": { + "type": "string", + "enum": [ + "nvme_disk" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/VirtioNic" + }, + "type": { + "type": "string", + "enum": [ + "virtio_nic" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/SerialPort" + }, + "type": { + "type": "string", + "enum": [ + "serial_port" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/PciPciBridge" + }, + "type": { + "type": "string", + "enum": [ + "pci_pci_bridge" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/QemuPvpanic" + }, + "type": { + "type": "string", + "enum": [ + "qemu_pvpanic" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/BootSettings" + }, + "type": { + "type": "string", + "enum": [ + "boot_settings" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/SoftNpuPciPort" + }, + "type": { + "type": "string", + "enum": [ + "soft_npu_pci_port" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/SoftNpuPort" + }, + "type": { + "type": "string", + "enum": [ + "soft_npu_port" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/SoftNpuP9" + }, + "type": { + "type": "string", + "enum": [ + "soft_npu_p9" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/P9fs" + }, + "type": { + "type": "string", + "enum": [ + "p9fs" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/MigrationFailureInjector" + }, + "type": { + "type": "string", + "enum": [ + "migration_failure_injector" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/CrucibleStorageBackend" + }, + "type": { + "type": "string", + "enum": [ + "crucible_storage_backend" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/FileStorageBackend" + }, + "type": { + "type": "string", + "enum": [ + "file_storage_backend" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/BlobStorageBackend" + }, + "type": { + "type": "string", + "enum": [ + "blob_storage_backend" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/VirtioNetworkBackend" + }, + "type": { + "type": "string", + "enum": [ + "virtio_network_backend" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/DlpiNetworkBackend" + }, + "type": { + "type": "string", + "enum": [ + "dlpi_network_backend" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + } + ] + }, + "CompressionAlgorithm": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "on" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "off" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "gzip" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "level": { + "$ref": "#/components/schemas/GzipLevel" + }, + "type": { + "type": "string", + "enum": [ + "gzip_n" + ] + } + }, + "required": [ + "level", + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "lz4" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "lzjb" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "zle" + ] + } + }, + "required": [ + "type" + ] + } + ] + }, + "ConfigReconcilerInventory": { + "description": "Describes the last attempt made by the sled-agent-config-reconciler to reconcile the current sled config against the actual state of the sled.", + "type": "object", + "properties": { + "boot_partitions": { + "$ref": "#/components/schemas/BootPartitionContents" + }, + "datasets": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/ConfigReconcilerInventoryResult" + } + }, + "external_disks": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/ConfigReconcilerInventoryResult" + } + }, + "last_reconciled_config": { + "$ref": "#/components/schemas/OmicronSledConfig" + }, + "orphaned_datasets": { + "title": "IdOrdMap", + "x-rust-type": { + "crate": "iddqd", + "parameters": [ + { + "$ref": "#/components/schemas/OrphanedDataset" + } + ], + "path": "iddqd::IdOrdMap", + "version": "*" + }, + "type": "array", + "items": { + "$ref": "#/components/schemas/OrphanedDataset" + }, + "uniqueItems": true + }, + "remove_mupdate_override": { + "nullable": true, + "description": "The result of removing the mupdate override file on disk.\n\n`None` if `remove_mupdate_override` was not provided in the sled config.", + "allOf": [ + { + "$ref": "#/components/schemas/RemoveMupdateOverrideInventory" + } + ] + }, + "zones": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/ConfigReconcilerInventoryResult" + } + } + }, + "required": [ + "boot_partitions", + "datasets", + "external_disks", + "last_reconciled_config", + "orphaned_datasets", + "zones" + ] + }, + "ConfigReconcilerInventoryResult": { + "oneOf": [ + { + "type": "object", + "properties": { + "result": { + "type": "string", + "enum": [ + "ok" + ] + } + }, + "required": [ + "result" + ] + }, + { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "result": { + "type": "string", + "enum": [ + "err" + ] + } + }, + "required": [ + "message", + "result" + ] + } + ] + }, + "ConfigReconcilerInventoryStatus": { + "description": "Status of the sled-agent-config-reconciler task.", + "oneOf": [ + { + "description": "The reconciler task has not yet run for the first time since sled-agent started.", + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "not_yet_run" + ] + } + }, + "required": [ + "status" + ] + }, + { + "description": "The reconciler task is actively running.", + "type": "object", + "properties": { + "config": { + "$ref": "#/components/schemas/OmicronSledConfig" + }, + "running_for": { + "$ref": "#/components/schemas/Duration" + }, + "started_at": { + "type": "string", + "format": "date-time" + }, + "status": { + "type": "string", + "enum": [ + "running" + ] + } + }, + "required": [ + "config", + "running_for", + "started_at", + "status" + ] + }, + { + "description": "The reconciler task is currently idle, but previously did complete a reconciliation attempt.\n\nThis variant does not include the `OmicronSledConfig` used in the last attempt, because that's always available via [`ConfigReconcilerInventory::last_reconciled_config`].", + "type": "object", + "properties": { + "completed_at": { + "type": "string", + "format": "date-time" + }, + "ran_for": { + "$ref": "#/components/schemas/Duration" + }, + "status": { + "type": "string", + "enum": [ + "idle" + ] + } + }, + "required": [ + "completed_at", + "ran_for", + "status" + ] + } + ] + }, + "Configuration": { + "description": "The configuration for a given epoch.\n\nOnly valid for non-lrtq configurations.", + "type": "object", + "properties": { + "coordinator": { + "description": "Who was the coordinator of this reconfiguration?", + "allOf": [ + { + "$ref": "#/components/schemas/BaseboardId" + } + ] + }, + "encrypted_rack_secrets": { + "nullable": true, + "description": "There are no encrypted rack secrets for the initial configuration.", + "allOf": [ + { + "$ref": "#/components/schemas/EncryptedRackSecrets" + } + ] + }, + "epoch": { + "description": "Unique, monotonically increasing identifier for a configuration.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "members": { + "description": "All members of the current configuration and the hash of their key shares.", + "type": "array", + "items": { + "$ref": "#/components/schemas/ConfigurationMember" + } + }, + "rack_id": { + "description": "Unique Id of the rack.", + "allOf": [ + { + "$ref": "#/components/schemas/RackUuid" + } + ] + }, + "threshold": { + "description": "The number of sleds required to reconstruct the rack secret.", + "type": "integer", + "format": "uint8", + "minimum": 0 + } + }, + "required": [ + "coordinator", + "epoch", + "members", + "rack_id", + "threshold" + ] + }, + "ConfigurationMember": { + "description": "A member entry in a trust quorum configuration.\n\nThis type is used for OpenAPI schema generation since OpenAPI v3.0.x doesn't support tuple arrays.", + "type": "object", + "properties": { + "id": { + "description": "The baseboard ID of the member.", + "allOf": [ + { + "$ref": "#/components/schemas/BaseboardId" + } + ] + }, + "share_digest": { + "description": "The SHA3-256 hash of the member's key share.", + "type": "string", + "format": "hex string (32 bytes)" + } + }, + "required": [ + "id", + "share_digest" + ] + }, + "CoordinatorStatus": { + "description": "Status of the node coordinating the reconfiguration or LRTQ upgrade.", + "type": "object", + "properties": { + "acked_prepares": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BaseboardId" + }, + "uniqueItems": true + }, + "config": { + "$ref": "#/components/schemas/Configuration" + } + }, + "required": [ + "acked_prepares", + "config" + ] + }, + "Cpuid": { + "description": "A set of CPUID values to expose to a guest.", + "type": "object", + "properties": { + "entries": { + "description": "A list of CPUID leaves/subleaves and their associated values.\n\nPropolis servers require that each entry's `leaf` be unique and that it falls in either the \"standard\" (0 to 0xFFFF) or \"extended\" (0x8000_0000 to 0x8000_FFFF) function ranges, since these are the only valid input ranges currently defined by Intel and AMD. See the Intel 64 and IA-32 Architectures Software Developer's Manual (June 2024) Table 3-17 and the AMD64 Architecture Programmer's Manual (March 2024) Volume 3's documentation of the CPUID instruction.", + "type": "array", + "items": { + "$ref": "#/components/schemas/CpuidEntry" + } + }, + "vendor": { + "description": "The CPU vendor to emulate.\n\nCPUID leaves in the extended range (0x8000_0000 to 0x8000_FFFF) have vendor-defined semantics. Propolis uses this value to determine these semantics when deciding whether it needs to specialize the supplied template values for these leaves.", + "allOf": [ + { + "$ref": "#/components/schemas/CpuidVendor" + } + ] + } + }, + "required": [ + "entries", + "vendor" + ], + "additionalProperties": false + }, + "CpuidEntry": { + "description": "A full description of a CPUID leaf/subleaf and the values it produces.", + "type": "object", + "properties": { + "eax": { + "description": "The value to return in eax.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "ebx": { + "description": "The value to return in ebx.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "ecx": { + "description": "The value to return in ecx.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "edx": { + "description": "The value to return in edx.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "leaf": { + "description": "The leaf (function) number for this entry.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "subleaf": { + "nullable": true, + "description": "The subleaf (index) number for this entry, if it uses subleaves.", + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "eax", + "ebx", + "ecx", + "edx", + "leaf" + ], + "additionalProperties": false + }, + "CpuidVendor": { + "description": "A CPU vendor to use when interpreting the meanings of CPUID leaves in the extended ID range (0x80000000 to 0x8000FFFF).", + "type": "string", + "enum": [ + "amd", + "intel" + ] + }, + "CrucibleStorageBackend": { + "description": "A Crucible storage backend.", + "type": "object", + "properties": { + "readonly": { + "description": "Indicates whether the storage is read-only.", + "type": "boolean" + }, + "request_json": { + "description": "A serialized `[crucible_client_types::VolumeConstructionRequest]`. This is stored in serialized form so that breaking changes to the definition of a `VolumeConstructionRequest` do not inadvertently break instance spec deserialization.\n\nWhen using a spec to initialize a new instance, the spec author must ensure this request is well-formed and can be deserialized by the version of `crucible_client_types` used by the target Propolis.", + "type": "string" + } + }, + "required": [ + "readonly", + "request_json" + ], + "additionalProperties": false + }, + "DatasetConfig": { + "description": "Configuration information necessary to request a single dataset.\n\nThese datasets are tracked directly by Nexus.", + "type": "object", + "properties": { + "compression": { + "description": "The compression mode to be used by the dataset", + "allOf": [ + { + "$ref": "#/components/schemas/CompressionAlgorithm" + } + ] + }, + "id": { + "description": "The UUID of the dataset being requested", + "allOf": [ + { + "$ref": "#/components/schemas/DatasetUuid" + } + ] + }, + "name": { + "description": "The dataset's name", + "allOf": [ + { + "$ref": "#/components/schemas/DatasetName" + } + ] + }, + "quota": { + "nullable": true, + "description": "The upper bound on the amount of storage used by this dataset", + "allOf": [ + { + "$ref": "#/components/schemas/ByteCount" + } + ] + }, + "reservation": { + "nullable": true, + "description": "The lower bound on the amount of storage usable by this dataset", + "allOf": [ + { + "$ref": "#/components/schemas/ByteCount" + } + ] + } + }, + "required": [ + "compression", + "id", + "name" + ] + }, + "DatasetKind": { + "description": "The kind of dataset. See the `DatasetKind` enum in omicron-common for possible values.", + "type": "string" + }, + "DatasetName": { + "type": "object", + "properties": { + "kind": { + "$ref": "#/components/schemas/DatasetKind" + }, + "pool_name": { + "$ref": "#/components/schemas/ZpoolName" + } + }, + "required": [ + "kind", + "pool_name" + ] + }, + "DatasetUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::DatasetUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, + "DecryptionError": { + "description": "Error decrypting rack secrets.", + "oneOf": [ + { + "description": "An opaque error indicating decryption failed.", + "type": "string", + "enum": [ + "aead" + ] + }, + { + "description": "The length of the plaintext is not the correct size and cannot be decoded.", + "type": "string", + "enum": [ + "invalid_length" + ] + } + ] + }, + "DelegatedZvol": { + "description": "Delegate a ZFS volume to a zone", + "oneOf": [ + { + "description": "Delegate a slice of the _unencrypted_ local storage dataset present on this pool into the zone.", + "type": "object", + "properties": { + "dataset_id": { + "$ref": "#/components/schemas/DatasetUuid" + }, + "type": { + "type": "string", + "enum": [ + "local_storage_unencrypted" + ] + }, + "zpool_id": { + "$ref": "#/components/schemas/ExternalZpoolUuid" + } + }, + "required": [ + "dataset_id", + "type", + "zpool_id" + ] + }, + { + "description": "Delegate a slice of the _encrypted_ local storage dataset present on this pool into the zone.", + "type": "object", + "properties": { + "dataset_id": { + "$ref": "#/components/schemas/DatasetUuid" + }, + "type": { + "type": "string", + "enum": [ + "local_storage_encrypted" + ] + }, + "zpool_id": { + "$ref": "#/components/schemas/ExternalZpoolUuid" + } + }, + "required": [ + "dataset_id", + "type", + "zpool_id" + ] + } + ] + }, + "DhcpConfig": { + "description": "DHCP configuration for a port\n\nNot present here: Hostname (DHCPv4 option 12; used in DHCPv6 option 39); we use `InstanceRuntimeState::hostname` for this value.", + "type": "object", + "properties": { + "dns_servers": { + "description": "DNS servers to send to the instance\n\n(DHCPv4 option 6; DHCPv6 option 23)", + "type": "array", + "items": { + "type": "string", + "format": "ip" + } + }, + "host_domain": { + "nullable": true, + "description": "DNS zone this instance's hostname belongs to (e.g. the `project.example` part of `instance1.project.example`)\n\n(DHCPv4 option 15; used in DHCPv6 option 39)", + "type": "string" + }, + "search_domains": { + "description": "DNS search domains\n\n(DHCPv4 option 119; DHCPv6 option 24)", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "dns_servers", + "search_domains" + ] + }, + "DiskEnsureBody": { + "description": "Sent from to a sled agent to establish the runtime state of a Disk", + "type": "object", + "properties": { + "initial_runtime": { + "description": "Last runtime state of the Disk known to Nexus (used if the agent has never seen this Disk before).", + "allOf": [ + { + "$ref": "#/components/schemas/DiskRuntimeState" + } + ] + }, + "target": { + "description": "requested runtime state of the Disk", + "allOf": [ + { + "$ref": "#/components/schemas/DiskStateRequested" + } + ] + } + }, + "required": [ + "initial_runtime", + "target" + ] + }, + "DiskIdentity": { + "description": "Uniquely identifies a disk.", + "type": "object", + "properties": { + "model": { + "type": "string" + }, + "serial": { + "type": "string" + }, + "vendor": { + "type": "string" + } + }, + "required": [ + "model", + "serial", + "vendor" + ] + }, + "DiskRuntimeState": { + "description": "Runtime state of the Disk, which includes its attach state and some minimal metadata", + "type": "object", + "properties": { + "disk_state": { + "description": "runtime state of the Disk", + "allOf": [ + { + "$ref": "#/components/schemas/DiskState" + } + ] + }, + "gen": { + "description": "generation number for this state", + "allOf": [ + { + "$ref": "#/components/schemas/Generation" + } + ] + }, + "time_updated": { + "description": "timestamp for this information", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "disk_state", + "gen", + "time_updated" + ] + }, + "DiskState": { + "description": "State of a Disk", + "oneOf": [ + { + "description": "Disk is being initialized", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "creating" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is ready but detached from any Instance", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "detached" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is ready to receive blocks from an external source", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "import_ready" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is importing blocks from a URL", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "importing_from_url" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is importing blocks from bulk writes", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "importing_from_bulk_writes" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is being finalized to state Detached", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "finalizing" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is undergoing maintenance", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "maintenance" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is being attached to the given Instance", + "type": "object", + "properties": { + "instance": { + "type": "string", + "format": "uuid" + }, + "state": { + "type": "string", + "enum": [ + "attaching" + ] + } + }, + "required": [ + "instance", + "state" + ] + }, + { + "description": "Disk is attached to the given Instance", + "type": "object", + "properties": { + "instance": { + "type": "string", + "format": "uuid" + }, + "state": { + "type": "string", + "enum": [ + "attached" + ] + } + }, + "required": [ + "instance", + "state" + ] + }, + { + "description": "Disk is being detached from the given Instance", + "type": "object", + "properties": { + "instance": { + "type": "string", + "format": "uuid" + }, + "state": { + "type": "string", + "enum": [ + "detaching" + ] + } + }, + "required": [ + "instance", + "state" + ] + }, + { + "description": "Disk has been destroyed", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "destroyed" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is unavailable", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "faulted" + ] + } + }, + "required": [ + "state" + ] + } + ] + }, + "DiskStateRequested": { + "description": "Used to request a Disk state change", + "oneOf": [ + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "detached" + ] + } + }, + "required": [ + "state" + ] + }, + { + "type": "object", + "properties": { + "instance": { + "type": "string", + "format": "uuid" + }, + "state": { + "type": "string", + "enum": [ + "attached" + ] + } + }, + "required": [ + "instance", + "state" + ] + }, + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "destroyed" + ] + } + }, + "required": [ + "state" + ] + }, + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "faulted" + ] + } + }, + "required": [ + "state" + ] + } + ] + }, + "DiskVariant": { + "type": "string", + "enum": [ + "U2", + "M2" + ] + }, + "DlpiNetworkBackend": { + "description": "A network backend associated with a DLPI VNIC on the host.", + "type": "object", + "properties": { + "vnic_name": { + "description": "The name of the VNIC to use as a backend.", + "type": "string" + } + }, + "required": [ + "vnic_name" + ], + "additionalProperties": false + }, + "Duration": { + "type": "object", + "properties": { + "nanos": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "secs": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "nanos", + "secs" + ] + }, + "EarlyNetworkConfig": { + "description": "Network configuration required to bring up the control plane\n\nThe fields in this structure are those from `RackInitializeRequest` necessary for use beyond RSS. This is just for the initial rack configuration and cold boot purposes. Updates come from Nexus.", + "type": "object", + "properties": { + "body": { + "$ref": "#/components/schemas/EarlyNetworkConfigBody" + }, + "generation": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "schema_version": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "body", + "generation", + "schema_version" + ] + }, + "EarlyNetworkConfigBody": { + "description": "This is the actual configuration of EarlyNetworking.\n\nWe nest it below the \"header\" of `generation` and `schema_version` so that we can perform partial deserialization of `EarlyNetworkConfig` to only read the header and defer deserialization of the body once we know the schema version. This is possible via the use of [`serde_json::value::RawValue`] in future (post-v1) deserialization paths.", + "type": "object", + "properties": { + "ntp_servers": { + "description": "The external NTP server addresses.", + "type": "array", + "items": { + "type": "string" + } + }, + "rack_network_config": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/RackNetworkConfigV2" + } + ] + } + }, + "required": [ + "ntp_servers" + ] + }, + "EncryptedRackSecrets": { + "description": "All possibly relevant __encrypted__ rack secrets for _prior_ committed configurations.", + "type": "object", + "properties": { + "data": { + "description": "Encrypted data.", + "type": "string", + "format": "hex string" + }, + "salt": { + "description": "A random value used to derive the key to encrypt the rack secrets for prior committed epochs.", + "type": "string", + "format": "hex string (32 bytes)" + } + }, + "required": [ + "data", + "salt" + ] + }, + "Error": { + "description": "Error information from a response.", + "type": "object", + "properties": { + "error_code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "message", + "request_id" + ] + }, + "EstablishedConnection": { + "description": "An established connection to a bootstore peer.", + "type": "object", + "properties": { + "addr": { + "type": "string" + }, + "baseboard": { + "$ref": "#/components/schemas/Baseboard" + } + }, + "required": [ + "addr", + "baseboard" + ] + }, + "ExpungedMetadata": { + "description": "Metadata about a node being expunged from the trust quorum.", + "type": "object", + "properties": { + "epoch": { + "description": "The committed epoch, later than its current configuration at which the node learned that it had been expunged.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "from": { + "description": "Which node this commit information was learned from.", + "allOf": [ + { + "$ref": "#/components/schemas/BaseboardId" + } + ] + } + }, + "required": [ + "epoch", + "from" + ] + }, + "ExternalIp": { + "description": "An external IP address used by a probe.", + "type": "object", + "properties": { + "first_port": { + "description": "The first port used by the address.", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "ip": { + "description": "The external IP address.", + "type": "string", + "format": "ip" + }, + "kind": { + "description": "The kind of address this is.", + "allOf": [ + { + "$ref": "#/components/schemas/IpKind" + } + ] + }, + "last_port": { + "description": "The last port used by the address.", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "first_port", + "ip", + "kind", + "last_port" + ] + }, + "ExternalIpConfig": { + "description": "A single- or dual-stack external IP configuration.", + "oneOf": [ + { + "description": "Single-stack IPv4 external IP configuration.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "v4" + ] + }, + "value": { + "$ref": "#/components/schemas/ExternalIpv4Config" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Single-stack IPv6 external IP configuration.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "v6" + ] + }, + "value": { + "$ref": "#/components/schemas/ExternalIpv6Config" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Both IPv4 and IPv6 external IP configuration.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "dual_stack" + ] + }, + "value": { + "type": "object", + "properties": { + "v4": { + "$ref": "#/components/schemas/ExternalIpv4Config" + }, + "v6": { + "$ref": "#/components/schemas/ExternalIpv6Config" + } + }, + "required": [ + "v4", + "v6" + ] + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, + "ExternalIpGatewayMap": { + "description": "Per-NIC mappings from external IP addresses to the Internet Gateways which can choose them as a source.", + "type": "object", + "properties": { + "mappings": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string", + "format": "uuid" + }, + "uniqueItems": true + } + } + } + }, + "required": [ + "mappings" + ] + }, + "ExternalIpv4Config": { + "description": "External IP address configuration.\n\nThis encapsulates all the external addresses of a single IP version, including source NAT, Ephemeral, and Floating IPs. Note that not all of these need to be specified, but this type can only be constructed if _at least one_ of them is.", + "type": "object", + "properties": { + "ephemeral_ip": { + "nullable": true, + "description": "An Ephemeral address for in- and outbound connectivity.", + "type": "string", + "format": "ipv4" + }, + "floating_ips": { + "description": "Additional Floating IPs for in- and outbound connectivity.", + "type": "array", + "items": { + "type": "string", + "format": "ipv4" + } + }, + "source_nat": { + "nullable": true, + "description": "Source NAT configuration, for outbound-only connectivity.", + "allOf": [ + { + "$ref": "#/components/schemas/SourceNatConfigV4" + } + ] + } + }, + "required": [ + "floating_ips" + ] + }, + "ExternalIpv6Config": { + "description": "External IP address configuration.\n\nThis encapsulates all the external addresses of a single IP version, including source NAT, Ephemeral, and Floating IPs. Note that not all of these need to be specified, but this type can only be constructed if _at least one_ of them is.", + "type": "object", + "properties": { + "ephemeral_ip": { + "nullable": true, + "description": "An Ephemeral address for in- and outbound connectivity.", + "type": "string", + "format": "ipv6" + }, + "floating_ips": { + "description": "Additional Floating IPs for in- and outbound connectivity.", + "type": "array", + "items": { + "type": "string", + "format": "ipv6" + } + }, + "source_nat": { + "nullable": true, + "description": "Source NAT configuration, for outbound-only connectivity.", + "allOf": [ + { + "$ref": "#/components/schemas/SourceNatConfigV6" + } + ] + } + }, + "required": [ + "floating_ips" + ] + }, + "ExternalZpoolUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::ExternalZpoolUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, + "FileStorageBackend": { + "description": "A storage backend backed by a file in the host system's file system.", + "type": "object", + "properties": { + "block_size": { + "description": "Block size of the backend", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "path": { + "description": "A path to a file that backs a disk.", + "type": "string" + }, + "readonly": { + "description": "Indicates whether the storage is read-only.", + "type": "boolean" + }, + "workers": { + "nullable": true, + "description": "Optional worker threads for the file backend, exposed for testing only.", + "type": "integer", + "format": "uint", + "minimum": 1 + } + }, + "required": [ + "block_size", + "path", + "readonly" + ], + "additionalProperties": false + }, + "Generation": { + "description": "Generation numbers stored in the database, used for optimistic concurrency control", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "GuestHypervisorInterface": { + "description": "A hypervisor interface to expose to the guest.", + "oneOf": [ + { + "description": "Expose a bhyve-like interface (\"bhyve bhyve \" as the hypervisor ID in leaf 0x4000_0000 and no additional leaves or features).", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "bhyve" + ] + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + { + "description": "Expose a Hyper-V-compatible hypervisor interface with the supplied features enabled.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "hyper_v" + ] + }, + "value": { + "type": "object", + "properties": { + "features": { + "type": "array", + "items": { + "$ref": "#/components/schemas/HyperVFeatureFlag" + }, + "uniqueItems": true + } + }, + "required": [ + "features" + ], + "additionalProperties": false + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + } + ] + }, + "GzipLevel": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "HealthMonitorInventory": { + "description": "Fields of sled-agent inventory reported by the health monitor subsystem.", + "type": "object", + "properties": { + "smf_services_in_maintenance": { + "x-rust-type": { + "crate": "std", + "parameters": [ + { + "$ref": "#/components/schemas/SvcsInMaintenanceResult" + }, + { + "type": "string" + } + ], + "path": "::std::result::Result", + "version": "*" + }, + "oneOf": [ + { + "type": "object", + "properties": { + "ok": { + "$ref": "#/components/schemas/SvcsInMaintenanceResult" + } + }, + "required": [ + "ok" + ] + }, + { + "type": "object", + "properties": { + "err": { + "type": "string" + } + }, + "required": [ + "err" + ] + } + ] + } + }, + "required": [ + "smf_services_in_maintenance" + ] + }, + "HostIdentifier": { + "description": "A `HostIdentifier` represents either an IP host or network (v4 or v6), or an entire VPC (identified by its VNI). It is used in firewall rule host filters.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ip" + ] + }, + "value": { + "$ref": "#/components/schemas/IpNet" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "vpc" + ] + }, + "value": { + "$ref": "#/components/schemas/Vni" + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, + "HostPhase2DesiredContents": { + "description": "Describes the desired contents of a host phase 2 slot (i.e., the boot partition on one of the internal M.2 drives).", + "oneOf": [ + { + "description": "Do not change the current contents.\n\nWe use this value when we've detected a sled has been mupdated (and we don't want to overwrite phase 2 images until we understand how to recover from that mupdate) and as the default value when reading an [`OmicronSledConfig`] that was ledgered before this concept existed.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "current_contents" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Set the phase 2 slot to the given artifact.\n\nThe artifact will come from an unpacked and distributed TUF repo.", + "type": "object", + "properties": { + "hash": { + "type": "string", + "format": "hex string (32 bytes)" + }, + "type": { + "type": "string", + "enum": [ + "artifact" + ] + } + }, + "required": [ + "hash", + "type" + ] + } + ] + }, + "HostPhase2DesiredSlots": { + "description": "Describes the desired contents for both host phase 2 slots.", + "type": "object", + "properties": { + "slot_a": { + "$ref": "#/components/schemas/HostPhase2DesiredContents" + }, + "slot_b": { + "$ref": "#/components/schemas/HostPhase2DesiredContents" + } + }, + "required": [ + "slot_a", + "slot_b" + ] + }, + "HostPortConfig": { + "type": "object", + "properties": { + "addrs": { + "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool). May also include an optional VLAN ID.", + "type": "array", + "items": { + "$ref": "#/components/schemas/UplinkAddressConfig" + } + }, + "lldp": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/LldpPortConfig" + } + ] + }, + "port": { + "description": "Switchport to use for external connectivity", + "type": "string" + }, + "tx_eq": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/TxEqConfig" + } + ] + } + }, + "required": [ + "addrs", + "port" + ] + }, + "Hostname": { + "title": "An RFC-1035-compliant hostname", + "description": "A hostname identifies a host on a network, and is usually a dot-delimited sequence of labels, where each label contains only letters, digits, or the hyphen. See RFCs 1035 and 952 for more details.", + "type": "string", + "pattern": "^([a-zA-Z0-9]+[a-zA-Z0-9\\-]*(? for background.", + "oneOf": [ + { + "description": "Start the switch zone if a switch is present.\n\nThis is the default policy.", + "type": "object", + "properties": { + "policy": { + "type": "string", + "enum": [ + "start_if_switch_present" + ] + } + }, + "required": [ + "policy" + ] + }, + { + "description": "Even if a switch zone is present, stop the switch zone.", + "type": "object", + "properties": { + "policy": { + "type": "string", + "enum": [ + "stop_despite_switch_presence" + ] + } + }, + "required": [ + "policy" + ] + } + ] + }, + "OrphanedDataset": { + "type": "object", + "properties": { + "available": { + "$ref": "#/components/schemas/ByteCount" + }, + "id": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/DatasetUuid" + } + ] + }, + "mounted": { + "type": "boolean" + }, + "name": { + "$ref": "#/components/schemas/DatasetName" + }, + "reason": { + "type": "string" + }, + "used": { + "$ref": "#/components/schemas/ByteCount" + } + }, + "required": [ + "available", + "mounted", + "name", + "reason", + "used" + ] + }, + "P9fs": { + "description": "Describes a filesystem to expose through a P9 device.\n\nThis is only supported by Propolis servers compiled with the `falcon` feature.", + "type": "object", + "properties": { + "chunk_size": { + "description": "The chunk size to use in the 9P protocol. Vanilla Helios images should use 8192. Falcon Helios base images and Linux can use up to 65536.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "pci_path": { + "description": "The PCI path at which to attach the guest to this P9 filesystem.", + "allOf": [ + { + "$ref": "#/components/schemas/PciPath" + } + ] + }, + "source": { + "description": "The host source path to mount into the guest.", + "type": "string" + }, + "target": { + "description": "The 9P target filesystem tag.", + "type": "string" + } + }, + "required": [ + "chunk_size", + "pci_path", + "source", + "target" + ], + "additionalProperties": false + }, + "PciPath": { + "description": "A PCI bus/device/function tuple.", + "type": "object", + "properties": { + "bus": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "device": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "function": { + "type": "integer", + "format": "uint8", + "minimum": 0 + } + }, + "required": [ + "bus", + "device", + "function" + ] + }, + "PciPciBridge": { + "description": "A PCI-PCI bridge.", + "type": "object", + "properties": { + "downstream_bus": { + "description": "The logical bus number of this bridge's downstream bus. Other devices may use this bus number in their PCI paths to indicate they should be attached to this bridge's bus.", + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "pci_path": { + "description": "The PCI path at which to attach this bridge.", + "allOf": [ + { + "$ref": "#/components/schemas/PciPath" + } + ] + } + }, + "required": [ + "downstream_bus", + "pci_path" + ], + "additionalProperties": false + }, + "PhysicalDiskUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::PhysicalDiskUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, + "PortConfigV2": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses and optional vlan IDs", + "type": "array", + "items": { + "$ref": "#/components/schemas/UplinkAddressConfig" + } + }, + "autoneg": { + "description": "Whether or not to set autonegotiation", + "default": false, + "type": "boolean" + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "lldp": { + "nullable": true, + "description": "LLDP configuration for this port", + "allOf": [ + { + "$ref": "#/components/schemas/LldpPortConfig" + } + ] + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "tx_eq": { + "nullable": true, + "description": "TX-EQ configuration for this port", + "allOf": [ + { + "$ref": "#/components/schemas/TxEqConfig" + } + ] + }, + "uplink_port_fec": { + "nullable": true, + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_speed" + ] + }, + "PortFec": { + "description": "Switchport FEC options", + "type": "string", + "enum": [ + "firecode", + "none", + "rs" + ] + }, + "PortSpeed": { + "description": "Switchport Speed options", + "type": "string", + "enum": [ + "speed0_g", + "speed1_g", + "speed10_g", + "speed25_g", + "speed40_g", + "speed50_g", + "speed100_g", + "speed200_g", + "speed400_g" + ] + }, + "PrepareAndCommitRequest": { + "description": "Request to prepare and commit a trust quorum configuration.\n\nThis is the `Configuration` sent to a node that missed the `Prepare` phase.", + "type": "object", + "properties": { + "config": { + "$ref": "#/components/schemas/Configuration" + } + }, + "required": [ + "config" + ] + }, + "PriorityDimension": { + "description": "A dimension along with bundles can be sorted, to determine priority.", + "oneOf": [ + { + "description": "Sorting by time, with older bundles with lower priority.", + "type": "string", + "enum": [ + "time" + ] + }, + { + "description": "Sorting by the cause for creating the bundle.", + "type": "string", + "enum": [ + "cause" + ] + } + ] + }, + "PriorityOrder": { + "description": "The priority order for bundles during cleanup.\n\nBundles are sorted along the dimensions in [`PriorityDimension`], with each dimension appearing exactly once. During cleanup, lesser-priority bundles are pruned first, to maintain the dataset quota. Note that bundles are sorted by each dimension in the order in which they appear, with each dimension having higher priority than the next.\n\nTODO: The serde deserializer does not currently verify uniqueness of dimensions.", + "type": "array", + "items": { + "$ref": "#/components/schemas/PriorityDimension" + }, + "minItems": 2, + "maxItems": 2 + }, + "PrivateIpConfig": { + "description": "VPC-private IP address configuration for a network interface.", + "oneOf": [ + { + "description": "The interface has only an IPv4 configuration.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "v4" + ] + }, + "value": { + "$ref": "#/components/schemas/PrivateIpv4Config" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "The interface has only an IPv6 configuration.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "v6" + ] + }, + "value": { + "$ref": "#/components/schemas/PrivateIpv6Config" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "The interface is dual-stack.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "dual_stack" + ] + }, + "value": { + "type": "object", + "properties": { + "v4": { + "description": "The interface's IPv4 configuration.", + "allOf": [ + { + "$ref": "#/components/schemas/PrivateIpv4Config" + } + ] + }, + "v6": { + "description": "The interface's IPv6 configuration.", + "allOf": [ + { + "$ref": "#/components/schemas/PrivateIpv6Config" + } + ] + } + }, + "required": [ + "v4", + "v6" + ] + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, + "PrivateIpv4Config": { + "description": "VPC-private IPv4 configuration for a network interface.", + "type": "object", + "properties": { + "ip": { + "description": "VPC-private IP address.", + "type": "string", + "format": "ipv4" + }, + "subnet": { + "description": "The IP subnet.", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Net" + } + ] + }, + "transit_ips": { + "description": "Additional networks on which the interface can send / receive traffic.", + "default": [], + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Net" + } + } + }, + "required": [ + "ip", + "subnet" + ] + }, + "PrivateIpv6Config": { + "description": "VPC-private IPv6 configuration for a network interface.", + "type": "object", + "properties": { + "ip": { + "description": "VPC-private IP address.", + "type": "string", + "format": "ipv6" + }, + "subnet": { + "description": "The IP subnet.", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Net" + } + ] + }, + "transit_ips": { + "description": "Additional networks on which the interface can send / receive traffic.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv6Net" + } + } + }, + "required": [ + "ip", + "subnet", + "transit_ips" + ] + }, + "ProbeCreate": { + "description": "Parameters used to create a probe.", + "type": "object", + "properties": { + "external_ips": { + "description": "The external IP addresses assigned to the probe.", + "type": "array", + "items": { + "$ref": "#/components/schemas/ExternalIp" + } + }, + "id": { + "description": "The ID for the probe.", + "allOf": [ + { + "$ref": "#/components/schemas/ProbeUuid" + } + ] + }, + "interface": { + "description": "The probe's networking interface.", + "allOf": [ + { + "$ref": "#/components/schemas/NetworkInterface" + } + ] + } + }, + "required": [ + "external_ips", + "id", + "interface" + ] + }, + "ProbeSet": { + "description": "A set of probes that the target sled should run.", + "type": "object", + "properties": { + "probes": { + "title": "IdHashMap", + "description": "The exact set of probes to run.", + "x-rust-type": { + "crate": "iddqd", + "parameters": [ + { + "$ref": "#/components/schemas/ProbeCreate" + } + ], + "path": "iddqd::IdHashMap", + "version": "*" + }, + "type": "array", + "items": { + "$ref": "#/components/schemas/ProbeCreate" + }, + "uniqueItems": true + } + }, + "required": [ + "probes" + ] + }, + "ProbeUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::ProbeUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, + "ProxyCommitRequest": { + "description": "Request to proxy a commit operation to another trust quorum node.", + "type": "object", + "properties": { + "destination": { + "description": "The target node to proxy the request to.", + "allOf": [ + { + "$ref": "#/components/schemas/BaseboardId" + } + ] + }, + "request": { + "description": "The commit request to proxy.", + "allOf": [ + { + "$ref": "#/components/schemas/CommitRequest" + } + ] + } + }, + "required": [ + "destination", + "request" + ] + }, + "ProxyPrepareAndCommitRequest": { + "description": "Request to proxy a prepare-and-commit operation to another trust quorum node.", + "type": "object", + "properties": { + "destination": { + "description": "The target node to proxy the request to.", + "allOf": [ + { + "$ref": "#/components/schemas/BaseboardId" + } + ] + }, + "request": { + "description": "The prepare-and-commit request to proxy.", + "allOf": [ + { + "$ref": "#/components/schemas/PrepareAndCommitRequest" + } + ] + } + }, + "required": [ + "destination", + "request" + ] + }, + "QemuPvpanic": { + "type": "object", + "properties": { + "enable_isa": { + "description": "Enable the QEMU PVPANIC ISA bus device (I/O port 0x505).", + "type": "boolean" + } + }, + "required": [ + "enable_isa" + ], + "additionalProperties": false + }, + "RackNetworkConfigV2": { + "description": "Initial network configuration", + "type": "object", + "properties": { + "bfd": { + "description": "BFD configuration for connecting the rack to external networks", + "default": [], + "type": "array", + "items": { + "$ref": "#/components/schemas/BfdPeerConfig" + } + }, + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, + "infra_ip_first": { + "description": "First ip address to be used for configuring network infrastructure", + "type": "string", + "format": "ipv4" + }, + "infra_ip_last": { + "description": "Last ip address to be used for configuring network infrastructure", + "type": "string", + "format": "ipv4" + }, + "ports": { + "description": "Uplinks for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/PortConfigV2" + } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Net" + } + }, + "required": [ + "bgp", + "infra_ip_first", + "infra_ip_last", + "ports", + "rack_subnet" + ] + }, + "RackSecretReconstructError": { + "description": "Error reconstructing a rack secret from shares.", + "oneOf": [ + { + "type": "object", + "properties": { + "combine": { + "$ref": "#/components/schemas/CombineError" + } + }, + "required": [ + "combine" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "size": { + "$ref": "#/components/schemas/InvalidRackSecretSizeError" + } + }, + "required": [ + "size" + ], + "additionalProperties": false + } + ] + }, + "RackUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::RackUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, + "ReconfigureMsg": { + "description": "A request from Nexus informing a node to start coordinating a reconfiguration.", + "type": "object", + "properties": { + "epoch": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "last_committed_epoch": { + "nullable": true, + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "members": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BaseboardId" + }, + "uniqueItems": true + }, + "rack_id": { + "$ref": "#/components/schemas/RackUuid" + }, + "threshold": { + "type": "integer", + "format": "uint8", + "minimum": 0 + } + }, + "required": [ + "epoch", + "members", + "rack_id", + "threshold" + ] + }, + "RemoveMupdateOverrideBootSuccessInventory": { + "description": "Status of removing the mupdate override on the boot disk.", + "oneOf": [ + { + "description": "The mupdate override was successfully removed.", + "type": "string", + "enum": [ + "removed" + ] + }, + { + "description": "No mupdate override was found.\n\nThis is considered a success for idempotency reasons.", + "type": "string", + "enum": [ + "no_override" + ] + } + ] + }, + "RemoveMupdateOverrideInventory": { + "description": "Status of removing the mupdate override in the inventory.", + "type": "object", + "properties": { + "boot_disk_result": { + "description": "The result of removing the mupdate override on the boot disk.", + "x-rust-type": { + "crate": "std", + "parameters": [ + { + "$ref": "#/components/schemas/RemoveMupdateOverrideBootSuccessInventory" + }, + { + "type": "string" + } + ], + "path": "::std::result::Result", + "version": "*" + }, + "oneOf": [ + { + "type": "object", + "properties": { + "ok": { + "$ref": "#/components/schemas/RemoveMupdateOverrideBootSuccessInventory" + } + }, + "required": [ + "ok" + ] + }, + { + "type": "object", + "properties": { + "err": { + "type": "string" + } + }, + "required": [ + "err" + ] + } + ] + }, + "non_boot_message": { + "description": "What happened on non-boot disks.\n\nWe aren't modeling this out in more detail, because we plan to not try and keep ledgered data in sync across both disks in the future.", + "type": "string" + } + }, + "required": [ + "boot_disk_result", + "non_boot_message" + ] + }, + "ResolvedVpcFirewallRule": { + "description": "VPC firewall rule after object name resolution has been performed by Nexus", + "type": "object", + "properties": { + "action": { + "$ref": "#/components/schemas/VpcFirewallRuleAction" + }, + "direction": { + "$ref": "#/components/schemas/VpcFirewallRuleDirection" + }, + "filter_hosts": { + "nullable": true, + "type": "array", + "items": { + "$ref": "#/components/schemas/HostIdentifier" + }, + "uniqueItems": true + }, + "filter_ports": { + "nullable": true, + "type": "array", + "items": { + "$ref": "#/components/schemas/L4PortRange" + } + }, + "filter_protocols": { + "nullable": true, + "type": "array", + "items": { + "$ref": "#/components/schemas/VpcFirewallRuleProtocol" + } + }, + "priority": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "status": { + "$ref": "#/components/schemas/VpcFirewallRuleStatus" + }, + "targets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/NetworkInterface" + } + } + }, + "required": [ + "action", + "direction", + "priority", + "status", + "targets" + ] + }, + "ResolvedVpcRoute": { + "description": "A VPC route resolved into a concrete target.", + "type": "object", + "properties": { + "dest": { + "$ref": "#/components/schemas/IpNet" + }, + "target": { + "$ref": "#/components/schemas/RouterTarget" + } + }, + "required": [ + "dest", + "target" + ] + }, + "ResolvedVpcRouteSet": { + "description": "An updated set of routes for a given VPC and/or subnet.", + "type": "object", + "properties": { + "id": { + "$ref": "#/components/schemas/RouterId" + }, + "routes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ResolvedVpcRoute" + }, + "uniqueItems": true + }, + "version": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/RouterVersion" + } + ] + } + }, + "required": [ + "id", + "routes" + ] + }, + "ResolvedVpcRouteState": { + "description": "Version information for routes on a given VPC subnet.", + "type": "object", + "properties": { + "id": { + "$ref": "#/components/schemas/RouterId" + }, + "version": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/RouterVersion" + } + ] + } + }, + "required": [ + "id" + ] + }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + }, + "rib_priority": { + "nullable": true, + "description": "The RIB priority (i.e. Admin Distance) associated with this route.", + "default": null, + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "vlan_id": { + "nullable": true, + "description": "The VLAN id associated with this route.", + "default": null, + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "destination", + "nexthop" + ] + }, + "RouterId": { + "description": "Identifier for a VPC and/or subnet.", + "type": "object", + "properties": { + "kind": { + "$ref": "#/components/schemas/RouterKind" + }, + "vni": { + "$ref": "#/components/schemas/Vni" + } + }, + "required": [ + "kind", + "vni" + ] + }, + "RouterKind": { + "description": "The scope of a set of VPC router rules.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "system" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "subnet": { + "$ref": "#/components/schemas/IpNet" + }, + "type": { + "type": "string", + "enum": [ + "custom" + ] + } + }, + "required": [ + "subnet", + "type" + ] + } + ] + }, + "RouterTarget": { + "description": "The target for a given router entry.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "drop" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "internet_gateway" + ] + }, + "value": { + "$ref": "#/components/schemas/InternetGatewayRouterTarget" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ip" + ] + }, + "value": { + "type": "string", + "format": "ip" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "vpc_subnet" + ] + }, + "value": { + "$ref": "#/components/schemas/IpNet" + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, + "RouterVersion": { + "description": "Information on the current parent router (and version) of a route set according to the control plane.", + "type": "object", + "properties": { + "router_id": { + "type": "string", + "format": "uuid" + }, + "version": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "router_id", + "version" + ] + }, + "SerialPort": { + "description": "A serial port device.", + "type": "object", + "properties": { + "num": { + "description": "The serial port number for this port.", + "allOf": [ + { + "$ref": "#/components/schemas/SerialPortNumber" + } + ] + } + }, + "required": [ + "num" + ], + "additionalProperties": false + }, + "SerialPortNumber": { + "description": "A serial port identifier, which determines what I/O ports a guest can use to access a port.", + "type": "string", + "enum": [ + "com1", + "com2", + "com3", + "com4" + ] + }, + "SingleMeasurementInventory": { + "description": "An attempt at resolving a single measurement file to a valid path", + "type": "object", + "properties": { + "path": { + "type": "string", + "format": "Utf8PathBuf" + }, + "result": { + "$ref": "#/components/schemas/ConfigReconcilerInventoryResult" + } + }, + "required": [ + "path", + "result" + ] + }, + "SledCpuFamily": { + "description": "Identifies the kind of CPU present on a sled, determined by reading CPUID.\n\nThis is intended to broadly support the control plane answering the question \"can I run this instance on that sled?\" given an instance with either no or some CPU platform requirement. It is not enough information for more precise placement questions - for example, is a CPU a high-frequency part or many-core part? We don't include Genoa here, but in that CPU family there are high frequency parts, many-core parts, and large-cache parts. To support those questions (or satisfactorily answer #8730) we would need to collect additional information and send it along.", + "oneOf": [ + { + "description": "The CPU vendor or its family number don't correspond to any of the known family variants.", + "type": "string", + "enum": [ + "unknown" + ] + }, + { + "description": "AMD Milan processors (or very close). Could be an actual Milan in a Gimlet, a close-to-Milan client Zen 3 part, or Zen 4 (for which Milan is the greatest common denominator).", + "type": "string", + "enum": [ + "amd_milan" + ] + }, + { + "description": "AMD Turin processors (or very close). Could be an actual Turin in a Cosmo, or a close-to-Turin client Zen 5 part.", + "type": "string", + "enum": [ + "amd_turin" + ] + }, + { + "description": "AMD Turin Dense processors. There are no \"Turin Dense-like\" CPUs unlike other cases, so this means a bona fide Zen 5c Turin Dense part.", + "type": "string", + "enum": [ + "amd_turin_dense" + ] + } + ] + }, + "SledDiagnosticsQueryOutput": { + "oneOf": [ + { + "type": "object", + "properties": { + "success": { + "type": "object", + "properties": { + "command": { + "description": "The command and its arguments.", + "type": "string" + }, + "exit_code": { + "nullable": true, + "description": "The exit code if one was present when the command exited.", + "type": "integer", + "format": "int32" + }, + "exit_status": { + "description": "The exit status of the command. This will be the exit code (if any) and exit reason such as from a signal.", + "type": "string" + }, + "stdio": { + "description": "Any stdout/stderr produced by the command.", + "type": "string" + } + }, + "required": [ + "command", + "exit_status", + "stdio" + ] + } + }, + "required": [ + "success" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "failure": { + "type": "object", + "properties": { + "error": { + "description": "The reason the command failed to execute.", + "type": "string" + } + }, + "required": [ + "error" + ] + } + }, + "required": [ + "failure" + ], + "additionalProperties": false + } + ] + }, + "SledIdentifiers": { + "description": "Identifiers for a single sled.\n\nThis is intended primarily to be used in timeseries, to identify sled from which metric data originates.", + "type": "object", + "properties": { + "model": { + "description": "Model name of the sled", + "type": "string" + }, + "rack_id": { + "description": "Control plane ID of the rack this sled is a member of", + "type": "string", + "format": "uuid" + }, + "revision": { + "description": "Revision number of the sled", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "serial": { + "description": "Serial number of the sled", + "type": "string" + }, + "sled_id": { + "description": "Control plane ID for the sled itself", + "type": "string", + "format": "uuid" + } + }, + "required": [ + "model", + "rack_id", + "revision", + "serial", + "sled_id" + ] + }, + "SledRole": { + "description": "Describes the role of the sled within the rack.\n\nNote that this may change if the sled is physically moved within the rack.", + "oneOf": [ + { + "description": "The sled is a general compute sled.", + "type": "string", + "enum": [ + "gimlet" + ] + }, + { + "description": "The sled is attached to the network switch, and has additional responsibilities.", + "type": "string", + "enum": [ + "scrimlet" + ] + } + ] + }, + "SledUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::SledUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, + "SledVmmState": { + "description": "A wrapper type containing a sled's total knowledge of the state of a VMM.", + "type": "object", + "properties": { + "migration_in": { + "nullable": true, + "description": "The current state of any inbound migration to this VMM.", + "allOf": [ + { + "$ref": "#/components/schemas/MigrationRuntimeState" + } + ] + }, + "migration_out": { + "nullable": true, + "description": "The state of any outbound migration from this VMM.", + "allOf": [ + { + "$ref": "#/components/schemas/MigrationRuntimeState" + } + ] + }, + "vmm_state": { + "description": "The most recent state of the sled's VMM process.", + "allOf": [ + { + "$ref": "#/components/schemas/VmmRuntimeState" + } + ] + } + }, + "required": [ + "vmm_state" + ] + }, + "SoftNpuP9": { + "description": "Describes a PCI device that shares host files with the guest using the P9 protocol.\n\nThis is only supported by Propolis servers compiled with the `falcon` feature.", + "type": "object", + "properties": { + "pci_path": { + "description": "The PCI path at which to attach the guest to this port.", + "allOf": [ + { + "$ref": "#/components/schemas/PciPath" + } + ] + } + }, + "required": [ + "pci_path" + ], + "additionalProperties": false + }, + "SoftNpuPciPort": { + "description": "Describes a SoftNPU PCI device.\n\nThis is only supported by Propolis servers compiled with the `falcon` feature.", + "type": "object", + "properties": { + "pci_path": { + "description": "The PCI path at which to attach the guest to this port.", + "allOf": [ + { + "$ref": "#/components/schemas/PciPath" + } + ] + } + }, + "required": [ + "pci_path" + ], + "additionalProperties": false + }, + "SoftNpuPort": { + "description": "Describes a port in a SoftNPU emulated ASIC.\n\nThis is only supported by Propolis servers compiled with the `falcon` feature.", + "type": "object", + "properties": { + "backend_id": { + "description": "The name of the port's associated DLPI backend.", + "allOf": [ + { + "$ref": "#/components/schemas/SpecKey" + } + ] + }, + "link_name": { + "description": "The data link name for this port.", + "type": "string" + } + }, + "required": [ + "backend_id", + "link_name" + ], + "additionalProperties": false + }, + "SourceNatConfigGeneric": { + "description": "An IP address and port range used for source NAT, i.e., making outbound network connections from guests or services.", + "type": "object", + "properties": { + "first_port": { + "description": "The first port used for source NAT, inclusive.", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "ip": { + "description": "The external address provided to the instance or service.", + "type": "string", + "format": "ip" + }, + "last_port": { + "description": "The last port used for source NAT, also inclusive.", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "first_port", + "ip", + "last_port" + ] + }, + "SourceNatConfigV4": { + "description": "An IP address and port range used for source NAT, i.e., making outbound network connections from guests or services.", + "type": "object", + "properties": { + "first_port": { + "description": "The first port used for source NAT, inclusive.", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "ip": { + "description": "The external address provided to the instance or service.", + "type": "string", + "format": "ipv4" + }, + "last_port": { + "description": "The last port used for source NAT, also inclusive.", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "first_port", + "ip", + "last_port" + ] + }, + "SourceNatConfigV6": { + "description": "An IP address and port range used for source NAT, i.e., making outbound network connections from guests or services.", + "type": "object", + "properties": { + "first_port": { + "description": "The first port used for source NAT, inclusive.", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "ip": { + "description": "The external address provided to the instance or service.", + "type": "string", + "format": "ipv6" + }, + "last_port": { + "description": "The last port used for source NAT, also inclusive.", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "first_port", + "ip", + "last_port" + ] + }, + "SpecKey": { + "description": "A key identifying a component in an instance spec.", + "oneOf": [ + { + "title": "uuid", + "allOf": [ + { + "type": "string", + "format": "uuid" + } + ] + }, + { + "title": "name", + "allOf": [ + { + "type": "string" + } + ] + } + ] + }, + "StartSledAgentRequest": { + "description": "Configuration information for launching a Sled Agent.", + "type": "object", + "properties": { + "body": { + "$ref": "#/components/schemas/StartSledAgentRequestBody" + }, + "generation": { + "description": "The current generation number of data as stored in CRDB.\n\nThe initial generation is set during RSS time and then only mutated by Nexus. For now, we don't actually anticipate mutating this data, but we leave open the possiblity.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "schema_version": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "body", + "generation", + "schema_version" + ] + }, + "StartSledAgentRequestBody": { + "description": "This is the actual app level data of `StartSledAgentRequest`\n\nWe nest it below the \"header\" of `generation` and `schema_version` so that we can perform partial deserialization of `EarlyNetworkConfig` to only read the header and defer deserialization of the body once we know the schema version. This is possible via the use of [`serde_json::value::RawValue`] in future (post-v1) deserialization paths.", + "type": "object", + "properties": { + "id": { + "description": "Uuid of the Sled Agent to be created.", + "allOf": [ + { + "$ref": "#/components/schemas/SledUuid" + } + ] + }, + "is_lrtq_learner": { + "description": "Is this node an LRTQ learner node?\n\nWe only put the node into learner mode if `use_trust_quorum` is also true.", + "type": "boolean" + }, + "rack_id": { + "description": "Uuid of the rack to which this sled agent belongs.", + "type": "string", + "format": "uuid" + }, + "subnet": { + "description": "Portion of the IP space to be managed by the Sled Agent.", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Subnet" + } + ] + }, + "use_trust_quorum": { + "description": "Use trust quorum for key generation", + "type": "boolean" + } + }, + "required": [ + "id", + "is_lrtq_learner", + "rack_id", + "subnet", + "use_trust_quorum" + ] + }, + "StorageLimit": { + "description": "The limit on space allowed for zone bundles, as a percentage of the overall dataset's quota.", + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "SupportBundleMetadata": { + "description": "Metadata about a support bundle.", + "type": "object", + "properties": { + "state": { + "$ref": "#/components/schemas/SupportBundleState" + }, + "support_bundle_id": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + "required": [ + "state", + "support_bundle_id" + ] + }, + "SupportBundleState": { + "description": "State of a support bundle.", + "type": "string", + "enum": [ + "complete", + "incomplete" + ] + }, + "SupportBundleUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::SupportBundleUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, + "SvcInMaintenance": { + "description": "Information about an SMF service that is enabled but not running", + "type": "object", + "properties": { + "fmri": { + "type": "string" + }, + "zone": { + "type": "string" + } + }, + "required": [ + "fmri", + "zone" + ] + }, + "SvcsInMaintenanceResult": { + "description": "Lists services in maintenance status if any, and the time the health check for SMF services ran", + "type": "object", + "properties": { + "errors": { + "type": "array", + "items": { + "type": "string" + } + }, + "services": { + "type": "array", + "items": { + "$ref": "#/components/schemas/SvcInMaintenance" + } + }, + "time_of_status": { + "nullable": true, + "type": "string", + "format": "date-time" + } + }, + "required": [ + "errors", + "services" + ] + }, + "SwitchLocation": { + "description": "Identifies switch physical location", + "oneOf": [ + { + "description": "Switch in upper slot", + "type": "string", + "enum": [ + "switch0" + ] + }, + { + "description": "Switch in lower slot", + "type": "string", + "enum": [ + "switch1" + ] + } + ] + }, + "SwitchPorts": { + "description": "A set of switch uplinks.", + "type": "object", + "properties": { + "uplinks": { + "type": "array", + "items": { + "$ref": "#/components/schemas/HostPortConfig" + } + } + }, + "required": [ + "uplinks" + ] + }, + "TrustQuorumNetworkConfig": { + "description": "Network configuration used to bring up the control plane.\n\nThis type mirrors `bootstore::schemes::v0::NetworkConfig` but adds `JsonSchema` for API compatibility.", + "type": "object", + "properties": { + "blob": { + "description": "A serialized blob of configuration data (base64 encoded).", + "type": "string" + }, + "generation": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "blob", + "generation" + ] + }, + "TxEqConfig": { + "description": "Per-port tx-eq overrides. This can be used to fine-tune the transceiver equalization settings to improve signal integrity.", + "type": "object", + "properties": { + "main": { + "nullable": true, + "description": "Main tap", + "type": "integer", + "format": "int32" + }, + "post1": { + "nullable": true, + "description": "Post-cursor tap1", + "type": "integer", + "format": "int32" + }, + "post2": { + "nullable": true, + "description": "Post-cursor tap2", + "type": "integer", + "format": "int32" + }, + "pre1": { + "nullable": true, + "description": "Pre-cursor tap1", + "type": "integer", + "format": "int32" + }, + "pre2": { + "nullable": true, + "description": "Pre-cursor tap2", + "type": "integer", + "format": "int32" + } + } + }, + "UplinkAddressConfig": { + "type": "object", + "properties": { + "address": { + "$ref": "#/components/schemas/IpNet" + }, + "vlan_id": { + "nullable": true, + "description": "The VLAN id (if any) associated with this address.", + "default": null, + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "address" + ] + }, + "VirtioDisk": { + "description": "A disk that presents a virtio-block interface to the guest.", + "type": "object", + "properties": { + "backend_id": { + "description": "The name of the disk's backend component.", + "allOf": [ + { + "$ref": "#/components/schemas/SpecKey" + } + ] + }, + "pci_path": { + "description": "The PCI bus/device/function at which this disk should be attached.", + "allOf": [ + { + "$ref": "#/components/schemas/PciPath" + } + ] + } + }, + "required": [ + "backend_id", + "pci_path" + ], + "additionalProperties": false + }, + "VirtioNetworkBackend": { + "description": "A network backend associated with a virtio-net (viona) VNIC on the host.", + "type": "object", + "properties": { + "vnic_name": { + "description": "The name of the viona VNIC to use as a backend.", + "type": "string" + } + }, + "required": [ + "vnic_name" + ], + "additionalProperties": false + }, + "VirtioNic": { + "description": "A network card that presents a virtio-net interface to the guest.", + "type": "object", + "properties": { + "backend_id": { + "description": "The name of the device's backend.", + "allOf": [ + { + "$ref": "#/components/schemas/SpecKey" + } + ] + }, + "interface_id": { + "description": "A caller-defined correlation identifier for this interface. If Propolis is configured to collect network interface kstats in its Oximeter metrics, the metric series for this interface will be associated with this identifier.", + "type": "string", + "format": "uuid" + }, + "pci_path": { + "description": "The PCI path at which to attach this device.", + "allOf": [ + { + "$ref": "#/components/schemas/PciPath" + } + ] + } + }, + "required": [ + "backend_id", + "interface_id", + "pci_path" + ], + "additionalProperties": false + }, + "VirtualNetworkInterfaceHost": { + "description": "A mapping from a virtual NIC to a physical host", + "type": "object", + "properties": { + "physical_host_ip": { + "type": "string", + "format": "ipv6" + }, + "virtual_ip": { + "type": "string", + "format": "ip" + }, + "virtual_mac": { + "$ref": "#/components/schemas/MacAddr" + }, + "vni": { + "$ref": "#/components/schemas/Vni" + } + }, + "required": [ + "physical_host_ip", + "virtual_ip", + "virtual_mac", + "vni" + ] + }, + "VmmIssueDiskSnapshotRequestBody": { + "description": "Request body for VMM disk snapshot requests.", + "type": "object", + "properties": { + "snapshot_id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "snapshot_id" + ] + }, + "VmmIssueDiskSnapshotRequestResponse": { + "description": "Response for VMM disk snapshot requests.", + "type": "object", + "properties": { + "snapshot_id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "snapshot_id" + ] + }, + "VmmPutStateBody": { + "description": "The body of a request to move a previously-ensured instance into a specific runtime state.", + "type": "object", + "properties": { + "state": { + "description": "The state into which the instance should be driven.", + "allOf": [ + { + "$ref": "#/components/schemas/VmmStateRequested" + } + ] + } + }, + "required": [ + "state" + ] + }, + "VmmPutStateResponse": { + "description": "The response sent from a request to move an instance into a specific runtime state.", + "type": "object", + "properties": { + "updated_runtime": { + "nullable": true, + "description": "The current runtime state of the instance after handling the request to change its state. If the instance's state did not change, this field is `None`.", + "allOf": [ + { + "$ref": "#/components/schemas/SledVmmState" + } + ] + } + } + }, + "VmmRuntimeState": { + "description": "The dynamic runtime properties of an individual VMM process.", + "type": "object", + "properties": { + "gen": { + "description": "The generation number for this VMM's state.", + "allOf": [ + { + "$ref": "#/components/schemas/Generation" + } + ] + }, + "state": { + "description": "The last state reported by this VMM.", + "allOf": [ + { + "$ref": "#/components/schemas/VmmState" + } + ] + }, + "time_updated": { + "description": "Timestamp for the VMM's state.", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "gen", + "state", + "time_updated" + ] + }, + "VmmSpec": { + "description": "Specifies the virtual hardware configuration of a new Propolis VMM in the form of a Propolis instance specification.", + "allOf": [ + { + "$ref": "#/components/schemas/InstanceSpecV0" + } + ] + }, + "VmmState": { + "description": "One of the states that a VMM can be in.", + "oneOf": [ + { + "description": "The VMM is initializing and has not started running guest CPUs yet.", + "type": "string", + "enum": [ + "starting" + ] + }, + { + "description": "The VMM has finished initializing and may be running guest CPUs.", + "type": "string", + "enum": [ + "running" + ] + }, + { + "description": "The VMM is shutting down.", + "type": "string", + "enum": [ + "stopping" + ] + }, + { + "description": "The VMM's guest has stopped, and the guest will not run again, but the VMM process may not have released all of its resources yet.", + "type": "string", + "enum": [ + "stopped" + ] + }, + { + "description": "The VMM is being restarted or its guest OS is rebooting.", + "type": "string", + "enum": [ + "rebooting" + ] + }, + { + "description": "The VMM is part of a live migration.", + "type": "string", + "enum": [ + "migrating" + ] + }, + { + "description": "The VMM process reported an internal failure.", + "type": "string", + "enum": [ + "failed" + ] + }, + { + "description": "The VMM process has been destroyed and its resources have been released.", + "type": "string", + "enum": [ + "destroyed" + ] + } + ] + }, + "VmmStateRequested": { + "description": "Requestable running state of an Instance.\n\nA subset of [`omicron_common::api::external::InstanceState`].", + "oneOf": [ + { + "description": "Run this instance by migrating in from a previous running incarnation of the instance.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "migration_target" + ] + }, + "value": { + "$ref": "#/components/schemas/InstanceMigrationTargetParams" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Start the instance if it is not already running.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "running" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Stop the instance.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "stopped" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Immediately reset the instance, as though it had stopped and immediately began to run again.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "reboot" + ] + } + }, + "required": [ + "type" + ] + } + ] + }, + "VmmUnregisterResponse": { + "description": "The response sent from a request to unregister an instance.", + "type": "object", + "properties": { + "updated_runtime": { + "nullable": true, + "description": "The current state of the instance after handling the request to unregister it. If the instance's state did not change, this field is `None`.", + "allOf": [ + { + "$ref": "#/components/schemas/SledVmmState" + } + ] + } + } + }, + "Vni": { + "description": "A Geneve Virtual Network Identifier", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "VpcFirewallIcmpFilter": { + "type": "object", + "properties": { + "code": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/IcmpParamRange" + } + ] + }, + "icmp_type": { + "type": "integer", + "format": "uint8", + "minimum": 0 + } + }, + "required": [ + "icmp_type" + ] + }, + "VpcFirewallRuleAction": { + "type": "string", + "enum": [ + "allow", + "deny" + ] + }, + "VpcFirewallRuleDirection": { + "type": "string", + "enum": [ + "inbound", + "outbound" + ] + }, + "VpcFirewallRuleProtocol": { + "description": "The protocols that may be specified in a firewall rule's filter", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "tcp" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "udp" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "icmp" + ] + }, + "value": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/VpcFirewallIcmpFilter" + } + ] + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, + "VpcFirewallRuleStatus": { + "type": "string", + "enum": [ + "disabled", + "enabled" + ] + }, + "VpcFirewallRulesEnsureBody": { + "description": "Update firewall rules for a VPC", + "type": "object", + "properties": { + "rules": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ResolvedVpcFirewallRule" + } + }, + "vni": { + "$ref": "#/components/schemas/Vni" + } + }, + "required": [ + "rules", + "vni" + ] + }, + "ZoneArtifactInventory": { + "description": "Inventory representation of a single zone artifact on a boot disk.\n\nPart of [`ManifestBootInventory`].", + "type": "object", + "properties": { + "expected_hash": { + "description": "The expected digest of the file's contents.", + "type": "string", + "format": "hex string (32 bytes)" + }, + "expected_size": { + "description": "The expected size of the file, in bytes.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "file_name": { + "description": "The name of the zone file on disk, for example `nexus.tar.gz`. Zone files are always \".tar.gz\".", + "type": "string" + }, + "path": { + "description": "The full path to the zone file.", + "type": "string", + "format": "Utf8PathBuf" + }, + "status": { + "description": "The status of the artifact.\n\nThis is `Ok(())` if the artifact is present and matches the expected size and digest, or an error message if it is missing or does not match.", + "x-rust-type": { + "crate": "std", + "parameters": [ + { + "type": "null" + }, + { + "type": "string" + } + ], + "path": "::std::result::Result", + "version": "*" + }, + "oneOf": [ + { + "type": "object", + "properties": { + "ok": { + "type": "string", + "enum": [ + null + ] + } + }, + "required": [ + "ok" + ] + }, + { + "type": "object", + "properties": { + "err": { + "type": "string" + } + }, + "required": [ + "err" + ] + } + ] + } + }, + "required": [ + "expected_hash", + "expected_size", + "file_name", + "path", + "status" + ] + }, + "ZoneBundleCause": { + "description": "The reason or cause for a zone bundle, i.e., why it was created.", + "oneOf": [ + { + "description": "Some other, unspecified reason.", + "type": "string", + "enum": [ + "other" + ] + }, + { + "description": "A zone bundle taken when a sled agent finds a zone that it does not expect to be running.", + "type": "string", + "enum": [ + "unexpected_zone" + ] + }, + { + "description": "An instance zone was terminated.", + "type": "string", + "enum": [ + "terminated_instance" + ] + } + ] + }, + "ZoneBundleId": { + "description": "An identifier for a zone bundle.", + "type": "object", + "properties": { + "bundle_id": { + "description": "The ID for this bundle itself.", + "type": "string", + "format": "uuid" + }, + "zone_name": { + "description": "The name of the zone this bundle is derived from.", + "type": "string" + } + }, + "required": [ + "bundle_id", + "zone_name" + ] + }, + "ZoneBundleMetadata": { + "description": "Metadata about a zone bundle.", + "type": "object", + "properties": { + "cause": { + "description": "The reason or cause a bundle was created.", + "allOf": [ + { + "$ref": "#/components/schemas/ZoneBundleCause" + } + ] + }, + "id": { + "description": "Identifier for this zone bundle", + "allOf": [ + { + "$ref": "#/components/schemas/ZoneBundleId" + } + ] + }, + "time_created": { + "description": "The time at which this zone bundle was created.", + "type": "string", + "format": "date-time" + }, + "version": { + "description": "A version number for this zone bundle.", + "type": "integer", + "format": "uint8", + "minimum": 0 + } + }, + "required": [ + "cause", + "id", + "time_created", + "version" + ] + }, + "ZpoolName": { + "title": "The name of a Zpool", + "description": "Zpool names are of the format ox{i,p}_. They are either Internal or External, and should be unique", + "type": "string", + "pattern": "^ox[ip]_[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" + }, + "ZpoolUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::ZpoolUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, + "PropolisUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::PropolisUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } +} diff --git a/openapi/sled-agent/sled-agent-latest.json b/openapi/sled-agent/sled-agent-latest.json index bb268bedb1f..a8812810d80 120000 --- a/openapi/sled-agent/sled-agent-latest.json +++ b/openapi/sled-agent/sled-agent-latest.json @@ -1 +1 @@ -sled-agent-17.0.0-cb6649.json \ No newline at end of file +sled-agent-18.0.0-ee9451.json \ No newline at end of file diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index 41d857dac5d..67f3bc33635 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -20,7 +20,7 @@ use omicron_common::api::internal::{ }, }; use sled_agent_types_versions::{ - latest, v1, v4, v6, v7, v9, v10, v11, v12, v14, + latest, v1, v4, v6, v7, v9, v10, v11, v12, v14, v17, }; use sled_diagnostics::SledDiagnosticsQueryOutput; @@ -36,6 +36,7 @@ api_versions!([ // | example for the next person. // v // (next_int, IDENT), + (18, ADD_ATTACHED_SUBNETS), (17, TWO_TYPES_OF_DELEGATED_ZVOL), (16, MEASUREMENT_PROPER_INVENTORY), (15, ADD_TRUST_QUORUM_STATUS), @@ -419,7 +420,7 @@ pub trait SledAgentApi { operation_id = "vmm_register", method = PUT, path = "/vmms/{propolis_id}", - versions = VERSION_TWO_TYPES_OF_DELEGATED_ZVOL.. + versions = VERSION_ADD_ATTACHED_SUBNETS.. }] async fn vmm_register( rqctx: RequestContext, @@ -427,6 +428,21 @@ pub trait SledAgentApi { body: TypedBody, ) -> Result, HttpError>; + #[endpoint { + operation_id = "vmm_register", + method = PUT, + path = "/vmms/{propolis_id}", + versions = + VERSION_TWO_TYPES_OF_DELEGATED_ZVOL..VERSION_ADD_ATTACHED_SUBNETS + }] + async fn vmm_register_v17( + rqctx: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result, HttpError> { + Self::vmm_register(rqctx, path_params, body.map(Into::into)).await + } + #[endpoint { operation_id = "vmm_register", method = PUT, @@ -438,7 +454,7 @@ pub trait SledAgentApi { path_params: Path, body: TypedBody, ) -> Result, HttpError> { - Self::vmm_register(rqctx, path_params, body.map(Into::into)).await + Self::vmm_register_v17(rqctx, path_params, body.map(Into::into)).await } #[endpoint { @@ -454,7 +470,7 @@ pub trait SledAgentApi { body: TypedBody, ) -> Result, HttpError> { let body = body.try_map(v11::instance::InstanceEnsureBody::try_from)?; - Self::vmm_register(rqctx, path_params, body.map(Into::into)).await + Self::vmm_register_v11(rqctx, path_params, body).await } #[endpoint { @@ -1312,4 +1328,50 @@ pub trait SledAgentApi { request_context: RequestContext, body: TypedBody, ) -> Result; + + /// Update the subnets attached to an instance. + #[endpoint { + method = PUT, + path = "/vmms/{propolis_id}/attached-subnets", + versions = VERSION_ADD_ATTACHED_SUBNETS.., + }] + async fn vmm_put_attached_subnets( + request_context: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result; + + /// Delete all subnets attached to an instance. + #[endpoint { + method = DELETE, + path = "/vmms/{propolis_id}/attached-subnets", + versions = VERSION_ADD_ATTACHED_SUBNETS.., + }] + async fn vmm_delete_attached_subnets( + request_context: RequestContext, + path_params: Path, + ) -> Result; + + /// Attach a subnet to an instance. + #[endpoint { + method = POST, + path = "/vmms/{propolis_id}/attached-subnets", + versions = VERSION_ADD_ATTACHED_SUBNETS.., + }] + async fn vmm_post_attached_subnet( + request_context: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result; + + /// Detach a subnet from an instance. + #[endpoint { + method = DELETE, + path = "/vmms/{propolis_id}/attached-subnets/{subnet}", + versions = VERSION_ADD_ATTACHED_SUBNETS.., + }] + async fn vmm_delete_attached_subnet( + request_context: RequestContext, + path_params: Path, + ) -> Result; } diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 042d1d49e20..725f421b6ab 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -29,6 +29,9 @@ use sled_agent_types::artifact::{ ArtifactListResponse, ArtifactPathParam, ArtifactPutResponse, ArtifactQueryParam, }; +use sled_agent_types::attached_subnet::{ + AttachedSubnet, AttachedSubnets, VmmSubnetPathParam, +}; use sled_agent_types::bootstore::BootstoreStatus; use sled_agent_types::dataset::{ LocalStorageDatasetDeleteRequest, LocalStorageDatasetEnsureRequest, @@ -1383,4 +1386,56 @@ impl SledAgentApi for SledAgentImpl { Ok(HttpResponseUpdatedNoContent()) } + + async fn vmm_put_attached_subnets( + request_context: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result { + let sa = request_context.context(); + let propolis_id = path_params.into_inner().propolis_id; + sa.instance_put_attached_subnets(propolis_id, body.into_inner()) + .await + .map(|_| HttpResponseUpdatedNoContent()) + .map_err(HttpError::from) + } + + async fn vmm_delete_attached_subnets( + request_context: RequestContext, + path_params: Path, + ) -> Result { + let sa = request_context.context(); + let propolis_id = path_params.into_inner().propolis_id; + sa.instance_delete_attached_subnets(propolis_id) + .await + .map(|_| HttpResponseDeleted()) + .map_err(HttpError::from) + } + + async fn vmm_post_attached_subnet( + request_context: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result { + let sa = request_context.context(); + let propolis_id = path_params.into_inner().propolis_id; + let subnet = body.into_inner(); + sa.instance_attach_subnet(propolis_id, subnet) + .await + .map(|_| HttpResponseUpdatedNoContent()) + .map_err(HttpError::from) + } + + async fn vmm_delete_attached_subnet( + request_context: RequestContext, + path_params: Path, + ) -> Result { + let sa = request_context.context(); + let VmmSubnetPathParam { propolis_id, subnet } = + path_params.into_inner(); + sa.instance_detach_subnet(propolis_id, subnet) + .await + .map(|_| HttpResponseDeleted()) + .map_err(HttpError::from) + } } diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 2227ce03780..bc6e39fb46d 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -16,9 +16,13 @@ use crate::profile::*; use crate::zone_bundle::ZoneBundler; use chrono::Utc; +use iddqd::IdOrdMap; +use iddqd::id_ord_map::Entry; use illumos_utils::dladm::Etherstub; use illumos_utils::link::VnicAllocator; -use illumos_utils::opte::{DhcpCfg, PortCreateParams, PortManager}; +use illumos_utils::opte::{ + DhcpCfg, PortCreateParams, PortManager, net_to_cidr, +}; use illumos_utils::running_zone::{RunningZone, ZoneBuilderFactory}; use illumos_utils::zone::PROPOLIS_ZONE_PREFIX; use illumos_utils::zpool::ZpoolOrRamdisk; @@ -33,6 +37,7 @@ use omicron_common::zpool_name::ZpoolName; use omicron_uuid_kinds::{ GenericUuid, InstanceUuid, OmicronZoneUuid, PropolisUuid, }; +use oxnet::IpNet; use propolis_api_types::ErrorCode as PropolisErrorCode; use propolis_client::Client as PropolisClient; use propolis_client::instance_spec::{ @@ -42,6 +47,7 @@ use rand::SeedableRng; use rand::prelude::IteratorRandom; use sled_agent_config_reconciler::AvailableDatasetsReceiver; use sled_agent_resolvable_files::ramdisk_file_source; +use sled_agent_types::attached_subnet::{AttachedSubnet, AttachedSubnets}; use sled_agent_types::instance::*; use sled_agent_types::zone_bundle::ZoneBundleCause; use slog::Logger; @@ -139,6 +145,9 @@ pub enum Error { #[error("Instance is terminating")] Terminating, + + #[error("The IP subnet {0} is already attached")] + SubnetAlreadyAttached(IpNet), } type PropolisClientError = @@ -254,6 +263,21 @@ enum InstanceRequest { RefreshMulticastGroups { tx: oneshot::Sender>, }, + SetAttachedSubnets { + subnets: AttachedSubnets, + tx: oneshot::Sender>, + }, + ClearAttachedSubnets { + tx: oneshot::Sender>, + }, + AttachSubnet { + subnet: AttachedSubnet, + tx: oneshot::Sender>, + }, + DetachSubnet { + subnet: IpNet, + tx: oneshot::Sender>, + }, } impl InstanceRequest { @@ -298,7 +322,11 @@ impl InstanceRequest { | Self::RefreshExternalIps { tx } | Self::JoinMulticastGroup { tx, .. } | Self::LeaveMulticastGroup { tx, .. } - | Self::RefreshMulticastGroups { tx } => tx + | Self::RefreshMulticastGroups { tx } + | Self::SetAttachedSubnets { tx, .. } + | Self::ClearAttachedSubnets { tx } + | Self::AttachSubnet { tx, .. } + | Self::DetachSubnet { tx, .. } => tx .send(Err(error.into())) .map_err(|_| Error::FailedSendClientClosed), } @@ -564,6 +592,9 @@ struct InstanceRunner { // Zvols to delegate to the Propolis zone delegated_zvols: Vec, + + // Subnets attached to this instance. + attached_subnets: IdOrdMap, } /// Translate a `propolis-client` `InstanceSpecV0` into the newer @@ -757,6 +788,22 @@ impl InstanceRunner { tx.send(self.refresh_multicast_groups().map_err(|e| e.into())) .map_err(|_| Error::FailedSendClientClosed) } + SetAttachedSubnets { tx, subnets } => { + tx.send(self.set_attached_subnets(subnets).map_err(|e| e.into())) + .map_err(|_| Error::FailedSendClientClosed) + } + ClearAttachedSubnets { tx } => { + tx.send(self.clear_attached_subnets().map_err(|e| e.into())) + .map_err(|_| Error::FailedSendClientClosed) + } + AttachSubnet { tx, subnet } => { + tx.send(self.attach_subnet(subnet).map_err(|e| e.into())) + .map_err(|_| Error::FailedSendClientClosed) + } + DetachSubnet { tx, subnet } => { + tx.send(self.detach_subnet(subnet).map_err(|e| e.into())) + .map_err(|_| Error::FailedSendClientClosed) + } } }; tokio::select! { @@ -863,6 +910,18 @@ impl InstanceRunner { RefreshMulticastGroups { tx } => { tx.send(Err(Error::Terminating.into())).map_err(|_| ()) } + SetAttachedSubnets { tx, .. } => { + tx.send(Err(Error::Terminating.into())).map_err(|_| ()) + } + ClearAttachedSubnets { tx } => { + tx.send(Err(Error::Terminating.into())).map_err(|_| ()) + } + AttachSubnet { tx, .. } => { + tx.send(Err(Error::Terminating.into())).map_err(|_| ()) + } + DetachSubnet { tx, .. } => { + tx.send(Err(Error::Terminating.into())).map_err(|_| ()) + } }; } @@ -1580,6 +1639,113 @@ impl InstanceRunner { fn primary_nic(&self) -> Option<&NetworkInterface> { self.requested_nics.iter().find(|nic| nic.primary) } + + /// Replace the set of attached subnets, setting it to exactly the input. + fn set_attached_subnets( + &mut self, + subnets: AttachedSubnets, + ) -> Result<(), Error> { + let Some(primary_nic) = self.primary_nic() else { + return Err(Error::Opte(illumos_utils::opte::Error::NoPrimaryNic)); + }; + + // OPTE itself either inserts a new subnet, or updates the mapping it + // currently has, so we can "add" the full set here. + // + // We do need to compute the set of subnets we want to remove, which are + // just those we have locally, but not in the requested set. + let requested = subnets.subnets; + let to_remove = self + .attached_subnets + .iter() + .filter_map(|s| match requested.get(&s.subnet) { + Some(_) => None, + None => Some(net_to_cidr(s.subnet)), + }) + .collect::>(); + let to_ensure = requested + .iter() + .map(|s| illumos_utils::opte::AttachedSubnet { + cidr: net_to_cidr(s.subnet), + is_external: s.is_external, + }) + .collect::>(); + let nic_id = primary_nic.id; + let nic_kind = primary_nic.kind; + self.port_manager + .attached_subnets_ensure(nic_id, nic_kind, to_remove, to_ensure) + .map_err(Error::from)?; + self.attached_subnets = requested; + Ok(()) + } + + /// Delete all attached subnets. + fn clear_attached_subnets(&mut self) -> Result<(), Error> { + let Some(primary_nic) = self.primary_nic() else { + return Err(Error::Opte(illumos_utils::opte::Error::NoPrimaryNic)); + }; + let nic_id = primary_nic.id; + let nic_kind = primary_nic.kind; + let to_remove = self + .attached_subnets + .iter() + .map(|s| net_to_cidr(s.subnet)) + .collect(); + self.port_manager + .attached_subnets_ensure(nic_id, nic_kind, to_remove, vec![]) + .map_err(Error::from)?; + self.attached_subnets.clear(); + Ok(()) + } + + /// Attach a single subnet. + /// + /// This returns an error if a subnet with the matching IP CIDR is already + /// attached. Detach it first. + fn attach_subnet(&mut self, subnet: AttachedSubnet) -> Result<(), Error> { + let Some((nic_id, nic_kind)) = + self.primary_nic().map(|nic| (nic.id, nic.kind)) + else { + return Err(Error::Opte(illumos_utils::opte::Error::NoPrimaryNic)); + }; + let entry = match self.attached_subnets.entry(&subnet.subnet) { + Entry::Vacant(entry) => entry, + Entry::Occupied(_) => { + return Err(Error::SubnetAlreadyAttached(subnet.subnet)); + } + }; + let subnet_ = illumos_utils::opte::AttachedSubnet { + cidr: net_to_cidr(subnet.subnet), + is_external: subnet.is_external, + }; + self.port_manager + .attach_subnet(nic_id, nic_kind, subnet_) + .map_err(Error::from)?; + entry.insert(subnet); + Ok(()) + } + + /// Detach a single subnet. + /// + /// This is idempotent, returning successfully even if the subnet is gone. + fn detach_subnet(&mut self, subnet: IpNet) -> Result<(), Error> { + let Some((nic_id, nic_kind)) = + self.primary_nic().map(|nic| (nic.id, nic.kind)) + else { + return Err(Error::Opte(illumos_utils::opte::Error::NoPrimaryNic)); + }; + let Some(entry) = self.attached_subnets.remove(&subnet) else { + return Ok(()); + }; + let res = self + .port_manager + .detach_subnet(nic_id, nic_kind, net_to_cidr(subnet)) + .map_err(Error::from); + if res.is_err() { + let _ = self.attached_subnets.insert_unique(entry); + } + res + } } fn propolis_error_code( @@ -1803,6 +1969,7 @@ impl Instance { zone_bundler, metrics_queue, delegated_zvols: local_config.delegated_zvols, + attached_subnets: IdOrdMap::new(), }; let runner_handle = tokio::task::spawn(async move { @@ -1963,6 +2130,45 @@ impl Instance { .try_send(InstanceRequest::RefreshMulticastGroups { tx }) .or_else(InstanceRequest::fail_try_send) } + + pub(crate) fn set_attached_subnets( + &self, + tx: oneshot::Sender>, + subnets: AttachedSubnets, + ) -> Result<(), Error> { + self.tx + .try_send(InstanceRequest::SetAttachedSubnets { subnets, tx }) + .or_else(InstanceRequest::fail_try_send) + } + + pub(crate) fn clear_attached_subnets( + &self, + tx: oneshot::Sender>, + ) -> Result<(), Error> { + self.tx + .try_send(InstanceRequest::ClearAttachedSubnets { tx }) + .or_else(InstanceRequest::fail_try_send) + } + + pub(crate) fn attach_subnet( + &self, + tx: oneshot::Sender>, + subnet: AttachedSubnet, + ) -> Result<(), Error> { + self.tx + .try_send(InstanceRequest::AttachSubnet { subnet, tx }) + .or_else(InstanceRequest::fail_try_send) + } + + pub(crate) fn detach_subnet( + &self, + tx: oneshot::Sender>, + subnet: IpNet, + ) -> Result<(), Error> { + self.tx + .try_send(InstanceRequest::DetachSubnet { subnet, tx }) + .or_else(InstanceRequest::fail_try_send) + } } // TODO: Move this implementation higher. I'm just keeping it here to make the @@ -2161,9 +2367,14 @@ impl InstanceRunner { external_ips: &self.external_ips, firewall_rules: &self.firewall_rules, dhcp_config: self.dhcp_config.clone(), - // TODO Accept these in the sled-agent API. See - // https://github.com/oxidecomputer/omicron/issues/9702. - attached_subnets: vec![], + attached_subnets: self + .attached_subnets + .iter() + .map(|att| illumos_utils::opte::AttachedSubnet { + cidr: net_to_cidr(att.subnet), + is_external: att.is_external, + }) + .collect(), })?; opte_port_names.push(port.0.name().to_string()); opte_ports.push(port); @@ -2822,6 +3033,7 @@ mod tests { search_domains: vec![], }, delegated_zvols: vec![], + attached_subnets: vec![], }; InstanceInitialState { @@ -3433,6 +3645,7 @@ mod tests { zone_bundler, metrics_queue, delegated_zvols: local_config.delegated_zvols, + attached_subnets: IdOrdMap::new(), } } } diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index 042db5e97d2..66199640643 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -20,8 +20,11 @@ use omicron_common::api::external::ByteCount; use omicron_common::api::internal::nexus::SledVmmState; use omicron_common::api::internal::shared::SledIdentifiers; use omicron_uuid_kinds::PropolisUuid; +use oxnet::IpNet; use sled_agent_config_reconciler::AvailableDatasetsReceiver; use sled_agent_config_reconciler::CurrentlyManagedZpoolsReceiver; +use sled_agent_types::attached_subnet::AttachedSubnet; +use sled_agent_types::attached_subnet::AttachedSubnets; use sled_agent_types::instance::*; use sled_agent_types::instance::{InstanceEnsureBody, InstanceMulticastBody}; use slog::Logger; @@ -356,6 +359,80 @@ impl InstanceManager { .map_err(|_| Error::FailedSendInstanceManagerClosed)?; rx.await? } + + /// Update the set of attached subnets for an instance. + pub(crate) async fn set_attached_subnets( + &self, + propolis_id: PropolisUuid, + subnets: AttachedSubnets, + ) -> Result<(), Error> { + let (tx, rx) = oneshot::channel(); + self.inner + .tx + .send(InstanceManagerRequest::SetAttachedSubnets { + propolis_id, + subnets, + tx, + }) + .await + .map_err(|_| Error::FailedSendInstanceManagerClosed)?; + rx.await? + } + + /// Delete the set of attached subnets for an instance. + pub(crate) async fn clear_attached_subnets( + &self, + propolis_id: PropolisUuid, + ) -> Result<(), Error> { + let (tx, rx) = oneshot::channel(); + self.inner + .tx + .send(InstanceManagerRequest::ClearAttachedSubnets { + propolis_id, + tx, + }) + .await + .map_err(|_| Error::FailedSendInstanceManagerClosed)?; + rx.await? + } + + /// Attach a subnet to an instance. + pub(crate) async fn attach_subnet( + &self, + propolis_id: PropolisUuid, + subnet: AttachedSubnet, + ) -> Result<(), Error> { + let (tx, rx) = oneshot::channel(); + self.inner + .tx + .send(InstanceManagerRequest::AttachSubnet { + propolis_id, + subnet, + tx, + }) + .await + .map_err(|_| Error::FailedSendInstanceManagerClosed)?; + rx.await? + } + + /// Detach a subnet from an instance + pub(crate) async fn detach_subnet( + &self, + propolis_id: PropolisUuid, + subnet: IpNet, + ) -> Result<(), Error> { + let (tx, rx) = oneshot::channel(); + self.inner + .tx + .send(InstanceManagerRequest::DetachSubnet { + propolis_id, + subnet, + tx, + }) + .await + .map_err(|_| Error::FailedSendInstanceManagerClosed)?; + rx.await? + } } // Most requests that can be sent to the "InstanceManagerRunner" task. @@ -420,6 +497,25 @@ enum InstanceManagerRequest { propolis_id: PropolisUuid, tx: oneshot::Sender>, }, + SetAttachedSubnets { + propolis_id: PropolisUuid, + subnets: AttachedSubnets, + tx: oneshot::Sender>, + }, + ClearAttachedSubnets { + propolis_id: PropolisUuid, + tx: oneshot::Sender>, + }, + AttachSubnet { + propolis_id: PropolisUuid, + subnet: AttachedSubnet, + tx: oneshot::Sender>, + }, + DetachSubnet { + propolis_id: PropolisUuid, + subnet: IpNet, + tx: oneshot::Sender>, + }, } // Requests that the instance manager stop processing information about a @@ -549,6 +645,18 @@ impl InstanceManagerRunner { // the state... self.get_instance_state(tx, propolis_id) }, + Some(SetAttachedSubnets { propolis_id, subnets, tx }) => { + self.set_attached_subnets(tx, propolis_id, subnets) + } + Some(ClearAttachedSubnets { propolis_id, tx }) =>{ + self.clear_attached_subnets(tx, propolis_id) + } + Some(AttachSubnet { propolis_id, subnet, tx }) => { + self.attach_subnet(tx, propolis_id, subnet) + } + Some(DetachSubnet { propolis_id, subnet, tx }) =>{ + self.detach_subnet(tx, propolis_id, subnet) + } None => { warn!(self.log, "InstanceManager's request channel closed; shutting down"); break; @@ -889,6 +997,53 @@ impl InstanceManagerRunner { } } } + + fn set_attached_subnets( + &self, + tx: oneshot::Sender>, + propolis_id: PropolisUuid, + subnets: AttachedSubnets, + ) -> Result<(), Error> { + let Some(instance) = self.get_propolis(propolis_id) else { + return Err(Error::NoSuchVmm(propolis_id)); + }; + instance.set_attached_subnets(tx, subnets).map_err(Error::from) + } + + fn clear_attached_subnets( + &self, + tx: oneshot::Sender>, + propolis_id: PropolisUuid, + ) -> Result<(), Error> { + let Some(instance) = self.get_propolis(propolis_id) else { + return Err(Error::NoSuchVmm(propolis_id)); + }; + instance.clear_attached_subnets(tx).map_err(Error::from) + } + + fn attach_subnet( + &self, + tx: oneshot::Sender>, + propolis_id: PropolisUuid, + subnet: AttachedSubnet, + ) -> Result<(), Error> { + let Some(instance) = self.get_propolis(propolis_id) else { + return Err(Error::NoSuchVmm(propolis_id)); + }; + instance.attach_subnet(tx, subnet).map_err(Error::from) + } + + fn detach_subnet( + &self, + tx: oneshot::Sender>, + propolis_id: PropolisUuid, + subnet: IpNet, + ) -> Result<(), Error> { + let Some(instance) = self.get_propolis(propolis_id) else { + return Err(Error::NoSuchVmm(propolis_id)); + }; + instance.detach_subnet(tx, subnet).map_err(Error::from) + } } /// Represents membership of an instance in the [`InstanceManager`]. diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index 69d6ca54952..583897afbaa 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -41,6 +41,9 @@ use sled_agent_types::artifact::{ ArtifactListResponse, ArtifactPathParam, ArtifactPutResponse, ArtifactQueryParam, }; +use sled_agent_types::attached_subnet::AttachedSubnet; +use sled_agent_types::attached_subnet::AttachedSubnets; +use sled_agent_types::attached_subnet::VmmSubnetPathParam; use sled_agent_types::bootstore::BootstoreStatus; use sled_agent_types::dataset::{ LocalStorageDatasetDeleteRequest, LocalStorageDatasetEnsureRequest, @@ -1004,6 +1007,59 @@ impl SledAgentApi for SledAgentSimImpl { ) -> Result { method_unimplemented() } + + async fn vmm_put_attached_subnets( + request_context: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result { + let sa = request_context.context(); + let id = path_params.into_inner().propolis_id; + let subnets = body.into_inner(); + sa.instance_put_attached_subnets(id, subnets) + .await + .map(|_| HttpResponseUpdatedNoContent()) + .map_err(HttpError::from) + } + + async fn vmm_delete_attached_subnets( + request_context: RequestContext, + path_params: Path, + ) -> Result { + let sa = request_context.context(); + let propolis_id = path_params.into_inner().propolis_id; + sa.instance_delete_attached_subnets(propolis_id) + .await + .map(|_| HttpResponseDeleted()) + .map_err(HttpError::from) + } + + async fn vmm_post_attached_subnet( + request_context: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result { + let sa = request_context.context(); + let id = path_params.into_inner().propolis_id; + let subnet = body.into_inner(); + sa.instance_post_attached_subnet(id, subnet) + .await + .map(|_| HttpResponseUpdatedNoContent()) + .map_err(HttpError::from) + } + + async fn vmm_delete_attached_subnet( + request_context: RequestContext, + path_params: Path, + ) -> Result { + let sa = request_context.context(); + let VmmSubnetPathParam { propolis_id, subnet } = + path_params.into_inner(); + sa.instance_delete_attached_subnet(propolis_id, subnet) + .await + .map(|_| HttpResponseDeleted()) + .map_err(HttpError::from) + } } fn method_unimplemented() -> Result { diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index b1e20d9207b..6401006bac6 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -24,6 +24,7 @@ use chrono::Utc; use dropshot::Body; use dropshot::HttpError; use futures::Stream; +use iddqd::IdOrdMap; use omicron_common::api::external::{ ByteCount, DiskState, Error, Generation, ResourceType, }; @@ -43,7 +44,7 @@ use omicron_uuid_kinds::{ DatasetUuid, GenericUuid, PhysicalDiskUuid, PropolisUuid, SledUuid, SupportBundleUuid, ZpoolUuid, }; -use oxnet::Ipv6Net; +use oxnet::{IpNet, Ipv6Net}; use propolis_client::instance_spec::FileStorageBackend; use propolis_client::instance_spec::SpecKey; use propolis_client::{ @@ -51,6 +52,7 @@ use propolis_client::{ }; use range_requests::PotentialRange; use sled_agent_health_monitor::HealthMonitorHandle; +use sled_agent_types::attached_subnet::{AttachedSubnet, AttachedSubnets}; use sled_agent_types::dataset::LocalStorageDatasetEnsureRequest; use sled_agent_types::disk::DiskStateRequested; use sled_agent_types::early_networking::{ @@ -104,6 +106,9 @@ pub struct SledAgent { /// lists of external IPs assigned to instances pub external_ips: Mutex>>, + /// subnets attached to instances. + pub attached_subnets: + Mutex>>, /// multicast group memberships for instances pub multicast_groups: Mutex>>, @@ -191,6 +196,7 @@ impl SledAgent { simulated_upstairs, v2p_mappings: Mutex::new(HashSet::new()), external_ips: Mutex::new(HashMap::new()), + attached_subnets: Mutex::new(HashMap::new()), multicast_groups: Mutex::new(HashMap::new()), vpc_routes: Mutex::new(HashMap::new()), mock_propolis: futures::lock::Mutex::new(None), @@ -729,6 +735,75 @@ impl SledAgent { Ok(()) } + pub async fn instance_put_attached_subnets( + &self, + propolis_id: PropolisUuid, + subnets: AttachedSubnets, + ) -> Result<(), Error> { + if !self.vmms.contains_key(&propolis_id.into_untyped_uuid()).await { + return Err(Error::internal_error( + "can't alter subnet state for VMM that's not registered", + )); + } + self.attached_subnets + .lock() + .unwrap() + .insert(propolis_id, subnets.subnets); + Ok(()) + } + + pub async fn instance_delete_attached_subnets( + &self, + propolis_id: PropolisUuid, + ) -> Result<(), Error> { + if !self.vmms.contains_key(&propolis_id.into_untyped_uuid()).await { + return Err(Error::internal_error( + "can't alter subnet state for VMM that's not registered", + )); + } + self.attached_subnets + .lock() + .unwrap() + .entry(propolis_id) + .or_default() + .clear(); + Ok(()) + } + + pub async fn instance_post_attached_subnet( + &self, + propolis_id: PropolisUuid, + subnet: AttachedSubnet, + ) -> Result<(), Error> { + if !self.vmms.contains_key(&propolis_id.into_untyped_uuid()).await { + return Err(Error::internal_error( + "can't alter subnet state for VMM that's not registered", + )); + } + let mut subnets = self.attached_subnets.lock().unwrap(); + let instance_subnets = subnets.entry(propolis_id).or_default(); + instance_subnets + .insert_unique(subnet) + .map_err(|_| Error::conflict("Subnet already attached")) + } + + pub async fn instance_delete_attached_subnet( + &self, + propolis_id: PropolisUuid, + subnet: IpNet, + ) -> Result<(), Error> { + if !self.vmms.contains_key(&propolis_id.into_untyped_uuid()).await { + return Err(Error::internal_error( + "can't alter subnet state for VMM that's not registered", + )); + } + let mut subnets = self.attached_subnets.lock().unwrap(); + if let Some(instance_subnets) = subnets.get_mut(&propolis_id) { + instance_subnets.remove(&subnet); + } + Ok(()) + } + pub async fn instance_join_multicast_group( &self, propolis_id: PropolisUuid, diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 984f6eaa90e..3228e9fa567 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -59,6 +59,7 @@ use omicron_ddm_admin_client::Client as DdmAdminClient; use omicron_uuid_kinds::{ GenericUuid, MupdateOverrideUuid, PropolisUuid, SledUuid, }; +use oxnet::IpNet; use sled_agent_config_reconciler::{ ConfigReconcilerHandle, ConfigReconcilerSpawnToken, InternalDisks, InternalDisksReceiver, LedgerNewConfigError, LedgerTaskError, @@ -66,6 +67,8 @@ use sled_agent_config_reconciler::{ }; use sled_agent_health_monitor::handle::HealthMonitorHandle; use sled_agent_measurements::MeasurementsHandle; +use sled_agent_types::attached_subnet::AttachedSubnet; +use sled_agent_types::attached_subnet::AttachedSubnets; use sled_agent_types::dataset::LocalStorageDatasetDeleteRequest; use sled_agent_types::dataset::LocalStorageDatasetEnsureRequest; use sled_agent_types::disk::DiskStateRequested; @@ -203,6 +206,7 @@ impl From for dropshot::HttpError { const NO_SUCH_INSTANCE: &str = "NO_SUCH_INSTANCE"; const INSTANCE_CHANNEL_FULL: &str = "INSTANCE_CHANNEL_FULL"; + const SUBNET_ALREADY_ATTACHED: &str = "SUBNET_ALREADY_ATTACHED"; match err { Error::Instance(crate::instance_manager::Error::Instance( instance_error, @@ -258,6 +262,13 @@ impl From for dropshot::HttpError { instance_error.to_string(), ) } + err @ crate::instance::Error::SubnetAlreadyAttached(_) => { + HttpError::for_client_error( + Some(SUBNET_ALREADY_ATTACHED.to_string()), + ClientErrorStatusCode::CONFLICT, + err.to_string(), + ) + } e => HttpError::for_internal_error(e.to_string()), } } @@ -1397,6 +1408,57 @@ impl SledAgent { Ok(()) } + + /// Update the set of subnets attached to an instance. + pub(crate) async fn instance_put_attached_subnets( + &self, + propolis_id: PropolisUuid, + subnets: AttachedSubnets, + ) -> Result<(), Error> { + self.inner + .instances + .set_attached_subnets(propolis_id, subnets) + .await + .map_err(|e| Error::Instance(e)) + } + + /// Delete the set of subnets attached to an instance. + pub(crate) async fn instance_delete_attached_subnets( + &self, + propolis_id: PropolisUuid, + ) -> Result<(), Error> { + self.inner + .instances + .clear_attached_subnets(propolis_id) + .await + .map_err(|e| Error::Instance(e)) + } + + /// Attach a subnet to an instance. + pub(crate) async fn instance_attach_subnet( + &self, + propolis_id: PropolisUuid, + subnet: AttachedSubnet, + ) -> Result<(), Error> { + self.inner + .instances + .attach_subnet(propolis_id, subnet) + .await + .map_err(|e| Error::Instance(e)) + } + + /// Detach a subnet from an instance + pub(crate) async fn instance_detach_subnet( + &self, + propolis_id: PropolisUuid, + subnet: IpNet, + ) -> Result<(), Error> { + self.inner + .instances + .detach_subnet(propolis_id, subnet) + .await + .map_err(|e| Error::Instance(e)) + } } #[derive(From, thiserror::Error, Debug, SlogInlineError)] diff --git a/sled-agent/types/src/attached_subnet.rs b/sled-agent/types/src/attached_subnet.rs new file mode 100644 index 00000000000..791af754c4a --- /dev/null +++ b/sled-agent/types/src/attached_subnet.rs @@ -0,0 +1,5 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +pub use sled_agent_types_versions::latest::attached_subnet::*; diff --git a/sled-agent/types/src/lib.rs b/sled-agent/types/src/lib.rs index 689b3ea9611..7e1cc994bda 100644 --- a/sled-agent/types/src/lib.rs +++ b/sled-agent/types/src/lib.rs @@ -5,6 +5,7 @@ //! Common types for sled-agent. pub mod artifact; +pub mod attached_subnet; pub mod boot_disk; pub mod bootstore; pub mod dataset; diff --git a/sled-agent/types/versions/src/add_attached_subnets/attached_subnet.rs b/sled-agent/types/versions/src/add_attached_subnets/attached_subnet.rs new file mode 100644 index 00000000000..ee57b6536af --- /dev/null +++ b/sled-agent/types/versions/src/add_attached_subnets/attached_subnet.rs @@ -0,0 +1,44 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use iddqd::IdOrdMap; +use omicron_uuid_kinds::PropolisUuid; +use oxnet::IpNet; +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; + +/// Subnets attached to a single instance. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] +pub struct AttachedSubnets { + pub subnets: IdOrdMap, +} + +/// A subnet attached to a single instance. +#[derive(Clone, Copy, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +pub struct AttachedSubnet { + /// The IP subnet. + pub subnet: IpNet, + /// Is this is a subnet in the external customer network. + /// + /// If false, this is a VPC Subnet attached to the instance. + pub is_external: bool, +} + +impl iddqd::IdOrdItem for AttachedSubnet { + type Key<'a> = &'a IpNet; + + fn key(&self) -> Self::Key<'_> { + &self.subnet + } + + iddqd::id_upcast!(); +} + +/// Path parameters for referring to a single subnet attached to an instance. +#[derive(Clone, Copy, Debug, Deserialize, JsonSchema, Serialize)] +pub struct VmmSubnetPathParam { + pub propolis_id: PropolisUuid, + pub subnet: IpNet, +} diff --git a/sled-agent/types/versions/src/add_attached_subnets/instance.rs b/sled-agent/types/versions/src/add_attached_subnets/instance.rs new file mode 100644 index 00000000000..9e5e952483f --- /dev/null +++ b/sled-agent/types/versions/src/add_attached_subnets/instance.rs @@ -0,0 +1,96 @@ +use std::net::SocketAddr; + +use omicron_common::api::external::Hostname; +use omicron_common::api::internal::nexus::VmmRuntimeState; +use omicron_common::api::internal::shared::DelegatedZvol; +use omicron_common::api::internal::shared::DhcpConfig; +use omicron_common::api::internal::shared::ExternalIpConfig; +use omicron_common::api::internal::shared::NetworkInterface; +use omicron_common::api::internal::shared::ResolvedVpcFirewallRule; +use omicron_uuid_kinds::InstanceUuid; +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; +use uuid::Uuid; + +use crate::v1::instance::InstanceMetadata; +use crate::v1::instance::VmmSpec; +use crate::v7::instance::InstanceMulticastMembership; +use crate::v18::attached_subnet::AttachedSubnet; + +use crate::v17; + +/// The body of a request to ensure that a instance and VMM are known to a sled +/// agent. +#[derive(Serialize, Deserialize, JsonSchema)] +pub struct InstanceEnsureBody { + /// The virtual hardware configuration this virtual machine should have when + /// it is started. + pub vmm_spec: VmmSpec, + + /// Information about the sled-local configuration that needs to be + /// established to make the VM's virtual hardware fully functional. + pub local_config: InstanceSledLocalConfig, + + /// The initial VMM runtime state for the VMM being registered. + pub vmm_runtime: VmmRuntimeState, + + /// The ID of the instance for which this VMM is being created. + pub instance_id: InstanceUuid, + + /// The ID of the migration in to this VMM, if this VMM is being + /// ensured is part of a migration in. If this is `None`, the VMM is not + /// being created due to a migration. + pub migration_id: Option, + + /// The address at which this VMM should serve a Propolis server API. + pub propolis_addr: SocketAddr, + + /// Metadata used to track instance statistics. + pub metadata: InstanceMetadata, +} + +/// Describes sled-local configuration that a sled-agent must establish to make +/// the instance's virtual hardware fully functional. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct InstanceSledLocalConfig { + pub hostname: Hostname, + pub nics: Vec, + pub external_ips: Option, + pub attached_subnets: Vec, + pub multicast_groups: Vec, + pub firewall_rules: Vec, + pub dhcp_config: DhcpConfig, + pub delegated_zvols: Vec, +} + +impl From for InstanceEnsureBody { + fn from(v17: v17::instance::InstanceEnsureBody) -> InstanceEnsureBody { + InstanceEnsureBody { + vmm_spec: v17.vmm_spec, + local_config: v17.local_config.into(), + vmm_runtime: v17.vmm_runtime, + instance_id: v17.instance_id, + migration_id: v17.migration_id, + propolis_addr: v17.propolis_addr, + metadata: v17.metadata, + } + } +} + +impl From for InstanceSledLocalConfig { + fn from( + v17: v17::instance::InstanceSledLocalConfig, + ) -> InstanceSledLocalConfig { + InstanceSledLocalConfig { + hostname: v17.hostname, + nics: v17.nics, + external_ips: v17.external_ips, + attached_subnets: vec![], + multicast_groups: v17.multicast_groups, + firewall_rules: v17.firewall_rules, + dhcp_config: v17.dhcp_config, + delegated_zvols: v17.delegated_zvols, + } + } +} diff --git a/sled-agent/types/versions/src/add_attached_subnets/mod.rs b/sled-agent/types/versions/src/add_attached_subnets/mod.rs new file mode 100644 index 00000000000..6e42275e4bc --- /dev/null +++ b/sled-agent/types/versions/src/add_attached_subnets/mod.rs @@ -0,0 +1,8 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Types for managing attached subnets. + +pub mod attached_subnet; +pub mod instance; diff --git a/sled-agent/types/versions/src/latest.rs b/sled-agent/types/versions/src/latest.rs index 974e6df327c..ae3b82ccca0 100644 --- a/sled-agent/types/versions/src/latest.rs +++ b/sled-agent/types/versions/src/latest.rs @@ -4,6 +4,12 @@ //! Re-exports of the latest versions of all types. +pub mod attached_subnet { + pub use crate::v18::attached_subnet::AttachedSubnet; + pub use crate::v18::attached_subnet::AttachedSubnets; + pub use crate::v18::attached_subnet::VmmSubnetPathParam; +} + pub mod artifact { pub use crate::v1::artifact::ArtifactConfig; pub use crate::v1::artifact::ArtifactCopyFromDepotBody; @@ -72,8 +78,8 @@ pub mod instance { pub use crate::v7::instance::InstanceMulticastBody; pub use crate::v7::instance::InstanceMulticastMembership; - pub use crate::v17::instance::InstanceEnsureBody; - pub use crate::v17::instance::InstanceSledLocalConfig; + pub use crate::v18::instance::InstanceEnsureBody; + pub use crate::v18::instance::InstanceSledLocalConfig; pub use omicron_common::api::internal::shared::ResolvedVpcFirewallRule; } diff --git a/sled-agent/types/versions/src/lib.rs b/sled-agent/types/versions/src/lib.rs index b1dcdecb57b..5b9e3e67f64 100644 --- a/sled-agent/types/versions/src/lib.rs +++ b/sled-agent/types/versions/src/lib.rs @@ -51,6 +51,8 @@ pub mod v15; pub mod v16; #[path = "two_types_of_delegated_zvol/mod.rs"] pub mod v17; +#[path = "add_attached_subnets/mod.rs"] +pub mod v18; #[path = "add_switch_zone_operator_policy/mod.rs"] pub mod v3; #[path = "add_nexus_lockstep_port_to_inventory/mod.rs"] From 2b032b658a0c0d673dadc9b7579c9d40d622b50e Mon Sep 17 00:00:00 2001 From: Benjamin Naecker Date: Mon, 26 Jan 2026 17:55:07 -0800 Subject: [PATCH 2/3] Add Nexus sagas for external subnet attach / detach - Adds a subnet attach and detach saga in Nexus, modeled after the existing floating IP attachment sagas. - Update the common code for sagas to include passing or removing the attached subnets to Dendrite and / or OPTE. This is to pick up those changes during the existing instance sagas, e.g. instance update. - Fixes #9685 --- common/src/api/internal/shared/mod.rs | 15 +- nexus/db-model/src/external_subnet.rs | 4 +- nexus/db-model/src/instance.rs | 14 +- nexus/db-model/src/ipnet.rs | 10 + .../src/db/datastore/external_ip.rs | 1 + .../src/db/datastore/external_subnet.rs | 680 +++++++++++++++++- nexus/db-queries/src/db/datastore/mod.rs | 2 + .../src/db/queries/external_subnet.rs | 3 + nexus/db-schema/src/schema.rs | 6 + nexus/src/app/instance_network.rs | 372 ++++++++-- nexus/src/app/sagas/instance_common.rs | 212 +++++- nexus/src/app/sagas/instance_delete.rs | 31 +- nexus/src/app/sagas/instance_ip_attach.rs | 7 +- nexus/src/app/sagas/instance_ip_detach.rs | 6 +- nexus/src/app/sagas/instance_start.rs | 8 +- nexus/src/app/sagas/instance_update/mod.rs | 3 +- nexus/src/app/sagas/mod.rs | 26 +- nexus/src/app/sagas/subnet_attach.rs | 528 ++++++++++++++ nexus/src/app/sagas/subnet_detach.rs | 532 ++++++++++++++ nexus/test-utils/src/resource_helpers.rs | 30 + nexus/tests/config.test.toml | 1 + 21 files changed, 2387 insertions(+), 104 deletions(-) create mode 100644 nexus/src/app/sagas/subnet_attach.rs create mode 100644 nexus/src/app/sagas/subnet_detach.rs diff --git a/common/src/api/internal/shared/mod.rs b/common/src/api/internal/shared/mod.rs index efa0892fb7a..d429e2312d2 100644 --- a/common/src/api/internal/shared/mod.rs +++ b/common/src/api/internal/shared/mod.rs @@ -11,10 +11,10 @@ use crate::{ zpool_name::ZpoolName, }; use daft::Diffable; -use omicron_uuid_kinds::ExternalZpoolUuid; use omicron_uuid_kinds::{ DatasetUuid, ExternalSubnetUuid, InstanceUuid, RackUuid, SledUuid, }; +use omicron_uuid_kinds::{ExternalZpoolUuid, PropolisUuid}; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; use schemars::JsonSchema; use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; @@ -1106,11 +1106,8 @@ pub enum AttachedSubnetId { } /// All details about an attached subnet and the Instance it's attached to. -pub struct AttachedExternalSubnet { - /// ID of the subnet itself. - pub subnet_id: AttachedSubnetId, - /// ID of the instance - pub instance_id: InstanceUuid, +#[derive(Debug)] +pub struct AttachedSubnet { /// ID of the rack hosting this instance. // // NOTE: We do not use this today. It's here for the future when we've @@ -1122,6 +1119,12 @@ pub struct AttachedExternalSubnet { pub sled_id: SledUuid, /// Underlay IP address of the sled hosting the instance. pub sled_ip: Ipv6Addr, + /// ID of the Propolis hypervisor managing this instance. + pub vmm_id: PropolisUuid, + /// ID of the instance + pub instance_id: InstanceUuid, + /// ID of the subnet itself. + pub subnet_id: AttachedSubnetId, /// The IP subnet that's attached. pub subnet: IpNet, /// The MAC address of the primary network interface. diff --git a/nexus/db-model/src/external_subnet.rs b/nexus/db-model/src/external_subnet.rs index 497fe47e688..074d509fa36 100644 --- a/nexus/db-model/src/external_subnet.rs +++ b/nexus/db-model/src/external_subnet.rs @@ -24,10 +24,10 @@ use nexus_db_schema::schema::subnet_pool_member; use nexus_db_schema::schema::subnet_pool_silo_link; use nexus_types::external_api::params; use nexus_types::external_api::views; -use nexus_types::identity::Resource; +use nexus_types::identity::Resource as _; use omicron_common::api::external; use omicron_common::api::external::Error; -use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::GenericUuid as _; use omicron_uuid_kinds::InstanceKind; use omicron_uuid_kinds::SubnetPoolKind; use omicron_uuid_kinds::SubnetPoolMemberKind; diff --git a/nexus/db-model/src/instance.rs b/nexus/db-model/src/instance.rs index f8c8f41b97c..b56cb41fa1d 100644 --- a/nexus/db-model/src/instance.rs +++ b/nexus/db-model/src/instance.rs @@ -7,6 +7,7 @@ use super::{ ByteCount, Disk, ExternalIp, Generation, InstanceAutoRestartPolicy, InstanceCpuCount, InstanceCpuPlatform, InstanceState, Vmm, VmmState, }; +use crate::ExternalSubnet; use crate::collection::DatastoreAttachTargetConfig; use crate::serde_time_delta::optional_time_delta; use chrono::{DateTime, TimeDelta, Utc}; @@ -15,7 +16,7 @@ use diesel::expression::{ValidGrouping, is_aggregate}; use diesel::pg; use diesel::prelude::*; use diesel::sql_types::{Bool, Nullable}; -use nexus_db_schema::schema::{disk, external_ip, instance}; +use nexus_db_schema::schema::{disk, external_ip, external_subnet, instance}; use nexus_types::external_api::params; use omicron_uuid_kinds::{GenericUuid, InstanceUuid}; use serde::Deserialize; @@ -225,6 +226,17 @@ impl DatastoreAttachTargetConfig for Instance { type ResourceTimeDeletedColumn = external_ip::dsl::time_deleted; } +impl DatastoreAttachTargetConfig for Instance { + type Id = Uuid; + + type CollectionIdColumn = instance::dsl::id; + type CollectionTimeDeletedColumn = instance::dsl::time_deleted; + + type ResourceIdColumn = external_subnet::dsl::id; + type ResourceCollectionIdColumn = external_subnet::dsl::instance_id; + type ResourceTimeDeletedColumn = external_subnet::dsl::time_deleted; +} + /// Runtime state of the Instance, including the actual running state and minimal /// metadata /// diff --git a/nexus/db-model/src/ipnet.rs b/nexus/db-model/src/ipnet.rs index b37d14251f5..c3ba4d349c9 100644 --- a/nexus/db-model/src/ipnet.rs +++ b/nexus/db-model/src/ipnet.rs @@ -2,6 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use crate::IpVersion; use diesel::backend::Backend; use diesel::deserialize; use diesel::deserialize::FromSql; @@ -30,6 +31,15 @@ pub enum IpNet { V6(crate::Ipv6Net), } +impl IpNet { + pub fn ip_version(&self) -> IpVersion { + match self { + IpNet::V4(_) => IpVersion::V4, + IpNet::V6(_) => IpVersion::V6, + } + } +} + impl ::std::fmt::Display for IpNet { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/nexus/db-queries/src/db/datastore/external_ip.rs b/nexus/db-queries/src/db/datastore/external_ip.rs index dafbb015eb5..d471ac878a0 100644 --- a/nexus/db-queries/src/db/datastore/external_ip.rs +++ b/nexus/db-queries/src/db/datastore/external_ip.rs @@ -570,6 +570,7 @@ impl DataStore { .into_boxed() .filter(nic_dsl::parent_id.eq(instance_id.into_untyped_uuid())) .filter(nic_dsl::time_deleted.is_null()) + .filter(nic_dsl::is_primary.eq(true)) .filter(nic_dsl::kind.eq(NetworkInterfaceKind::Instance)); let has_matching_ip_stack = match ip_version { IpVersion::V4 => base_nic_query.select(nic_dsl::ip.is_not_null()), diff --git a/nexus/db-queries/src/db/datastore/external_subnet.rs b/nexus/db-queries/src/db/datastore/external_subnet.rs index f3e37855596..1547778284d 100644 --- a/nexus/db-queries/src/db/datastore/external_subnet.rs +++ b/nexus/db-queries/src/db/datastore/external_subnet.rs @@ -5,7 +5,15 @@ //! [`DataStore`] methods on Subnet Pools and External Subnets. use crate::db::DataStore; +use crate::db::collection_attach::AttachError; +use crate::db::collection_attach::DatastoreAttachTarget as _; +use crate::db::collection_detach::DatastoreDetachTarget as _; +use crate::db::collection_detach::DetachError; +use crate::db::datastore::SQL_BATCH_SIZE; +use crate::db::pagination::Paginator; use crate::db::pagination::paginated; +use crate::db::queries::external_ip::SAFE_TO_ATTACH_INSTANCE_STATES; +use crate::db::queries::external_subnet::MAX_ATTACHED_SUBNETS; use crate::db::queries::external_subnet::decode_delete_external_subnet_error; use crate::db::queries::external_subnet::decode_insert_external_subnet_error; use crate::db::queries::external_subnet::decode_unlink_subnet_pool_from_silo_result; @@ -14,14 +22,19 @@ use crate::db::queries::external_subnet::insert_external_subnet_query; use crate::db::queries::external_subnet::insert_subnet_pool_member_query; use crate::db::queries::external_subnet::link_subnet_pool_to_silo_query; use crate::db::queries::external_subnet::unlink_subnet_pool_from_silo_query; +use crate::db::update_and_check::UpdateAndCheck as _; +use crate::db::update_and_check::UpdateStatus; use async_bb8_diesel::AsyncRunQueryDsl as _; use chrono::Utc; +use diesel::BoolExpressionMethods as _; use diesel::ExpressionMethods as _; use diesel::JoinOnDsl as _; +use diesel::NullableExpressionMethods as _; use diesel::QueryDsl as _; use diesel::SelectableHelper as _; use diesel::result::DatabaseErrorKind; use diesel::result::Error as DieselError; +use dropshot::PaginationOrder; use nexus_auth::authz; use nexus_auth::authz::SUBNET_POOL_LIST; use nexus_auth::context::OpContext; @@ -32,14 +45,19 @@ use nexus_db_lookup::lookup; use nexus_db_model::ExternalSubnet; use nexus_db_model::ExternalSubnetIdentity; use nexus_db_model::ExternalSubnetUpdate; +use nexus_db_model::Instance; use nexus_db_model::IpAttachState; use nexus_db_model::IpNet; use nexus_db_model::IpVersion; +use nexus_db_model::Ipv6Addr; +use nexus_db_model::MacAddr; use nexus_db_model::Name; +use nexus_db_model::NetworkInterfaceKind; use nexus_db_model::SubnetPool; use nexus_db_model::SubnetPoolMember; use nexus_db_model::SubnetPoolSiloLink; use nexus_db_model::SubnetPoolUpdate; +use nexus_db_model::Vni; use nexus_db_model::to_db_typed_uuid; use nexus_types::external_api::params; use nexus_types::external_api::params::ExternalSubnetCreate; @@ -53,12 +71,60 @@ use omicron_common::api::external::NameOrId; use omicron_common::api::external::ResourceType; use omicron_common::api::external::UpdateResult; use omicron_common::api::external::http_pagination::PaginatedBy; +use omicron_common::api::internal::shared::AttachedSubnet; +use omicron_common::api::internal::shared::AttachedSubnetId; use omicron_uuid_kinds::ExternalSubnetUuid; -use omicron_uuid_kinds::GenericUuid as _; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; +use omicron_uuid_kinds::PropolisUuid; +use omicron_uuid_kinds::RackUuid; +use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::SubnetPoolUuid; use ref_cast::RefCast as _; use uuid::Uuid; +// Helper containing database records across the several tables we need to join +// for all the attached subnet details. +#[derive(diesel::Queryable, diesel::Selectable)] +struct AttachedSubnetDetails { + #[diesel(select_expression = nexus_db_schema::schema::external_subnet::id)] + subnet_id: Uuid, + #[diesel(select_expression = nexus_db_schema::schema::instance::id)] + instance_id: Uuid, + #[diesel(select_expression = nexus_db_schema::schema::sled::rack_id)] + rack_id: Uuid, + #[diesel(select_expression = nexus_db_schema::schema::sled::id)] + sled_id: Uuid, + #[diesel(select_expression = nexus_db_schema::schema::sled::ip)] + sled_ip: Ipv6Addr, + #[diesel(select_expression = nexus_db_schema::schema::vmm::id)] + vmm_id: Uuid, + #[diesel(select_expression = nexus_db_schema::schema::external_subnet::subnet)] + ip_subnet: IpNet, + #[diesel(select_expression = nexus_db_schema::schema::network_interface::mac)] + mac: MacAddr, + #[diesel(select_expression = nexus_db_schema::schema::vpc::vni)] + vni: Vni, +} + +impl From for AttachedSubnet { + fn from(value: AttachedSubnetDetails) -> Self { + Self { + _rack_id: RackUuid::from_untyped_uuid(value.rack_id), + sled_id: SledUuid::from_untyped_uuid(value.sled_id), + sled_ip: value.sled_ip.into(), + vmm_id: PropolisUuid::from_untyped_uuid(value.vmm_id), + instance_id: InstanceUuid::from_untyped_uuid(value.instance_id), + subnet_id: AttachedSubnetId::External( + ExternalSubnetUuid::from_untyped_uuid(value.subnet_id), + ), + subnet: value.ip_subnet.into(), + mac: value.mac.0, + vni: value.vni.0, + } + } +} + impl DataStore { /// Lookup a Subnet Pool by name or ID. pub fn lookup_subnet_pool<'a>( @@ -583,6 +649,33 @@ impl DataStore { .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } + /// Detach all external subnets from an instance. + /// + /// IMPORTANT: This method should only be called from the instance_delete + /// saga. It ignores the current state of the external subnet, and simply + /// force-detaches it. + pub async fn instance_detach_external_subnets( + &self, + opctx: &OpContext, + instance_id: Uuid, + ) -> Result { + use nexus_db_schema::schema::external_subnet::dsl; + diesel::update( + dsl::external_subnet + .filter(dsl::instance_id.eq(instance_id)) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::attach_state.eq(IpAttachState::Attached)), + ) + .set(( + dsl::time_deleted.eq(Utc::now()), + dsl::instance_id.eq(Option::::None), + dsl::attach_state.eq(IpAttachState::Detached), + )) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + /// List external subnets. pub async fn list_external_subnets( &self, @@ -609,6 +702,591 @@ impl DataStore { .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } + + /// Fetch the attached subnets, internal or external, to an instance. + pub async fn instance_lookup_attached_external_subnets( + &self, + opctx: &OpContext, + instance_id: InstanceUuid, + ) -> ListResultVec { + use nexus_db_schema::schema::external_subnet; + use nexus_db_schema::schema::instance; + use nexus_db_schema::schema::network_interface; + use nexus_db_schema::schema::sled; + use nexus_db_schema::schema::vmm; + use nexus_db_schema::schema::vpc; + use nexus_db_schema::schema::vpc_subnet; + + external_subnet::dsl::external_subnet + .inner_join( + instance::dsl::instance.on(instance::dsl::id + .nullable() + .eq(external_subnet::dsl::instance_id)), + ) + .inner_join(vmm::dsl::vmm.on( + vmm::dsl::id.nullable().eq(instance::dsl::active_propolis_id), + )) + .inner_join(sled::dsl::sled.on(vmm::dsl::sled_id.eq(sled::dsl::id))) + .inner_join( + network_interface::dsl::network_interface.on( + network_interface::dsl::kind + .eq(NetworkInterfaceKind::Instance) + .and( + network_interface::dsl::parent_id + .eq(instance::dsl::id), + ) + .and(network_interface::dsl::is_primary.eq(true)), + ), + ) + .inner_join( + vpc_subnet::dsl::vpc_subnet + .on(vpc_subnet::dsl::id + .eq(network_interface::dsl::subnet_id)), + ) + .inner_join( + vpc::dsl::vpc.on(vpc::dsl::id.eq(vpc_subnet::dsl::vpc_id)), + ) + .filter( + external_subnet::dsl::instance_id + .eq(to_db_typed_uuid(instance_id)), + ) + .filter(external_subnet::dsl::time_deleted.is_null()) + .filter( + external_subnet::dsl::attach_state.eq(IpAttachState::Attached), + ) + .filter(instance::dsl::time_deleted.is_null()) + .filter(instance::dsl::id.eq(to_db_typed_uuid(instance_id))) + .filter(network_interface::dsl::time_deleted.is_null()) + .filter(vpc_subnet::dsl::time_deleted.is_null()) + .filter(vpc::dsl::time_deleted.is_null()) + .select(AttachedSubnetDetails::as_select()) + .get_results_async::( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + .map(|rows| rows.into_iter().map(Into::into).collect()) + } + + /// List all subnets attached to any Instance. + /// + /// In contrast to `instance_lookup_external_subnets`, this fetches all + /// attached subnets (external or VPC), and joins it with the data needed to + /// actually send those attachment details to other components, mostly + /// Dendrite and, through the sled-agent, OPTE. + /// + /// This makes as many requests as needed, in pages, to get all the results. + /// It can be useful in latency-insensitive contexts like background tasks. + pub async fn list_all_attached_subnets_batched( + &self, + opctx: &OpContext, + ) -> ListResultVec { + let mut paginator = + Paginator::new(SQL_BATCH_SIZE, PaginationOrder::Ascending); + let mut results = Vec::new(); + while let Some(page) = paginator.next() { + let mut attachments = self + .list_attached_external_subnets( + opctx, + &page.current_pagparams(), + ) + .await?; + paginator = page.found_batch(&attachments, &|a| { + let AttachedSubnetId::External(id) = a.subnet_id else { + unreachable!(); + }; + id + }); + results.append(&mut attachments); + } + + // TODO-completeness: We need to fetch the attached VPC Subnets here + // too. See https://github.com/oxidecomputer/omicron/issues/9580. + Ok(results) + } + + /// List a page of External Subnets that are attached to an instance. + pub async fn list_attached_external_subnets( + &self, + opctx: &OpContext, + pagparams: &DataPageParams<'_, ExternalSubnetUuid>, + ) -> ListResultVec { + // This query JOINs a mass of tables together, since the external subnet + // attachment information is quite scattered today. + // + // Specifically, we start with `external_subnet` and join + // + // - `instance`, to get the VMM record ... + // - `vmm`, to get the sled ID ... + // - `sled`, to get the sled underlay IP and rack ID ... + // - `network_interface` to get the VPC Subnet ID ... + // - `vpc_subnet` to get the VPC ID ... + // - and `vpc` to get the VNI. + use nexus_db_schema::schema::external_subnet; + use nexus_db_schema::schema::instance; + use nexus_db_schema::schema::network_interface; + use nexus_db_schema::schema::sled; + use nexus_db_schema::schema::vmm; + use nexus_db_schema::schema::vpc; + use nexus_db_schema::schema::vpc_subnet; + + paginated( + external_subnet::dsl::external_subnet, + external_subnet::dsl::id, + &pagparams.map_name(|id| id.as_untyped_uuid()), + ) + .inner_join(instance::dsl::instance.on( + instance::dsl::id.nullable().eq(external_subnet::dsl::instance_id), + )) + .inner_join( + vmm::dsl::vmm.on(vmm::dsl::id + .nullable() + .eq(instance::dsl::active_propolis_id)), + ) + .inner_join(sled::dsl::sled.on(vmm::dsl::sled_id.eq(sled::dsl::id))) + .inner_join( + network_interface::dsl::network_interface.on( + network_interface::dsl::kind + .eq(NetworkInterfaceKind::Instance) + .and( + network_interface::dsl::parent_id.eq(instance::dsl::id), + ) + .and(network_interface::dsl::is_primary.eq(true)), + ), + ) + .inner_join( + vpc_subnet::dsl::vpc_subnet + .on(vpc_subnet::dsl::id.eq(network_interface::dsl::subnet_id)), + ) + .inner_join(vpc::dsl::vpc.on(vpc::dsl::id.eq(vpc_subnet::dsl::vpc_id))) + .filter(external_subnet::dsl::time_deleted.is_null()) + .filter(external_subnet::dsl::attach_state.eq(IpAttachState::Attached)) + .filter(instance::dsl::time_deleted.is_null()) + .filter(network_interface::dsl::time_deleted.is_null()) + .filter(vpc_subnet::dsl::time_deleted.is_null()) + .filter(vpc::dsl::time_deleted.is_null()) + .select(AttachedSubnetDetails::as_select()) + .get_results_async::( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + .map(|rows| rows.into_iter().map(Into::into).collect()) + } + + /// Start attaching an external subnet to an instance. + /// + /// This marks the subnet as in the "attaching" state and sets the parent to + /// the instance. + /// + /// If the attachment could not start, such as if the subnet was already + /// attached to another instance, or the instance was in an invalid state, + /// this returns an `Error`. + pub async fn begin_attach_subnet( + &self, + opctx: &OpContext, + authz_instance: &authz::Instance, + authz_subnet: &authz::ExternalSubnet, + ip_version: IpVersion, + ) -> Result { + use nexus_db_schema::schema::external_subnet::dsl; + use nexus_db_schema::schema::instance::dsl as instance_dsl; + use nexus_db_schema::schema::network_interface::dsl as nic_dsl; + + opctx.authorize(authz::Action::Modify, authz_instance).await?; + opctx.authorize(authz::Action::Modify, authz_subnet).await?; + + // Only allow attachment if the instance has a primary NIC with the same + // IP stack as the subnet being attached. + let base_nic_query = nic_dsl::network_interface + .into_boxed() + .filter(nic_dsl::parent_id.eq(authz_instance.id())) + .filter(nic_dsl::time_deleted.is_null()) + .filter(nic_dsl::is_primary.eq(true)) + .filter(nic_dsl::kind.eq(NetworkInterfaceKind::Instance)); + let has_matching_ip_stack = match ip_version { + IpVersion::V4 => base_nic_query.select(nic_dsl::ip.is_not_null()), + IpVersion::V6 => base_nic_query.select(nic_dsl::ipv6.is_not_null()), + }; + + let query = Instance::attach_resource_with_update_condition( + authz_instance.id(), + authz_subnet.id().into_untyped_uuid(), + instance_dsl::instance + .into_boxed() + .filter( + instance_dsl::state.eq_any(SAFE_TO_ATTACH_INSTANCE_STATES), + ) + .filter(instance_dsl::migration_id.is_null()), + dsl::external_subnet + .into_boxed() + .filter(dsl::attach_state.eq(IpAttachState::Detached)) + .filter(dsl::instance_id.is_null()), + MAX_ATTACHED_SUBNETS, + diesel::update(dsl::external_subnet).set(( + dsl::time_modified.eq(Utc::now()), + dsl::instance_id.eq(Some(authz_instance.id())), + dsl::attach_state.eq(IpAttachState::Attaching), + )), + has_matching_ip_stack, + ); + query + .attach_and_get_result_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map(|(_instance, subnet)| ExternalSubnetBeginAttachResult { subnet, do_saga: true }) + .or_else(|e| match e { + AttachError::CollectionNotFound => { + Err(Error::not_found_by_id( + ResourceType::Instance, + &authz_instance.id(), + )) + } + AttachError::ResourceNotFound => { + Err(Error::not_found_by_id( + ResourceType::ExternalSubnet, + &authz_subnet.id().into_untyped_uuid(), + )) + } + AttachError::NoUpdate { + attached_count, + update_condition_satisfied, + resource, + collection + } => { + let resource_instance_id = resource.instance_id.map(GenericUuid::into_untyped_uuid); + let instance_id = Some(authz_instance.id()); + match resource.attach_state { + IpAttachState::Detached => { + // Subnet was detached which means some other error + // occurred. Fall through to detect it. + } + IpAttachState::Attached => { + if resource_instance_id == instance_id { + // We're already attached to this instance. + // Return the record, but we are already done + // with the attach saga here. + return Ok(ExternalSubnetBeginAttachResult { + subnet: resource, + do_saga: false, + }); + } + // We're attached to some _other_ instance. + return Err(Error::invalid_request( + "External subnets cannot be attached to \ + one instance while still attached to another" + )); + } + IpAttachState::Attaching if resource_instance_id == instance_id => { + // The subnet is still being attached, e.g., the + // saga node for doing the attachment is running + // again. We need to continue the saga. + return Ok(ExternalSubnetBeginAttachResult { + subnet: resource, + do_saga: true, + }); + } + IpAttachState::Detaching | IpAttachState::Attaching => { + // This one is transient, and could be retried + // safely in the future, in either direction. + return Err(Error::unavail( + "The external subnet is in the process of \ + attaching to or detaching from another \ + instance." + )); + } + } + + // Attempt during a migration. + if collection.runtime_state.migration_id.is_some() { + return Err(Error::unavail( + "Cannot attach a subnet while instance is migrating" + )); + } + + // Instance is in a transitory state, e.g., starting. + if !SAFE_TO_ATTACH_INSTANCE_STATES + .contains(&collection.runtime_state.nexus_state) + { + return Err(Error::invalid_request(&format!( + "Cannot attach subnet to instance in {} state", + collection.runtime_state.nexus_state, + ))); + } + + // Too many subnets attached. + if attached_count >= i64::from(MAX_ATTACHED_SUBNETS) { + return Err(Error::invalid_request(&format!( + "An instance may not have more than \ + {MAX_ATTACHED_SUBNETS} attached subnets" + ))); + } + + // Mismatched IP stack. + if !update_condition_satisfied { + return Err(Error::invalid_request(&format!( + "Cannot attach an IP{} external subnet to \ + an instance that does not have a \ + matching IP stack", + ip_version, + ))); + } + + Err(Error::internal_error("failed to attach subnet")) + } + AttachError::DatabaseError(e) => Err(public_error_from_diesel(e, ErrorHandler::Server)) + }) + } + + /// Start detaching a subnet from an instance. + pub async fn begin_detach_subnet( + &self, + opctx: &OpContext, + authz_instance: &authz::Instance, + authz_subnet: &authz::ExternalSubnet, + ) -> Result { + use nexus_db_schema::schema::external_subnet::dsl; + use nexus_db_schema::schema::instance::dsl as instance_dsl; + + opctx.authorize(authz::Action::Modify, authz_instance).await?; + opctx.authorize(authz::Action::Modify, authz_subnet).await?; + + let query = Instance::detach_resource( + authz_instance.id(), + authz_subnet.id().into_untyped_uuid(), + instance_dsl::instance + .into_boxed() + .filter( + instance_dsl::state.eq_any(SAFE_TO_ATTACH_INSTANCE_STATES), + ) + .filter(instance_dsl::migration_id.is_null()), + dsl::external_subnet + .into_boxed() + .filter(dsl::attach_state.eq(IpAttachState::Attached)) + .filter(dsl::instance_id.eq(authz_instance.id())), + diesel::update(dsl::external_subnet).set(( + dsl::time_modified.eq(Utc::now()), + dsl::attach_state.eq(IpAttachState::Detaching), + )), + ); + query + .detach_and_get_result_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map(|subnet| ExternalSubnetBeginAttachResult { subnet, do_saga: true }) + .or_else(|e| match e { + DetachError::CollectionNotFound => { + Err(Error::not_found_by_id( + ResourceType::Instance, + &authz_instance.id(), + )) + } + DetachError::ResourceNotFound => { + Err(Error::not_found_by_id( + ResourceType::ExternalSubnet, + &authz_subnet.id().into_untyped_uuid(), + )) + } + DetachError::NoUpdate { + resource, + collection, + } => { + let resource_instance_id = resource.instance_id.map(GenericUuid::into_untyped_uuid); + let instance_id = Some(authz_instance.id()); + match resource.attach_state { + IpAttachState::Detached => { + // Subnet was already detached. Return success, but + // we don't need to run the saga. + return Ok(ExternalSubnetBeginAttachResult { + subnet: resource, + do_saga: false, + }); + } + IpAttachState::Detaching if resource_instance_id == instance_id => { + // Re-running the detach saga node again. + return Ok(ExternalSubnetBeginAttachResult { + subnet: resource, + do_saga: true, + }); + } + IpAttachState::Attached if resource_instance_id != instance_id => { + // Subnet is attached to some other instance. + return Err(Error::invalid_request( + "External subnet is not attached to the target instance" + )); + } + IpAttachState::Attached => { + // We're attached to this instance, which means some + // other precondition was violated. Fall through to + // detect it. + } + IpAttachState::Detaching | IpAttachState::Attaching => { + // This one is transient, and could be retried + // safely in the future, in either direction. + return Err(Error::unavail( + "The external subnet is in the process of \ + attaching to or detaching from another \ + instance." + )); + } + } + + // Attempt during a migration. + if collection.runtime_state.migration_id.is_some() { + return Err(Error::unavail( + "Cannot detach a subnet while instance is migrating" + )); + } + + // Instance is in a transitory state, e.g., starting. + if !SAFE_TO_ATTACH_INSTANCE_STATES + .contains(&collection.runtime_state.nexus_state) + { + return Err(Error::invalid_request(&format!( + "Cannot detach subnet from instance in {} state", + collection.runtime_state.nexus_state, + ))); + } + + Err(Error::internal_error("failed to attach subnet")) + } + DetachError::DatabaseError(e) => Err(public_error_from_diesel(e, ErrorHandler::Server)) + }) + } + + /// Move an external subnet from a transitional to final state. + pub async fn external_subnet_complete_op( + &self, + opctx: &OpContext, + id: ExternalSubnetUuid, + from: IpAttachState, + to: IpAttachState, + ) -> Result { + use nexus_db_schema::schema::external_subnet::dsl; + if !matches!(from, IpAttachState::Attaching | IpAttachState::Detaching) + { + return Err(Error::internal_error(&format!( + "external subnet must be in a transitory state, not {from}" + ))); + } + if !matches!(to, IpAttachState::Attached | IpAttachState::Detached) { + return Err(Error::internal_error(&format!( + "external subnet must move to a final state, not {from}" + ))); + } + let initial_update = diesel::update(dsl::external_subnet) + .filter(dsl::id.eq(to_db_typed_uuid(id))) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::attach_state.eq(from)); + + let now = Utc::now(); + let conn = self.pool_connection_authorized(opctx).await?; + match (from, to) { + // We're either: + // + // - Finalizing a detach operation, or + // - undoing a later step in a failed attach saga. + // + // In either case, we can just set the values we need. + (_, IpAttachState::Detached) => { + let mut rows = initial_update + .set(( + dsl::instance_id.eq(Option::::None), + dsl::time_modified.eq(now), + dsl::attach_state.eq(to), + )) + .returning(ExternalSubnet::as_returning()) + .get_results_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + match rows.len() { + 0 => Ok(ExternalSubnetCompleteAttachResult::NoChanges), + 1 => Ok(ExternalSubnetCompleteAttachResult::Modified( + rows.pop().expect("just checked it has 1 element"), + )), + n => Err(Error::internal_error(&format!( + "In `external_subnet_complete_op` expected 0 or 1 \ + rows to be modified, found {n}", + ))), + } + } + // Finalizing the attachment. + // + // Similar to the Floating IP case, we need special handling here + // for two cases: + // + // - the subnet is deleted, which can happen during instance delete + // - the subnet is suddenly detached, if there are concurrent sagas. + (IpAttachState::Attaching, IpAttachState::Attached) => { + initial_update + .set((dsl::time_modified.eq(now), dsl::attach_state.eq(to))) + .check_if_exists::(id.into_untyped_uuid()) + .execute_and_check(&conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + .and_then(|r| match r.status { + UpdateStatus::Updated => { + Ok(ExternalSubnetCompleteAttachResult::Modified( + r.found, + )) + } + UpdateStatus::NotUpdatedButExists + if r.found.attach_state + == IpAttachState::Detached + || r.found.identity.time_deleted.is_some() => + { + Err(Error::internal_error( + "unwinding due to concurrent instance delete", + )) + } + UpdateStatus::NotUpdatedButExists => { + Ok(ExternalSubnetCompleteAttachResult::NoChanges) + } + }) + } + // Undoing a failed detach saga. + (IpAttachState::Detaching, IpAttachState::Attached) => { + let mut rows = initial_update + .set((dsl::time_modified.eq(now), dsl::attach_state.eq(to))) + .returning(ExternalSubnet::as_returning()) + .get_results_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + match rows.len() { + 0 => Ok(ExternalSubnetCompleteAttachResult::NoChanges), + 1 => Ok(ExternalSubnetCompleteAttachResult::Modified( + rows.pop().expect("just checked it has 1 element"), + )), + n => Err(Error::internal_error(&format!( + "In `external_subnet_complete_op` expected 0 or 1 \ + rows to be modified, found {n}", + ))), + } + } + (_, _) => { + return Err(Error::internal_error(&format!( + "Invalid set of states in external_subnet_complete_op: \ + from={from:#?}, to={to:#?}" + ))); + } + } + } +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +pub struct ExternalSubnetBeginAttachResult { + pub subnet: ExternalSubnet, + pub do_saga: bool, +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +pub enum ExternalSubnetCompleteAttachResult { + Modified(ExternalSubnet), + NoChanges, } #[cfg(test)] diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index 6a4d1a44b10..f3935639384 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -142,6 +142,8 @@ pub use dns::DataStoreDnsTest; pub use dns::DnsVersionUpdateBuilder; pub use ereport::EreportFilters; pub use external_ip::FloatingIpAllocation; +pub use external_subnet::ExternalSubnetBeginAttachResult; +pub use external_subnet::ExternalSubnetCompleteAttachResult; pub use instance::{ InstanceAndActiveVmm, InstanceGestalt, InstanceStateComputer, }; diff --git a/nexus/db-queries/src/db/queries/external_subnet.rs b/nexus/db-queries/src/db/queries/external_subnet.rs index f339fe19676..af7b45f1b90 100644 --- a/nexus/db-queries/src/db/queries/external_subnet.rs +++ b/nexus/db-queries/src/db/queries/external_subnet.rs @@ -33,6 +33,9 @@ use oxnet::Ipv4Net; use oxnet::Ipv6Net; use uuid::Uuid; +/// The maximum number of subnets that can be attached to an instance. +pub const MAX_ATTACHED_SUBNETS: u32 = 32; + /// Query to insert a member in a Subnet Pool, which checks for overlapping /// subnets. pub fn insert_subnet_pool_member_query( diff --git a/nexus/db-schema/src/schema.rs b/nexus/db-schema/src/schema.rs index 903c6961aa6..4af1022f119 100644 --- a/nexus/db-schema/src/schema.rs +++ b/nexus/db-schema/src/schema.rs @@ -826,6 +826,12 @@ table! { } allow_tables_to_appear_in_same_query!(external_subnet, project); +allow_tables_to_appear_in_same_query!(external_subnet, instance); +allow_tables_to_appear_in_same_query!(external_subnet, vmm); +allow_tables_to_appear_in_same_query!(external_subnet, sled); +allow_tables_to_appear_in_same_query!(external_subnet, network_interface); +allow_tables_to_appear_in_same_query!(external_subnet, vpc); +allow_tables_to_appear_in_same_query!(external_subnet, vpc_subnet); table! { silo (id) { diff --git a/nexus/src/app/instance_network.rs b/nexus/src/app/instance_network.rs index 1a34605ab6e..8da80e755c6 100644 --- a/nexus/src/app/instance_network.rs +++ b/nexus/src/app/instance_network.rs @@ -20,13 +20,57 @@ use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::api::internal::shared::SwitchLocation; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; +use oxnet::IpNet; use oxnet::Ipv6Net; +use slog_error_chain::InlineErrorChain; use std::collections::HashSet; use std::str::FromStr; use uuid::Uuid; use super::Nexus; +/// Filter external IP addresses when modifying instance networking state. +#[derive(Clone, Copy, Debug)] +pub enum InstanceIpFilter { + /// Notify Dendrite about all external IPs for an instance. + All, + /// Notify Dendrite about exactly one IP by ID for an instance. + Exactly(Uuid), +} + +/// Filter attached subnets when modifying instance networking state. +#[derive(Clone, Copy, Debug)] +pub enum AttachedSubnetFilter { + /// Do not notify Dendrite about any attached subnets. + None, + /// Notify Dendrite about all attached subnets for an instance. + All, +} + +/// Filters applied during `instance_ensure_dpd_config()` below. +#[derive(Clone, Copy, Debug)] +pub struct InstanceNetworkFilters { + /// Filters applied to external IPs. + pub ips: InstanceIpFilter, + /// Filters applied to attached subnets. + pub subnets: AttachedSubnetFilter, +} + +impl InstanceNetworkFilters { + /// Construct a filter to push a single external IP for an instance. + pub fn single_ip(id: Uuid) -> Self { + Self { + ips: InstanceIpFilter::Exactly(id), + subnets: AttachedSubnetFilter::None, + } + } + + /// Construct a filter to push all networking state. + pub fn all() -> Self { + Self { ips: InstanceIpFilter::All, subnets: AttachedSubnetFilter::All } + } +} + impl Nexus { /// Returns the set of switches with uplinks configured and boundary /// services enabled. @@ -51,20 +95,19 @@ impl Nexus { /// - `instance_id`: The ID of the instance to act on. /// - `sled_ip_address`: The internal IP address assigned to the sled's /// sled agent. - /// - `ip_filter`: An optional filter on the index into the instance's - /// external IP array. - /// - If this is `Some(id)`, this routine configures DPD state for only the - /// external IP with `id` in the collection returned from CRDB. This will - /// proceed even when the target IP is 'attaching'. - /// - If this is `None`, this routine configures DPD for all external - /// IPs and *will back out* if any IPs are not yet fully attached to - /// the instance. + /// - `filters`: Filters describing which networking state to push. See the + /// `InstanceNetworkFilters` type for the contents. This can be used to + /// push either all or a subset of the networking state for the instance. + /// For example, it is used to push exactly one new IP address during the + /// instance IP attachment saga; or all IP addresses during an instance + /// start saga. In the latter case, the method will fail if any of the IP + /// addresses are not fully attached to this instance. pub(crate) async fn instance_ensure_dpd_config( &self, opctx: &OpContext, instance_id: InstanceUuid, sled_ip_address: &std::net::SocketAddrV6, - ip_filter: Option, + filters: InstanceNetworkFilters, ) -> Result, Error> { instance_ensure_dpd_config( &self.db_datastore, @@ -74,7 +117,7 @@ impl Nexus { &self.opctx_alloc, instance_id, sled_ip_address, - ip_filter, + filters, ) .await } @@ -181,6 +224,38 @@ impl Nexus { .await } + /// Send a single attached subnet to Dendrite. + pub(crate) async fn send_attached_subnet_to_dendrite( + &self, + subnets: &[IpNet], + target: dpd_client::types::InstanceTarget, + ) -> Result<(), Error> { + send_subnet_attachments_to_dendrite_inner( + &self.db_datastore, + &self.log, + self.resolver(), + &self.opctx_alloc, + subnets, + target, + ) + .await + } + + /// Attempt to delete a single attached subnet from Dendrite. + pub(crate) async fn delete_attached_subnet_from_dendrite( + &self, + subnet: IpNet, + ) -> Result<(), Error> { + delete_attached_subnets_from_dendrite_inner( + &self.db_datastore, + &self.log, + self.resolver(), + &self.opctx_alloc, + &[subnet], + ) + .await + } + // The logic of this function should follow very closely what // `instance_delete_dpd_config` does. However, there are enough differences // in the mechanics of how the logic is being carried out to justify having @@ -247,14 +322,8 @@ pub(crate) async fn boundary_switches( /// - `instance_id`: The ID of the instance to act on. /// - `sled_ip_address`: The internal IP address assigned to the sled's /// sled agent. -/// - `ip_filter`: An optional filter on the index into the instance's -/// external IP array. -/// - If this is `Some(id)`, this routine configures DPD state for only the -/// external IP with `id` in the collection returned from CRDB. This will -/// proceed even when the target IP is 'attaching'. -/// - If this is `None`, this routine configures DPD for all external -/// IPs and *will back out* if any IPs are not yet fully attached to -/// the instance. +/// - `filters`: Describes which networking state to push. See +/// `InstanceNetworkFilters`. #[allow(clippy::too_many_arguments)] // I don't like it either, clippy... pub(crate) async fn instance_ensure_dpd_config( datastore: &DataStore, @@ -264,7 +333,7 @@ pub(crate) async fn instance_ensure_dpd_config( opctx_alloc: &OpContext, instance_id: InstanceUuid, sled_ip_address: &std::net::SocketAddrV6, - ip_filter: Option, + filters: InstanceNetworkFilters, ) -> Result, Error> { info!( log, @@ -277,10 +346,10 @@ pub(crate) async fn instance_ensure_dpd_config( .lookup_for(authz::Action::ListChildren) .await?; - // All external IPs map to the primary network interface, so find that - // interface. If there is no such interface, there's no way to route - // traffic destined to those IPs, so there's nothing to configure and - // it's safe to return early. + // All networking resources map to the primary network interface, so find + // that interface. If there is no such interface, there's no way to route + // traffic destined to those the instance, so there's nothing to configure + // and it's safe to return early. let mut nat_entries = vec![]; let network_interface = match datastore .derive_guest_network_interface_info(&opctx, &authz_instance) @@ -308,18 +377,17 @@ pub(crate) async fn instance_ensure_dpd_config( let ips = datastore.instance_lookup_external_ips(&opctx, instance_id).await?; - let (ips_of_interest, must_all_be_attached) = if let Some(wanted_id) = - ip_filter - { - if let Some(ip) = ips.iter().find(|v| v.id == wanted_id) { - (std::slice::from_ref(ip), false) - } else { - return Err(Error::internal_error(&format!( - "failed to find external ip address with id: {wanted_id}, saw {ips:?}", - ))); + let (ips_of_interest, must_all_be_attached) = match filters.ips { + InstanceIpFilter::All => (&ips[..], true), + InstanceIpFilter::Exactly(wanted_id) => { + if let Some(ip) = ips.iter().find(|v| v.id == wanted_id) { + (std::slice::from_ref(ip), false) + } else { + return Err(Error::internal_error(&format!( + "failed to find external ip address with id: {wanted_id}, saw {ips:?}", + ))); + } } - } else { - (&ips[..], true) }; // This is performed so that an IP attach/detach will block the @@ -427,7 +495,30 @@ pub(crate) async fn instance_ensure_dpd_config( "error encountered when notifying dendrite"; "error" => %e ) - }; + } + + match filters.subnets { + AttachedSubnetFilter::None => {} + AttachedSubnetFilter::All => { + let instance_target = dpd_client::types::InstanceTarget { + inner_mac: dpd_client::types::MacAddr { + a: network_interface.mac.into_array(), + }, + internal_ip: *sled_ip_address.ip(), + vni: dpd_client::types::Vni(network_interface.vni.into()), + }; + let _ = instance_send_attached_subnets_to_dendrite( + datastore, + log, + resolver, + opctx, + opctx_alloc, + instance_id, + instance_target, + ) + .await; + } + } Ok(nat_entries) } @@ -517,8 +608,8 @@ pub(crate) async fn probe_ensure_dpd_config( Ok(()) } -/// Attempts to delete all of the Dendrite NAT configuration for the -/// instance identified by `authz_instance`. +/// Attempts to delete all of the Dendrite configuration for the instance +/// identified by `authz_instance`. /// /// Unlike `instance_ensure_dpd_config`, this function will disregard the /// attachment states of any external IPs because likely callers (instance @@ -533,12 +624,6 @@ pub(crate) async fn probe_ensure_dpd_config( /// - If an operation fails while this routine is walking NAT entries, it /// will continue trying to delete subsequent entries but will return the /// first error it encountered. -/// - `ip_filter`: An optional filter on the index into the instance's -/// external IP array. -/// - If this is `Some(id)`, this routine configures DPD state for only the -/// external IP with `id` in the collection returned from CRDB. -/// - If this is `None`, this routine configures DPD for all external -/// IPs. pub(crate) async fn instance_delete_dpd_config( datastore: &DataStore, log: &slog::Logger, @@ -559,7 +644,24 @@ pub(crate) async fn instance_delete_dpd_config( external_ip_delete_dpd_config_inner(&datastore, &log, opctx, &entry) .await?; } - + // TODO-performance: This duplicates some work with the below call notifying + // Dendrite about NAT state, such as looking up boundary switches and doing + // DNS resolution for those clients. + // + // That call below should go away or at least change with the resolution of + // https://github.com/oxidecomputer/omicron/issues/8748 which removes the + // "upcall" from Dendrite to Nexus about the NAT state, in favor of Nexus + // only pushing it to Dendrite. This also isn't that performance-sensitive + // of a code path, but it would still be nice to share some work. + let _ = instance_delete_attached_subnets_from_dendrite( + datastore, + log, + resolver, + opctx, + opctx_alloc, + instance_id, + ) + .await; notify_dendrite_nat_state( datastore, log, @@ -570,6 +672,188 @@ pub(crate) async fn instance_delete_dpd_config( .await } +async fn instance_send_attached_subnets_to_dendrite( + datastore: &DataStore, + log: &slog::Logger, + resolver: &internal_dns_resolver::Resolver, + opctx: &OpContext, + opctx_alloc: &OpContext, + instance_id: InstanceUuid, + instance_target: dpd_client::types::InstanceTarget, +) -> Result<(), Error> { + let subnets = match datastore + .instance_lookup_attached_external_subnets(opctx, instance_id) + .await + { + Ok(s) => s, + Err(e) => { + error!( + log, + "failed to lookup instance's attached subnet"; + "instance_id" => %instance_id, + "error" => InlineErrorChain::new(&e), + ); + return Err(e); + } + }; + if subnets.is_empty() { + debug!( + log, + "no subnets attached to instance, \ + nothing to notify Dendrite about"; + "instance_id" => %instance_id, + ); + return Ok(()); + } + let subnets = subnets.into_iter().map(|s| s.subnet).collect::>(); + send_subnet_attachments_to_dendrite_inner( + datastore, + log, + resolver, + opctx_alloc, + &subnets, + instance_target, + ) + .await +} + +async fn send_subnet_attachments_to_dendrite_inner( + datastore: &DataStore, + log: &slog::Logger, + resolver: &internal_dns_resolver::Resolver, + opctx_alloc: &OpContext, + subnets: &[IpNet], + instance_target: dpd_client::types::InstanceTarget, +) -> Result<(), Error> { + let boundary_switches = + match boundary_switches(datastore, opctx_alloc).await { + Ok(s) => s, + Err(e) => { + error!( + log, + "failed to lookup boundary switches"; + "error" => InlineErrorChain::new(&e), + ); + return Err(e); + } + }; + let clients = match super::dpd_clients(resolver, log).await { + Ok(c) => c, + Err(e) => { + let msg = format!("failed to get dpd clients: {e}"); + error!( + log, + "failed to look up Dendrite clients"; + "error" => %msg, + ); + return Err(Error::internal_error(&e)); + } + }; + // Find the boundary switches, and create clients for them through + // the resolver. + // + // TODO-correctness: This should take into account the rack in the + // list of attached subnets, once we address + // https://github.com/oxidecomputer/omicron/issues/5201. + + // Add all the subnets for this instance to each Dendrite. + for switch in &boundary_switches { + let Some(client) = clients.get(switch) else { + error!( + log, + "cannot find Dendrite client for boundary switch, \ + will not send instance's attached subnets"; + "switch" => %switch, + ); + continue; + }; + for subnet in subnets.iter() { + match client.attached_subnet_create(&subnet, &instance_target).await + { + Ok(_) => debug!( + log, + "deleted instance attached subnet from Dendrite"; + "subnet" => %subnet, + ), + Err(e) => error!( + log, + "failed to delete instance attached subnet from Dendrite"; + "subnet" => %subnet, + "error" => InlineErrorChain::new(&e), + ), + } + } + } + Ok(()) +} + +async fn instance_delete_attached_subnets_from_dendrite( + datastore: &DataStore, + log: &slog::Logger, + resolver: &internal_dns_resolver::Resolver, + opctx: &OpContext, + opctx_alloc: &OpContext, + instance_id: InstanceUuid, +) -> Result<(), Error> { + let subnets = datastore + .instance_lookup_attached_external_subnets(opctx, instance_id) + .await? + .into_iter() + .map(|s| s.subnet) + .collect::>(); + if subnets.is_empty() { + return Ok(()); + } + delete_attached_subnets_from_dendrite_inner( + datastore, + log, + resolver, + opctx_alloc, + &subnets, + ) + .await +} + +async fn delete_attached_subnets_from_dendrite_inner( + datastore: &DataStore, + log: &slog::Logger, + resolver: &internal_dns_resolver::Resolver, + opctx_alloc: &OpContext, + subnets: &[IpNet], +) -> Result<(), Error> { + let boundary_switches = boundary_switches(datastore, opctx_alloc).await?; + let clients = super::dpd_clients(resolver, log).await.map_err(|e| { + Error::internal_error(&format!("failed to get dpd clients: {e}")) + })?; + for switch in boundary_switches.iter() { + let Some(client) = clients.get(switch) else { + error!( + log, + "cannot find Dendrite client for boundary switch, \ + will not delete attached subnet"; + "switch" => %switch, + ); + continue; + }; + for subnet in subnets.iter() { + match client.attached_subnet_delete(subnet).await { + Ok(_) => debug!( + log, + "deleted attached subnet from Dendrite"; + "subnet" => %subnet, + ), + Err(e) => error!( + log, + "failed to delete attached subnet from Dendrite"; + "subnet" => %subnet, + "error" => InlineErrorChain::new(&e), + ), + } + } + } + Ok(()) +} + // The logic of this function should follow very closely what // `instance_delete_dpd_config` does. However, there are enough differences // in the mechanics of how the logic is being carried out to justify having diff --git a/nexus/src/app/sagas/instance_common.rs b/nexus/src/app/sagas/instance_common.rs index eb4315dc1d8..0b3f0520326 100644 --- a/nexus/src/app/sagas/instance_common.rs +++ b/nexus/src/app/sagas/instance_common.rs @@ -6,21 +6,23 @@ use std::net::{IpAddr, Ipv6Addr}; +use super::NexusActionContext; use crate::Nexus; +use crate::app::instance_network::InstanceNetworkFilters; +use http::StatusCode; use nexus_db_lookup::LookupPath; use nexus_db_model::{ - ByteCount, ExternalIp, InstanceState, IpAttachState, NatEntry, + ByteCount, ExternalIp, InstanceState, IpAttachState, IpNet, NatEntry, SledReservationConstraints, SledResourceVmm, VmmCpuPlatform, VmmState, }; use nexus_db_queries::authz; +use nexus_db_queries::db::datastore::ExternalSubnetBeginAttachResult; use nexus_db_queries::{authn, context::OpContext, db, db::DataStore}; use omicron_common::api::external::{Error, IpVersion, NameOrId}; use omicron_uuid_kinds::{GenericUuid, InstanceUuid, PropolisUuid, SledUuid}; use serde::{Deserialize, Serialize}; use steno::ActionError; -use super::NexusActionContext; - /// The port propolis-server listens on inside the propolis zone. const DEFAULT_PROPOLIS_PORT: u16 = 12400; @@ -179,17 +181,17 @@ pub async fn instance_ip_move_state( } } -/// Yields the sled on which an instance is found to be running so that IP -/// attachment and detachment operations can be propagated there. +/// Yields the sled on which an instance is found to be running so that IP and +/// subnet attachment and detachment operations can be propagated there. /// /// # Preconditions /// /// To synchronize correctly with other concurrent operations on an instance, -/// the calling saga must have placed the IP it is attaching or detaching into -/// the Attaching or Detaching state so that concurrent attempts to start the -/// instance will notice that the IP state is in flux and ask the caller to -/// retry. -pub(super) async fn instance_ip_get_instance_state( +/// the calling saga must have placed the IP or subnet it is attaching or +/// detaching into the Attaching or Detaching state so that concurrent attempts +/// to start the instance will notice that the IP state is in flux and ask the +/// caller to retry. +pub(super) async fn networking_resource_instance_state( sagactx: &NexusActionContext, serialized_authn: &authn::saga::Serialized, authz_instance: &authz::Instance, @@ -219,7 +221,8 @@ pub(super) async fn instance_ip_get_instance_state( }); slog::debug!( - osagactx.log(), "evaluating instance state for IP attach/detach"; + osagactx.log(), + "evaluating instance state for networking attach/detach"; "instance_state" => ?found_instance_state, "vmm_state" => ?found_vmm_state ); @@ -231,8 +234,8 @@ pub(super) async fn instance_ip_get_instance_state( // otherwise convert OPTE ensure to 'service unavailable' // and undo. // - deleting: can only be called from stopped -- we won't push to dpd - // or sled-agent, and IP record might be deleted or forcibly - // detached. Catch here just in case. + // or sled-agent, and IP / subnet record might be deleted + // or forcibly detached. Catch here just in case. // - starting: see below. match (found_instance_state, found_vmm_state) { // If there's no VMM, the instance is definitely not on any sled. @@ -254,20 +257,20 @@ pub(super) async fn instance_ip_get_instance_state( // Although an instance with a Starting (or Creating) VMM has a sled // assignment, there's no way to tell at this point whether or not // there's a concurrent instance-start saga that has passed the point - // where it sends IP assignments to the instance's new sled: + // where it sends IP / subnet assignments to the instance's new sled: // - // - If the start saga is still in progress and hasn't pushed any IP - // information to the instance's new sled yet, then either of two - // things can happen: - // - This function's caller can finish modifying IPs before the start - // saga propagates IP information to the sled. In this case the - // calling saga should do nothing--the start saga will send the - // right IP set to the sled. + // - If the start saga is still in progress and hasn't pushed any + // network information to the instance's new sled yet, then either of + // two things can happen: + // - This function's caller can finish modifying IPs or subnets before + // the start saga propagates information to the sled. In this case, + // the calling saga should do nothing--the start saga will send the + // right IP or subnet set to the sled. // - If the start saga "wins" the race, it will see that the instance // still has an attaching/detaching IP and bail out. // - If the start saga is already done, and Nexus is just waiting for // the VMM to report that it's Running, the calling saga needs to - // send the IP change to the instance's sled. + // send the IP or subnet change to the instance's sled. // // There's no way to distinguish these cases, so if a VMM is Starting, // block the attach/detach. @@ -291,13 +294,15 @@ pub(super) async fn instance_ip_get_instance_state( } (InstanceState::Creating, _) => { return Err(ActionError::action_failed(Error::invalid_request( - "cannot modify instance IPs, instance is still being created", + "cannot modify instance IP or subnets, instance is \ + still being created", ))); } (InstanceState::Failed, _) | (InstanceState::Vmm, Some(VmmState::Failed)) => { return Err(ActionError::action_failed(Error::invalid_request( - "cannot modify instance IPs, instance is in unhealthy state", + "cannot modify instance IPs or subnets, instance is \ + in unhealthy state", ))); } @@ -324,6 +329,163 @@ pub(super) async fn instance_ip_get_instance_state( Ok(propolis_and_sled_id) } +/// Send details about a new attached subnet to Dendrite. +pub(super) async fn send_subnet_attachment_to_dpd( + sagactx: &NexusActionContext, + serialized_authn: &authn::saga::Serialized, + authz_instance: &authz::Instance, + sled_uuid: Option, + subnet: ExternalSubnetBeginAttachResult, +) -> Result, ActionError> { + let osagactx = sagactx.user_data(); + let datastore = osagactx.datastore(); + let opctx = + crate::context::op_context_for_saga_action(&sagactx, serialized_authn); + + // Check that we have a sled and actually want to proceed with the saga. + let Some(sled_uuid) = sled_uuid else { + return Ok(None); + }; + let ExternalSubnetBeginAttachResult { subnet, do_saga } = subnet; + if !do_saga { + return Ok(None); + } + + // Querying sleds requires fleet access; use the instance allocator context + // for this. + let (.., sled) = LookupPath::new(&osagactx.nexus().opctx_alloc, datastore) + .sled_id(sled_uuid) + .fetch() + .await + .map_err(ActionError::action_failed)?; + + // Build the instance target for Dendrite. + // + // We first need the primary NIC of the instance, since we're mapping to the + // inner MAC and VNI. + let primary_nic = datastore + .derive_guest_network_interface_info(&opctx, &authz_instance) + .await + .map_err(ActionError::action_failed)? + .into_iter() + .find(|nic| nic.primary) + .ok_or_else(|| { + ActionError::action_failed(Error::internal_error( + "instance does not have a primary NIC, \ + cannot yet send Dendrite attached subnet info", + )) + })?; + let target = dpd_client::types::InstanceTarget { + inner_mac: dpd_client::types::MacAddr { + a: primary_nic.mac.into_array(), + }, + internal_ip: sled.ip.into(), + vni: dpd_client::types::Vni(primary_nic.vni.into()), + }; + osagactx + .nexus() + .send_attached_subnet_to_dendrite(&[subnet.subnet.into()], target) + .await + .map(|_| Some(subnet.subnet)) + .map_err(ActionError::action_failed) +} + +/// Delete an attached subnet from Dendrite. +pub(super) async fn delete_subnet_attachment_from_dpd( + sagactx: &NexusActionContext, + subnet: IpNet, +) -> Result<(), ActionError> { + sagactx + .user_data() + .nexus() + .delete_attached_subnet_from_dendrite(subnet.into()) + .await + .map_err(ActionError::action_failed) +} + +/// Send details about a new attached subnet to OPTE. +pub(super) async fn send_subnet_attachment_to_opte( + sagactx: &NexusActionContext, + vmm_and_sled: Option, + subnet: ExternalSubnetBeginAttachResult, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let Some(VmmAndSledIds { vmm_id: propolis_id, sled_id }) = vmm_and_sled + else { + return Ok(()); + }; + let ExternalSubnetBeginAttachResult { subnet, do_saga } = subnet; + if !do_saga { + return Ok(()); + } + let request = sled_agent_client::types::AttachedSubnet { + // TODO-completeness: Expand this code to handle VPC subnets too. See + // https://github.com/oxidecomputer/omicron/issues/9580. + is_external: true, + subnet: subnet.subnet.into(), + }; + let result = osagactx + .nexus() + .sled_client(&sled_id) + .await + .map_err(|_| { + ActionError::action_failed(Error::unavail( + "sled agent client went away mid-attach/detach", + )) + })? + .vmm_post_attached_subnet(&propolis_id, &request) + .await; + let result = match result { + Ok(_) => Ok(()), + Err(progenitor_client::Error::ErrorResponse(err)) + if err.status() == StatusCode::CONFLICT => + { + Ok(()) + } + Err(progenitor_client::Error::CommunicationError(_)) => { + Err(Error::unavail("sled agent client went away mid-attach/detach")) + } + Err(e) => Err(Error::internal_error(e.to_string().as_str())), + }; + result.map_err(ActionError::action_failed) +} + +/// Delete a single attached subnet from OPTE. +pub(super) async fn delete_subnet_attachment_from_opte( + sagactx: &NexusActionContext, + vmm_and_sled: Option, + subnet: ExternalSubnetBeginAttachResult, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let Some(VmmAndSledIds { vmm_id: propolis_id, sled_id }) = vmm_and_sled + else { + return Ok(()); + }; + let ExternalSubnetBeginAttachResult { subnet, do_saga } = subnet; + if !do_saga { + return Ok(()); + } + osagactx + .nexus() + .sled_client(&sled_id) + .await + .map_err(|_| { + ActionError::action_failed(Error::unavail( + "sled agent client went away mid-attach/detach", + )) + })? + .vmm_delete_attached_subnet(&propolis_id, &subnet.subnet.into()) + .await + .map(|_| ()) + .map_err(|e| match e { + progenitor_client::Error::CommunicationError(_) => { + Error::unavail("sled agent client went away mid-attach/detach") + } + e => Error::internal_error(e.to_string().as_str()), + }) + .map_err(ActionError::action_failed) +} + /// Adds a NAT entry to DPD, routing packets bound for `target_ip` to a /// target sled. /// @@ -369,7 +531,7 @@ pub async fn instance_ip_add_nat( &opctx, InstanceUuid::from_untyped_uuid(authz_instance.id()), &sled.address(), - Some(target_ip.id), + InstanceNetworkFilters::single_ip(target_ip.id), ) .await .and_then(|v| { diff --git a/nexus/src/app/sagas/instance_delete.rs b/nexus/src/app/sagas/instance_delete.rs index e2ae087fe9f..f9ad9f61b27 100644 --- a/nexus/src/app/sagas/instance_delete.rs +++ b/nexus/src/app/sagas/instance_delete.rs @@ -41,11 +41,14 @@ declare_saga_actions! { DEALLOCATE_EXTERNAL_IP -> "no_result3" { + sid_deallocate_external_ip } - LEAVE_MULTICAST_GROUPS -> "no_result4" { + DETACH_EXTERNAL_SUBNETS -> "no_result4" { + + sid_detach_external_subnets + } + LEAVE_MULTICAST_GROUPS -> "no_result5" { + sid_leave_multicast_groups } - INSTANCE_DELETE_NAT -> "no_result5" { - + sid_delete_nat + INSTANCE_DELETE_DENDRITE_CONFIG -> "no_result6" { + + sid_delete_dendrite_config } } @@ -65,10 +68,11 @@ impl NexusSaga for SagaInstanceDelete { _params: &Self::Params, mut builder: steno::DagBuilder, ) -> Result { - builder.append(instance_delete_nat_action()); + builder.append(instance_delete_dendrite_config_action()); builder.append(instance_delete_record_action()); builder.append(delete_network_interfaces_action()); builder.append(deallocate_external_ip_action()); + builder.append(detach_external_subnets_action()); builder.append(leave_multicast_groups_action()); Ok(builder.build()?) } @@ -112,7 +116,7 @@ async fn sid_delete_network_interfaces( Ok(()) } -async fn sid_delete_nat( +async fn sid_delete_dendrite_config( sagactx: NexusActionContext, ) -> Result<(), ActionError> { let params = sagactx.saga_params::()?; @@ -202,6 +206,23 @@ async fn sid_deallocate_external_ip( Ok(()) } +async fn sid_detach_external_subnets( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + osagactx + .datastore() + .instance_detach_external_subnets(&opctx, params.authz_instance.id()) + .await + .map_err(ActionError::action_failed)?; + Ok(()) +} + #[cfg(test)] mod test { use crate::{ diff --git a/nexus/src/app/sagas/instance_ip_attach.rs b/nexus/src/app/sagas/instance_ip_attach.rs index 182fc11c911..bd6b0c75070 100644 --- a/nexus/src/app/sagas/instance_ip_attach.rs +++ b/nexus/src/app/sagas/instance_ip_attach.rs @@ -4,8 +4,8 @@ use super::instance_common::{ ExternalIpAttach, ModifyStateForExternalIp, VmmAndSledIds, - instance_ip_add_nat, instance_ip_add_opte, instance_ip_get_instance_state, - instance_ip_move_state, instance_ip_remove_opte, + instance_ip_add_nat, instance_ip_add_opte, instance_ip_move_state, + instance_ip_remove_opte, networking_resource_instance_state, }; use super::{ActionRegistry, NexusActionContext, NexusSaga}; use crate::app::sagas::declare_saga_actions; @@ -170,7 +170,7 @@ async fn siia_get_instance_state( sagactx: NexusActionContext, ) -> Result, ActionError> { let params = sagactx.saga_params::()?; - instance_ip_get_instance_state( + networking_resource_instance_state( &sagactx, ¶ms.serialized_authn, ¶ms.authz_instance, @@ -179,7 +179,6 @@ async fn siia_get_instance_state( .await } -// XXX: Need to abstract over v4 and v6 NAT entries when the time comes. async fn siia_nat( sagactx: NexusActionContext, ) -> Result, ActionError> { diff --git a/nexus/src/app/sagas/instance_ip_detach.rs b/nexus/src/app/sagas/instance_ip_detach.rs index a6666683294..95d30681c22 100644 --- a/nexus/src/app/sagas/instance_ip_detach.rs +++ b/nexus/src/app/sagas/instance_ip_detach.rs @@ -4,8 +4,8 @@ use super::instance_common::{ ModifyStateForExternalIp, VmmAndSledIds, instance_ip_add_nat, - instance_ip_add_opte, instance_ip_get_instance_state, - instance_ip_move_state, instance_ip_remove_nat, instance_ip_remove_opte, + instance_ip_add_opte, instance_ip_move_state, instance_ip_remove_nat, + instance_ip_remove_opte, networking_resource_instance_state, }; use super::{ActionRegistry, NexusActionContext, NexusSaga}; use crate::app::sagas::declare_saga_actions; @@ -174,7 +174,7 @@ async fn siid_get_instance_state( sagactx: NexusActionContext, ) -> Result, ActionError> { let params = sagactx.saga_params::()?; - instance_ip_get_instance_state( + networking_resource_instance_state( &sagactx, ¶ms.serialized_authn, ¶ms.authz_instance, diff --git a/nexus/src/app/sagas/instance_start.rs b/nexus/src/app/sagas/instance_start.rs index 94a826364f1..dff273fe71a 100644 --- a/nexus/src/app/sagas/instance_start.rs +++ b/nexus/src/app/sagas/instance_start.rs @@ -16,6 +16,7 @@ use crate::app::instance::{ InstanceEnsureRegisteredApiResources, InstanceRegisterReason, InstanceStateChangeError, }; +use crate::app::instance_network::InstanceNetworkFilters; use crate::app::sagas::declare_saga_actions; use chrono::Utc; use nexus_db_lookup::LookupPath; @@ -741,7 +742,12 @@ async fn sis_dpd_ensure( osagactx .nexus() - .instance_ensure_dpd_config(&opctx, instance_id, &sled.address(), None) + .instance_ensure_dpd_config( + &opctx, + instance_id, + &sled.address(), + InstanceNetworkFilters::all(), + ) .await .map_err(ActionError::action_failed)?; diff --git a/nexus/src/app/sagas/instance_update/mod.rs b/nexus/src/app/sagas/instance_update/mod.rs index 99fe754b320..ece906a09b8 100644 --- a/nexus/src/app/sagas/instance_update/mod.rs +++ b/nexus/src/app/sagas/instance_update/mod.rs @@ -355,6 +355,7 @@ use crate::app::db::model::InstanceState; use crate::app::db::model::MigrationState; use crate::app::db::model::Vmm; use crate::app::db::model::VmmState; +use crate::app::instance_network::InstanceNetworkFilters; use crate::app::sagas::declare_saga_actions; use anyhow::Context; use chrono::Utc; @@ -1068,7 +1069,7 @@ async fn siu_update_network_config( &opctx, instance_id, &sled.address(), - None, + InstanceNetworkFilters::all(), ) .await .map_err(ActionError::action_failed)?; diff --git a/nexus/src/app/sagas/mod.rs b/nexus/src/app/sagas/mod.rs index 5f5c01a30af..a691fe6e9a7 100644 --- a/nexus/src/app/sagas/mod.rs +++ b/nexus/src/app/sagas/mod.rs @@ -50,6 +50,8 @@ pub mod region_snapshot_replacement_step; pub mod region_snapshot_replacement_step_garbage_collect; pub mod snapshot_create; pub mod snapshot_delete; +pub mod subnet_attach; +pub mod subnet_detach; pub mod test_saga; pub mod volume_delete; pub mod volume_remove_rop; @@ -162,6 +164,8 @@ fn make_action_registry() -> ActionRegistry { disk_create::SagaDiskCreate, disk_delete::SagaDiskDelete, finalize_disk::SagaFinalizeDisk, + image_delete::SagaImageDelete, + image_create::SagaImageCreate, instance_create::SagaInstanceCreate, instance_delete::SagaInstanceDelete, instance_ip_attach::SagaInstanceIpAttach, @@ -169,16 +173,8 @@ fn make_action_registry() -> ActionRegistry { instance_migrate::SagaInstanceMigrate, instance_start::SagaInstanceStart, instance_update::SagaInstanceUpdate, + multicast_group_dpd_ensure::SagaMulticastGroupDpdEnsure, project_create::SagaProjectCreate, - snapshot_create::SagaSnapshotCreate, - snapshot_delete::SagaSnapshotDelete, - volume_delete::SagaVolumeDelete, - volume_remove_rop::SagaVolumeRemoveROP, - vpc_create::SagaVpcCreate, - vpc_subnet_create::SagaVpcSubnetCreate, - vpc_subnet_delete::SagaVpcSubnetDelete, - vpc_subnet_update::SagaVpcSubnetUpdate, - image_delete::SagaImageDelete, region_replacement_start::SagaRegionReplacementStart, region_replacement_drive::SagaRegionReplacementDrive, region_replacement_finish::SagaRegionReplacementFinish, @@ -187,8 +183,16 @@ fn make_action_registry() -> ActionRegistry { region_snapshot_replacement_step::SagaRegionSnapshotReplacementStep, region_snapshot_replacement_step_garbage_collect::SagaRegionSnapshotReplacementStepGarbageCollect, region_snapshot_replacement_finish::SagaRegionSnapshotReplacementFinish, - image_create::SagaImageCreate, - multicast_group_dpd_ensure::SagaMulticastGroupDpdEnsure + snapshot_create::SagaSnapshotCreate, + snapshot_delete::SagaSnapshotDelete, + subnet_attach::SagaSubnetAttach, + subnet_detach::SagaSubnetDetach, + volume_delete::SagaVolumeDelete, + volume_remove_rop::SagaVolumeRemoveROP, + vpc_create::SagaVpcCreate, + vpc_subnet_create::SagaVpcSubnetCreate, + vpc_subnet_delete::SagaVpcSubnetDelete, + vpc_subnet_update::SagaVpcSubnetUpdate ]; #[cfg(test)] diff --git a/nexus/src/app/sagas/subnet_attach.rs b/nexus/src/app/sagas/subnet_attach.rs new file mode 100644 index 00000000000..1a665070b18 --- /dev/null +++ b/nexus/src/app/sagas/subnet_attach.rs @@ -0,0 +1,528 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Nexus saga to attach a subnet to an instance. + +use super::ActionRegistry; +use super::NexusActionContext; +use super::NexusSaga; +use super::instance_common::VmmAndSledIds; +use super::instance_common::networking_resource_instance_state; +use crate::app::authn; +use crate::app::authz; +use crate::app::sagas::declare_saga_actions; +use crate::app::sagas::instance_common::delete_subnet_attachment_from_dpd; +use crate::app::sagas::instance_common::delete_subnet_attachment_from_opte; +use crate::app::sagas::instance_common::send_subnet_attachment_to_dpd; +use crate::app::sagas::instance_common::send_subnet_attachment_to_opte; +use anyhow::Context as _; +use nexus_db_model::IpAttachState; +use nexus_db_model::IpNet; +use nexus_db_model::IpVersion; +use nexus_db_queries::db::datastore::ExternalSubnetBeginAttachResult; +use nexus_db_queries::db::datastore::ExternalSubnetCompleteAttachResult; +use nexus_types::external_api::views; +use nexus_types::identity::Resource; +use serde::Deserialize; +use serde::Serialize; +use steno::ActionError; + +declare_saga_actions! { + subnet_attach; + BEGIN_ATTACH -> "begin_attach_result" { + + ssa_begin_attach_subnet + - ssa_begin_attach_subnet_undo + } + + INSTANCE_STATE -> "instance_state" { + + ssa_get_instance_state + } + + NOTIFY_DPD -> "pushed_subnet" { + + ssa_notify_dpd + - ssa_notify_dpd_undo + } + + ENSURE_OPTE_PORT -> "no_result2" { + + ssa_update_opte + - ssa_update_opte_undo + } + + COMPLETE_ATTACH -> "output" { + + ssa_complete_attach + } +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct Params { + pub authz_subnet: authz::ExternalSubnet, + pub ip_version: IpVersion, + pub authz_instance: authz::Instance, + /// Authentication context to use to fetch the instance's current state from + /// the database. + pub serialized_authn: authn::saga::Serialized, +} + +// Mark the external subnet record as "attaching" to the provided instance. +async fn ssa_begin_attach_subnet( + sagactx: NexusActionContext, +) -> Result { + let osagactx = sagactx.user_data(); + let datastore = osagactx.datastore(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + datastore + .begin_attach_subnet( + &opctx, + ¶ms.authz_instance, + ¶ms.authz_subnet, + params.ip_version, + ) + .await + .map_err(ActionError::action_failed) +} + +async fn ssa_begin_attach_subnet_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let osagactx = sagactx.user_data(); + let log = osagactx.log(); + let datastore = osagactx.datastore(); + + debug!(log, "ensuring subnet is detached"); + let params = sagactx.saga_params::()?; + let ExternalSubnetBeginAttachResult { subnet, do_saga } = + sagactx + .lookup::("begin_attach_result")?; + if !do_saga { + return Ok(()); + } + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + match datastore + .external_subnet_complete_op( + &opctx, + subnet.id(), + IpAttachState::Attaching, + IpAttachState::Detached, + ) + .await + .map_err(ActionError::action_failed) + { + Ok(ExternalSubnetCompleteAttachResult::Modified(_)) => Ok(()), + Ok(ExternalSubnetCompleteAttachResult::NoChanges) => { + warn!(log, "subnet is deleted, could not fully detach"); + Ok(()) + } + Err(e) => Err(anyhow::anyhow!("failed to fully detach subnet: {e}",)), + } +} + +async fn ssa_get_instance_state( + sagactx: NexusActionContext, +) -> Result, ActionError> { + let params = sagactx.saga_params::()?; + networking_resource_instance_state( + &sagactx, + ¶ms.serialized_authn, + ¶ms.authz_instance, + "attach", + ) + .await +} + +async fn ssa_notify_dpd( + sagactx: NexusActionContext, +) -> Result, ActionError> { + let params = sagactx.saga_params::()?; + let sled_id = sagactx + .lookup::>("instance_state")? + .map(|ids| ids.sled_id); + let subnet = sagactx + .lookup::("begin_attach_result")?; + send_subnet_attachment_to_dpd( + &sagactx, + ¶ms.serialized_authn, + ¶ms.authz_instance, + sled_id, + subnet, + ) + .await +} + +async fn ssa_notify_dpd_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let Some(subnet) = sagactx.lookup::>("pushed_subnet")? else { + // Never sent the subnet to Dendrite, nothing to undo. + return Ok(()); + }; + delete_subnet_attachment_from_dpd(&sagactx, subnet) + .await + .context("deleting attached subnet from Dendrite") +} + +async fn ssa_update_opte( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let ids = sagactx.lookup::>("instance_state")?; + let subnet = sagactx + .lookup::("begin_attach_result")?; + send_subnet_attachment_to_opte(&sagactx, ids, subnet).await +} + +async fn ssa_update_opte_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let ids = sagactx.lookup::>("instance_state")?; + let subnet = sagactx + .lookup::("begin_attach_result")?; + delete_subnet_attachment_from_opte(&sagactx, ids, subnet) + .await + .context("deleting attached subnet from OPTE") +} + +async fn ssa_complete_attach( + sagactx: NexusActionContext, +) -> Result { + let osagactx = sagactx.user_data(); + let log = osagactx.log(); + let datastore = osagactx.datastore(); + debug!(log, "finalizing subnet attachment"); + let params = sagactx.saga_params::()?; + let ExternalSubnetBeginAttachResult { subnet, do_saga } = + sagactx + .lookup::("begin_attach_result")?; + if !do_saga { + return Ok(subnet.into()); + } + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + match datastore + .external_subnet_complete_op( + &opctx, + subnet.id(), + IpAttachState::Attaching, + IpAttachState::Attached, + ) + .await + { + Ok(ExternalSubnetCompleteAttachResult::Modified(subnet)) => { + Ok(subnet.into()) + } + Ok(ExternalSubnetCompleteAttachResult::NoChanges) => { + warn!(log, "ssa_complete_attach ran more than once"); + Ok(subnet.into()) + } + Err(e) => Err(ActionError::action_failed(e)), + } +} + +#[derive(Debug)] +pub struct SagaSubnetAttach; +impl NexusSaga for SagaSubnetAttach { + const NAME: &'static str = "subnet-attach"; + type Params = Params; + + fn register_actions(registry: &mut ActionRegistry) { + subnet_attach_register_actions(registry); + } + + fn make_saga_dag( + _params: &Self::Params, + mut builder: steno::DagBuilder, + ) -> Result { + builder.append(begin_attach_action()); + builder.append(instance_state_action()); + builder.append(notify_dpd_action()); + builder.append(ensure_opte_port_action()); + builder.append(complete_attach_action()); + Ok(builder.build()?) + } +} + +#[cfg(test)] +pub(crate) mod test { + use super::*; + use crate::app::db; + use crate::app::saga::create_saga_dag; + use crate::app::sagas::test_helpers; + use dropshot::test_util::ClientTestContext; + use nexus_db_lookup::LookupPath; + use nexus_db_queries::context::OpContext; + use nexus_test_utils::resource_helpers::create_default_ip_pools; + use nexus_test_utils::resource_helpers::create_external_subnet_in_pool; + use nexus_test_utils::resource_helpers::create_instance; + use nexus_test_utils::resource_helpers::create_project; + use nexus_test_utils::resource_helpers::create_subnet_pool; + use nexus_test_utils::resource_helpers::create_subnet_pool_member; + use nexus_test_utils_macros::nexus_test; + use nexus_types::external_api::views::ExternalSubnet; + use nexus_types::external_api::views::Project; + use nexus_types::external_api::views::SubnetPool; + use nexus_types::external_api::views::SubnetPoolMember; + use omicron_common::address::IpVersion; + use omicron_common::api::external::SimpleIdentityOrName; + use omicron_uuid_kinds::GenericUuid as _; + use omicron_uuid_kinds::InstanceUuid; + + type ControlPlaneTestContext = + nexus_test_utils::ControlPlaneTestContext; + + const PROJECT_NAME: &str = "cafe"; + const INSTANCE_NAME: &str = "menu"; + const SUBNET_POOL_NAME: &str = "coffee"; + const EXTERNAL_SUBNET_NAME: &str = "espresso"; + + struct Context { + _subnet_pool: SubnetPool, + _member: SubnetPoolMember, + subnet: ExternalSubnet, + _project: Project, + } + + async fn setup_test(client: &ClientTestContext) -> Context { + let subnet_pool = + create_subnet_pool(client, SUBNET_POOL_NAME, IpVersion::V4).await; + let member = create_subnet_pool_member( + client, + SUBNET_POOL_NAME, + "192.0.2.0/24".parse().unwrap(), + ) + .await; + // Need an IPv4 and IPv6 pool. + let _ = create_default_ip_pools(client).await; + let project = create_project(client, PROJECT_NAME).await; + let subnet = create_external_subnet_in_pool( + client, + SUBNET_POOL_NAME, + PROJECT_NAME, + EXTERNAL_SUBNET_NAME, + 28, + ) + .await; + Context { + _subnet_pool: subnet_pool, + _member: member, + subnet, + _project: project, + } + } + + pub async fn new_test_params( + opctx: &OpContext, + datastore: &db::DataStore, + ) -> Params { + let project_name = db::model::Name(PROJECT_NAME.parse().unwrap()); + let (.., authz_subnet, db_subnet) = LookupPath::new(opctx, datastore) + .project_name(&project_name) + .external_subnet_name(&db::model::Name( + EXTERNAL_SUBNET_NAME.parse().unwrap(), + )) + .fetch_for(authz::Action::Modify) + .await + .unwrap(); + let (.., authz_instance) = LookupPath::new(opctx, datastore) + .project_name(&project_name) + .instance_name(&db::model::Name(INSTANCE_NAME.parse().unwrap())) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + Params { + authz_subnet, + ip_version: db_subnet.subnet.ip_version(), + authz_instance, + serialized_authn: authn::saga::Serialized::for_opctx(opctx), + } + } + + async fn verify_clean_slate( + cptestctx: &ControlPlaneTestContext, + instance_id: InstanceUuid, + ) { + let sled_agent = cptestctx.first_sled_agent(); + let datastore = cptestctx.server.server_context().nexus.datastore(); + let opctx = test_helpers::test_opctx(cptestctx); + + // We should have no subnets in the attached state. + let subnets = + datastore.list_all_attached_subnets_batched(&opctx).await.unwrap(); + assert!(subnets.is_empty(), "There should be no attached subnets"); + + // And the sled-agent should have no records either. + let VmmAndSledIds { vmm_id, .. } = + test_helpers::instance_fetch_vmm_and_sled_ids( + cptestctx, + &instance_id, + ) + .await; + assert!( + sled_agent + .attached_subnets + .lock() + .unwrap() + .entry(vmm_id) + .or_default() + .is_empty(), + "sled agent should have no record of any attached subnets", + ); + } + + #[nexus_test(server = crate::Server)] + async fn test_saga_basic_usage_succeeds( + cptestctx: &ControlPlaneTestContext, + ) { + let client = &cptestctx.external_client; + let apictx = &cptestctx.server.server_context(); + let nexus = &apictx.nexus; + let sled_agent = cptestctx.first_sled_agent(); + let opctx = test_helpers::test_opctx(cptestctx); + let datastore = &nexus.db_datastore; + let context = setup_test(&client).await; + let instance = + create_instance(client, PROJECT_NAME, INSTANCE_NAME).await; + let instance_id = InstanceUuid::from_untyped_uuid(instance.id()); + crate::app::sagas::test_helpers::instance_simulate( + cptestctx, + &instance_id, + ) + .await; + + // Run the saga itself. + let params = new_test_params(&opctx, datastore).await; + nexus + .sagas + .saga_execute::(params) + .await + .expect("subnet attach saga should succeed"); + + // The sled agent should now know about these attached subnets. + let VmmAndSledIds { vmm_id, .. } = + test_helpers::instance_fetch_vmm_and_sled_ids( + cptestctx, + &instance_id, + ) + .await; + let on_sled_agent = sled_agent + .attached_subnets + .lock() + .unwrap() + .get(&vmm_id) + .expect("sled agent should have entry for this instance") + .get(&context.subnet.subnet) + .copied() + .unwrap_or_else(|| { + panic!( + "sled agent should have an entry for the subnet attached \ + in the saga: {}", + context.subnet.subnet, + ) + }); + assert!( + on_sled_agent.is_external, + "All attached subnets should be external at this point" + ); + + // The database records should also indicate it's now attached. + let subnets = datastore + .instance_lookup_attached_external_subnets(&opctx, instance_id) + .await + .unwrap(); + assert_eq!(subnets.len(), 1); + assert_eq!(subnets[0].instance_id, instance_id); + assert_eq!(subnets[0].subnet, on_sled_agent.subnet); + } + + #[nexus_test(server = crate::Server)] + async fn test_action_failure_can_unwind( + cptestctx: &ControlPlaneTestContext, + ) { + let log = &cptestctx.logctx.log; + let client = &cptestctx.external_client; + let apictx = &cptestctx.server.server_context(); + let nexus = &apictx.nexus; + let opctx = test_helpers::test_opctx(cptestctx); + let datastore = &nexus.db_datastore; + let _context = setup_test(&client).await; + let instance = + create_instance(client, PROJECT_NAME, INSTANCE_NAME).await; + let instance_id = InstanceUuid::from_untyped_uuid(instance.identity.id); + crate::app::sagas::test_helpers::instance_simulate( + cptestctx, + &instance_id, + ) + .await; + + test_helpers::action_failure_can_unwind::( + nexus, + || Box::pin(new_test_params(&opctx, &datastore)), + || Box::pin(verify_clean_slate(&cptestctx, instance_id)), + log, + ) + .await + } + + #[nexus_test(server = crate::Server)] + async fn test_action_failure_can_unwind_idempotently( + cptestctx: &ControlPlaneTestContext, + ) { + let log = &cptestctx.logctx.log; + let client = &cptestctx.external_client; + let apictx = &cptestctx.server.server_context(); + let nexus = &apictx.nexus; + let opctx = test_helpers::test_opctx(cptestctx); + let datastore = &nexus.db_datastore; + let _context = setup_test(&client).await; + let instance = + create_instance(client, PROJECT_NAME, INSTANCE_NAME).await; + let instance_id = InstanceUuid::from_untyped_uuid(instance.identity.id); + crate::app::sagas::test_helpers::instance_simulate( + cptestctx, + &instance_id, + ) + .await; + + test_helpers::action_failure_can_unwind_idempotently::< + SagaSubnetAttach, + _, + _, + >( + nexus, + || Box::pin(new_test_params(&opctx, &datastore)), + || Box::pin(verify_clean_slate(&cptestctx, instance_id)), + log, + ) + .await + } + + #[nexus_test(server = crate::Server)] + async fn test_actions_succeed_idempotently( + cptestctx: &ControlPlaneTestContext, + ) { + let client = &cptestctx.external_client; + let apictx = &cptestctx.server.server_context(); + let nexus = &apictx.nexus; + let opctx = test_helpers::test_opctx(cptestctx); + let datastore = &nexus.db_datastore; + let _context = setup_test(&client).await; + let instance = + create_instance(client, PROJECT_NAME, INSTANCE_NAME).await; + crate::app::sagas::test_helpers::instance_simulate( + cptestctx, + &InstanceUuid::from_untyped_uuid(instance.identity.id), + ) + .await; + + let params = new_test_params(&opctx, &datastore).await; + let dag = create_saga_dag::(params).unwrap(); + test_helpers::actions_succeed_idempotently(nexus, dag).await; + } +} diff --git a/nexus/src/app/sagas/subnet_detach.rs b/nexus/src/app/sagas/subnet_detach.rs new file mode 100644 index 00000000000..486508ca2ec --- /dev/null +++ b/nexus/src/app/sagas/subnet_detach.rs @@ -0,0 +1,532 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use super::ActionRegistry; +use super::NexusActionContext; +use super::NexusSaga; +use super::instance_common::VmmAndSledIds; +use super::instance_common::networking_resource_instance_state; +use crate::app::authn; +use crate::app::authz; +use crate::app::sagas::declare_saga_actions; +use crate::app::sagas::instance_common::delete_subnet_attachment_from_dpd; +use crate::app::sagas::instance_common::delete_subnet_attachment_from_opte; +use crate::app::sagas::instance_common::send_subnet_attachment_to_dpd; +use crate::app::sagas::instance_common::send_subnet_attachment_to_opte; +use anyhow::Context as _; +use nexus_db_model::IpAttachState; +use nexus_db_queries::db::datastore::ExternalSubnetBeginAttachResult; +use nexus_db_queries::db::datastore::ExternalSubnetCompleteAttachResult; +use nexus_types::external_api::views; +use serde::Deserialize; +use serde::Serialize; +use steno::ActionError; + +declare_saga_actions! { + subnet_detach; + DETACH_SUBNET -> "begin_detach_result" { + + ssd_begin_detach_subnet + - ssd_begin_detach_subnet_undo + } + + INSTANCE_STATE -> "instance_state" { + + ssd_get_instance_state + } + + NOTIFY_DPD -> "no_result0" { + + ssd_notify_dpd + - ssd_notify_dpd_undo + } + + NOTIFY_OPTE -> "no_result1" { + + ssd_notify_opte + - ssd_notify_opte_undo + } + + COMPLETE_DETACH -> "output" { + + ssd_complete_detach + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Params { + pub authz_instance: authz::Instance, + pub authz_subnet: authz::ExternalSubnet, + /// Authentication context to use to fetch the instance's current state from + /// the database. + pub serialized_authn: authn::saga::Serialized, +} + +async fn ssd_begin_detach_subnet( + sagactx: NexusActionContext, +) -> Result { + let osagactx = sagactx.user_data(); + let datastore = osagactx.datastore(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + datastore + .begin_detach_subnet( + &opctx, + ¶ms.authz_instance, + ¶ms.authz_subnet, + ) + .await + .map_err(ActionError::action_failed) +} + +async fn ssd_begin_detach_subnet_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let osagactx = sagactx.user_data(); + let log = osagactx.log(); + let datastore = osagactx.datastore(); + warn!(log, "ssd_begin_detach_subnet_undo: Reverting attached->detaching"); + let params = sagactx.saga_params::()?; + let ExternalSubnetBeginAttachResult { subnet, do_saga: _ } = + sagactx + .lookup::("begin_detach_result")?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + match datastore + .external_subnet_complete_op( + &opctx, + subnet.identity.id.into(), + IpAttachState::Detaching, + IpAttachState::Attached, + ) + .await + { + Ok(ExternalSubnetCompleteAttachResult::Modified(_)) => Ok(()), + Ok(ExternalSubnetCompleteAttachResult::NoChanges) => { + warn!(log, "subnet is deleted, could not reattach"); + Ok(()) + } + Err(e) => Err(anyhow::anyhow!("failed to reattach subnet: {e}")), + } +} + +async fn ssd_get_instance_state( + sagactx: NexusActionContext, +) -> Result, ActionError> { + let params = sagactx.saga_params::()?; + networking_resource_instance_state( + &sagactx, + ¶ms.serialized_authn, + ¶ms.authz_instance, + "detach", + ) + .await +} + +async fn ssd_notify_dpd( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let ExternalSubnetBeginAttachResult { subnet, do_saga } = + sagactx + .lookup::("begin_detach_result")?; + if !do_saga { + return Ok(()); + } + delete_subnet_attachment_from_dpd(&sagactx, subnet.subnet) + .await + .map_err(ActionError::action_failed) +} + +async fn ssd_notify_dpd_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let params = sagactx.saga_params::()?; + let sled_id = sagactx + .lookup::>("instance_state")? + .map(|ids| ids.sled_id); + let subnet = sagactx + .lookup::("begin_detach_result")?; + let ip_subnet = subnet.subnet.subnet; + send_subnet_attachment_to_dpd( + &sagactx, + ¶ms.serialized_authn, + ¶ms.authz_instance, + sled_id, + subnet, + ) + .await + .map(|_| ()) + .with_context(|| { + format!( + "ssd_notify_dpd_undo: sending attached subnet back to \ + dendrite for reattachment, subnet={}", + ip_subnet, + ) + }) +} + +async fn ssd_notify_opte( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let ids = sagactx.lookup::>("instance_state")?; + let subnet = sagactx + .lookup::("begin_detach_result")?; + delete_subnet_attachment_from_opte(&sagactx, ids, subnet) + .await + .map_err(ActionError::action_failed) +} + +async fn ssd_notify_opte_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let ids = sagactx.lookup::>("instance_state")?; + let subnet = sagactx + .lookup::("begin_detach_result")?; + let ip_subnet = subnet.subnet.subnet; + send_subnet_attachment_to_opte(&sagactx, ids, subnet).await.with_context( + || { + format!( + "ssd_notify_opte_undo: sending attached subnet back to \ + OPTE for reattachment, subnet={}", + ip_subnet, + ) + }, + ) +} + +async fn ssd_complete_detach( + sagactx: NexusActionContext, +) -> Result { + let log = sagactx.user_data().log(); + let datastore = sagactx.user_data().datastore(); + let params = sagactx.saga_params::()?; + let ExternalSubnetBeginAttachResult { subnet, do_saga } = + sagactx + .lookup::("begin_detach_result")?; + if !do_saga { + return Ok(subnet.into()); + } + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + match datastore + .external_subnet_complete_op( + &opctx, + subnet.identity.id.into(), + IpAttachState::Detaching, + IpAttachState::Detached, + ) + .await + { + Ok(ExternalSubnetCompleteAttachResult::Modified(subnet)) => { + Ok(subnet.into()) + } + Ok(ExternalSubnetCompleteAttachResult::NoChanges) => { + warn!(log, "ssd_complete_detach ran more than once"); + Ok(subnet.into()) + } + Err(e) => Err(ActionError::action_failed(e)), + } +} + +#[derive(Debug)] +pub struct SagaSubnetDetach; +impl NexusSaga for SagaSubnetDetach { + const NAME: &'static str = "subnet-detach"; + type Params = Params; + + fn register_actions(registry: &mut ActionRegistry) { + subnet_detach_register_actions(registry); + } + + fn make_saga_dag( + _params: &Self::Params, + mut builder: steno::DagBuilder, + ) -> Result { + builder.append(detach_subnet_action()); + builder.append(instance_state_action()); + builder.append(notify_dpd_action()); + builder.append(notify_opte_action()); + builder.append(complete_detach_action()); + Ok(builder.build()?) + } +} + +#[cfg(test)] +pub(crate) mod test { + use super::*; + use crate::app::saga::create_saga_dag; + use crate::app::sagas::subnet_attach::SagaSubnetAttach; + use crate::app::sagas::test_helpers; + use nexus_db_lookup::LookupPath; + use nexus_test_utils::resource_helpers::create_default_ip_pools; + use nexus_test_utils::resource_helpers::create_external_subnet_in_pool; + use nexus_test_utils::resource_helpers::create_instance; + use nexus_test_utils::resource_helpers::create_project; + use nexus_test_utils::resource_helpers::create_subnet_pool; + use nexus_test_utils::resource_helpers::create_subnet_pool_member; + use nexus_test_utils_macros::nexus_test; + use nexus_types::external_api::views::ExternalSubnet; + use nexus_types::external_api::views::Project; + use nexus_types::external_api::views::SubnetPool; + use nexus_types::external_api::views::SubnetPoolMember; + use omicron_common::address::IpVersion; + use omicron_common::api::external::LookupType; + use omicron_common::api::external::SimpleIdentityOrName; + use omicron_uuid_kinds::ExternalSubnetUuid; + use omicron_uuid_kinds::GenericUuid; + use omicron_uuid_kinds::InstanceUuid; + use oxnet::IpNet; + + type ControlPlaneTestContext = + nexus_test_utils::ControlPlaneTestContext; + + const PROJECT_NAME: &str = "cafe"; + const INSTANCE_NAME: &str = "menu"; + const SUBNET_POOL_NAME: &str = "coffee"; + const EXTERNAL_SUBNET_NAME: &str = "espresso"; + + struct Context { + _subnet_pool: SubnetPool, + _member: SubnetPoolMember, + subnet: ExternalSubnet, + _project: Project, + authz_instance: authz::Instance, + detach_params: Params, + } + + async fn setup_test(cptestctx: &ControlPlaneTestContext) -> Context { + let client = &cptestctx.external_client; + let opctx = test_helpers::test_opctx(cptestctx); + let datastore = cptestctx.server.server_context().nexus.datastore(); + + let subnet_pool = + create_subnet_pool(client, SUBNET_POOL_NAME, IpVersion::V4).await; + let member = create_subnet_pool_member( + client, + SUBNET_POOL_NAME, + "192.0.2.0/24".parse().unwrap(), + ) + .await; + // Need an IPv4 and IPv6 pool. + let _ = create_default_ip_pools(client).await; + let project = create_project(client, PROJECT_NAME).await; + + // Lookup the project since we need that to make the authz stuff. + let (.., authz_project, _db_project) = + LookupPath::new(&opctx, datastore) + .project_id(project.identity.id) + .fetch() + .await + .unwrap(); + + // Create the subnet. + let subnet = create_external_subnet_in_pool( + client, + SUBNET_POOL_NAME, + PROJECT_NAME, + EXTERNAL_SUBNET_NAME, + 28, + ) + .await; + let authz_subnet = authz::ExternalSubnet::new( + authz_project.clone(), + ExternalSubnetUuid::from_untyped_uuid(subnet.identity.id), + LookupType::ById(subnet.identity.id), + ); + + // Create an instance. + let instance = + create_instance(client, PROJECT_NAME, INSTANCE_NAME).await; + let authz_instance = authz::Instance::new( + authz_project.clone(), + instance.identity.id, + LookupType::ById(instance.identity.id), + ); + let instance_id = InstanceUuid::from_untyped_uuid(instance.id()); + crate::app::sagas::test_helpers::instance_simulate( + cptestctx, + &instance_id, + ) + .await; + + // Actually attach the subnet. + let attach_params = crate::app::sagas::subnet_attach::Params { + authz_subnet: authz_subnet.clone(), + ip_version: IpVersion::V4.into(), + authz_instance: authz_instance.clone(), + serialized_authn: authn::saga::Serialized::for_opctx(&opctx), + }; + cptestctx + .server + .server_context() + .nexus + .sagas + .saga_execute::(attach_params) + .await + .expect("subnet attach saga should succeed"); + + Context { + _subnet_pool: subnet_pool, + _member: member, + subnet, + _project: project, + authz_instance: authz_instance.clone(), + detach_params: Params { + authz_instance, + authz_subnet, + serialized_authn: authn::saga::Serialized::for_opctx(&opctx), + }, + } + } + + async fn verify_clean_slate( + cptestctx: &ControlPlaneTestContext, + instance_id: InstanceUuid, + subnet: IpNet, + ) { + let sled_agent = cptestctx.first_sled_agent(); + let datastore = cptestctx.server.server_context().nexus.datastore(); + let opctx = test_helpers::test_opctx(cptestctx); + + // We should be back to having the subnet attached in the DB. + let subnets = + datastore.list_all_attached_subnets_batched(&opctx).await.unwrap(); + assert_eq!(subnets.len(), 1); + assert_eq!(subnets[0].instance_id, instance_id); + assert_eq!(subnets[0].subnet, subnet); + + // And the sled-agent should know about it too. + let VmmAndSledIds { vmm_id, .. } = + test_helpers::instance_fetch_vmm_and_sled_ids( + cptestctx, + &instance_id, + ) + .await; + let on_sled_agent = sled_agent + .attached_subnets + .lock() + .unwrap() + .get(&vmm_id) + .expect("sled agent should have entry for this instance") + .get(&subnet) + .copied() + .unwrap_or_else(|| { + panic!( + "sled agent should have an entry for the subnet attached \ + in the saga: {}", + subnet, + ) + }); + assert!( + on_sled_agent.is_external, + "All attached subnets should be external at this point" + ); + } + + #[nexus_test(server = crate::Server)] + async fn test_saga_basic_usage_succeeds( + cptestctx: &ControlPlaneTestContext, + ) { + let apictx = &cptestctx.server.server_context(); + let nexus = &apictx.nexus; + let sled_agent = cptestctx.first_sled_agent(); + let opctx = test_helpers::test_opctx(cptestctx); + let datastore = &nexus.db_datastore; + let context = setup_test(&cptestctx).await; + + // Run the detach saga itself. + nexus + .sagas + .saga_execute::(context.detach_params.clone()) + .await + .expect("subnet detach saga should succeed"); + + // Now the sled agent should not have a record of this subnet. + let instance_id = + InstanceUuid::from_untyped_uuid(context.authz_instance.id()); + let VmmAndSledIds { vmm_id, .. } = + test_helpers::instance_fetch_vmm_and_sled_ids( + cptestctx, + &instance_id, + ) + .await; + assert!( + !sled_agent + .attached_subnets + .lock() + .unwrap() + .get(&vmm_id) + .expect("sled agent should have entry for this instance") + .contains_key(&context.subnet.subnet), + "sled agent should not have an entry for this subnet \ + after running the detach saga" + ); + + // The database records should also indicate it's now detached. + let subnets = datastore + .instance_lookup_attached_external_subnets(&opctx, instance_id) + .await + .unwrap(); + assert!(subnets.is_empty()); + } + + #[nexus_test(server = crate::Server)] + async fn test_action_failure_can_unwind( + cptestctx: &ControlPlaneTestContext, + ) { + let log = &cptestctx.logctx.log; + let apictx = &cptestctx.server.server_context(); + let nexus = &apictx.nexus; + let context = setup_test(cptestctx).await; + let instance_id = + InstanceUuid::from_untyped_uuid(context.authz_instance.id()); + let ip_subnet = context.subnet.subnet; + test_helpers::action_failure_can_unwind::( + nexus, + || Box::pin(async { context.detach_params.clone() }), + || Box::pin(verify_clean_slate(&cptestctx, instance_id, ip_subnet)), + log, + ) + .await + } + + #[nexus_test(server = crate::Server)] + async fn test_action_failure_can_unwind_idempotently( + cptestctx: &ControlPlaneTestContext, + ) { + let log = &cptestctx.logctx.log; + let apictx = &cptestctx.server.server_context(); + let nexus = &apictx.nexus; + let context = setup_test(cptestctx).await; + let instance_id = + InstanceUuid::from_untyped_uuid(context.authz_instance.id()); + let ip_subnet = context.subnet.subnet; + test_helpers::action_failure_can_unwind_idempotently::< + SagaSubnetDetach, + _, + _, + >( + nexus, + || Box::pin(async { context.detach_params.clone() }), + || Box::pin(verify_clean_slate(&cptestctx, instance_id, ip_subnet)), + log, + ) + .await + } + + #[nexus_test(server = crate::Server)] + async fn test_actions_succeed_idempotently( + cptestctx: &ControlPlaneTestContext, + ) { + let apictx = &cptestctx.server.server_context(); + let nexus = &apictx.nexus; + let context = setup_test(cptestctx).await; + let dag = + create_saga_dag::(context.detach_params.clone()) + .unwrap(); + test_helpers::actions_succeed_idempotently(nexus, dag).await; + } +} diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs index 2f490aaeef3..49babb5c0ff 100644 --- a/nexus/test-utils/src/resource_helpers.rs +++ b/nexus/test-utils/src/resource_helpers.rs @@ -18,6 +18,8 @@ use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; use nexus_test_interface::NexusServer; use nexus_types::deployment::Blueprint; use nexus_types::external_api::params; +use nexus_types::external_api::params::ExternalSubnetAllocator; +use nexus_types::external_api::params::PoolSelector; use nexus_types::external_api::params::{ DeviceAccessTokenRequest, DeviceAuthRequest, DeviceAuthVerify, }; @@ -28,6 +30,7 @@ use nexus_types::external_api::views; use nexus_types::external_api::views::AffinityGroup; use nexus_types::external_api::views::AntiAffinityGroup; use nexus_types::external_api::views::Certificate; +use nexus_types::external_api::views::ExternalSubnet; use nexus_types::external_api::views::FloatingIp; use nexus_types::external_api::views::InternetGateway; use nexus_types::external_api::views::InternetGatewayIpAddress; @@ -461,6 +464,33 @@ pub async fn create_subnet_pool_member_with_prefix_lengths( .await } +pub async fn create_external_subnet_in_pool( + client: &ClientTestContext, + pool_name: &str, + project_name: &str, + subnet_name: &str, + prefix_len: u8, +) -> ExternalSubnet { + let params = params::ExternalSubnetCreate { + identity: IdentityMetadataCreateParams { + name: subnet_name.parse().unwrap(), + description: format!("external subnet {subnet_name}"), + }, + allocator: ExternalSubnetAllocator::Auto { + prefix_len, + pool_selector: PoolSelector::Explicit { + pool: pool_name.parse::().unwrap().into(), + }, + }, + }; + object_create( + client, + &format!("/v1/external-subnets?project={project_name}"), + ¶ms, + ) + .await +} + pub async fn create_certificate( client: &ClientTestContext, cert_name: &str, diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index 93e71a8cc45..db10bae4d18 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -206,6 +206,7 @@ multicast_reconciler.period_secs = 60 multicast_reconciler.sled_cache_ttl_secs = 60 multicast_reconciler.backplane_cache_ttl_secs = 120 trust_quorum.period_secs = 60 +attached_subnet_manager.period_secs = 60 [multicast] # Enable multicast functionality for tests (disabled by default in production) From 8e62a718c8bb3a9bc39de2a96b0b1c14331c0a2e Mon Sep 17 00:00:00 2001 From: Benjamin Naecker Date: Mon, 2 Feb 2026 19:30:02 +0000 Subject: [PATCH 3/3] Nexus background task for distributing attached subnets - Add a background task to Nexus that periodically pushes all attached subnets to Dendrite and the sled-agents / OPTE. - Add task output to `omdb`. - Fixes #9581 and fixes #9582 --- common/src/api/internal/shared/mod.rs | 15 +- dev-tools/omdb/src/bin/omdb/nexus.rs | 49 + dev-tools/omdb/tests/env.out | 12 + dev-tools/omdb/tests/successes.out | 18 + nexus-config/src/nexus_config.rs | 16 + nexus/background-task-interface/src/init.rs | 1 + nexus/examples/config-second.toml | 1 + nexus/examples/config.toml | 1 + nexus/src/app/background/init.rs | 19 +- .../app/background/tasks/attached_subnets.rs | 858 ++++++++++++++++++ nexus/src/app/background/tasks/mod.rs | 1 + nexus/types/src/internal_api/background.rs | 34 + smf/nexus/multi-sled/config-partial.toml | 1 + smf/nexus/single-sled/config-partial.toml | 1 + 14 files changed, 1019 insertions(+), 8 deletions(-) create mode 100644 nexus/src/app/background/tasks/attached_subnets.rs diff --git a/common/src/api/internal/shared/mod.rs b/common/src/api/internal/shared/mod.rs index d429e2312d2..d7b73e8ada9 100644 --- a/common/src/api/internal/shared/mod.rs +++ b/common/src/api/internal/shared/mod.rs @@ -11,10 +11,13 @@ use crate::{ zpool_name::ZpoolName, }; use daft::Diffable; -use omicron_uuid_kinds::{ - DatasetUuid, ExternalSubnetUuid, InstanceUuid, RackUuid, SledUuid, -}; -use omicron_uuid_kinds::{ExternalZpoolUuid, PropolisUuid}; +use omicron_uuid_kinds::DatasetUuid; +use omicron_uuid_kinds::ExternalSubnetUuid; +use omicron_uuid_kinds::ExternalZpoolUuid; +use omicron_uuid_kinds::InstanceUuid; +use omicron_uuid_kinds::PropolisUuid; +use omicron_uuid_kinds::RackUuid; +use omicron_uuid_kinds::SledUuid; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; use schemars::JsonSchema; use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; @@ -1106,7 +1109,7 @@ pub enum AttachedSubnetId { } /// All details about an attached subnet and the Instance it's attached to. -#[derive(Debug)] +#[derive(Clone, Copy, Debug)] pub struct AttachedSubnet { /// ID of the rack hosting this instance. // @@ -1123,7 +1126,7 @@ pub struct AttachedSubnet { pub vmm_id: PropolisUuid, /// ID of the instance pub instance_id: InstanceUuid, - /// ID of the subnet itself. + /// ID of the attached subnet itself. pub subnet_id: AttachedSubnetId, /// The IP subnet that's attached. pub subnet: IpNet, diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs index e0be6d58ccd..071ee0aebe8 100644 --- a/dev-tools/omdb/src/bin/omdb/nexus.rs +++ b/dev-tools/omdb/src/bin/omdb/nexus.rs @@ -52,6 +52,7 @@ use nexus_types::deployment::OximeterReadMode; use nexus_types::deployment::OximeterReadPolicy; use nexus_types::fm; use nexus_types::internal_api::background::AbandonedVmmReaperStatus; +use nexus_types::internal_api::background::AttachedSubnetManagerStatus; use nexus_types::internal_api::background::BlueprintPlannerStatus; use nexus_types::internal_api::background::BlueprintRendezvousStats; use nexus_types::internal_api::background::BlueprintRendezvousStatus; @@ -1204,6 +1205,9 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) { "abandoned_vmm_reaper" => { print_task_abandoned_vmm_reaper(details); } + "attached_subnet_manager" => { + print_task_attached_subnet_manager_status(details); + } "blueprint_planner" => { print_task_blueprint_planner(details); } @@ -2228,6 +2232,51 @@ fn print_task_probe_distributor(details: &serde_json::Value) { }; } +fn print_task_attached_subnet_manager_status(details: &serde_json::Value) { + match serde_json::from_value::(details.clone()) + { + Err(error) => eprintln!( + "warning: failed to interpret task details: {:?}: {:?}", + error, details + ), + Ok(AttachedSubnetManagerStatus { db_error, dendrite, sled }) => { + if let Some(err) = db_error { + println!( + " error accessing database to list attached subnets:" + ); + println!(" {err}"); + } + if dendrite.is_empty() { + println!(" no dendrite instances found"); + } else { + for (loc, details) in dendrite.iter() { + println!(" dendrite instance on switch {loc}"); + println!( + " n_subnets_removed={}", + details.n_subnets_removed + ); + println!( + " n_subnets_added={}", + details.n_subnets_added + ); + println!( + " n_subnets_total={}", + details.n_total_subnets + ); + } + } + if sled.is_empty() { + println!(" no sleds found"); + } else { + for (sled_id, details) in sled.iter() { + println!(" sled {sled_id}"); + println!(" n_subnets={}", details.n_subnets); + } + } + } + }; +} + fn print_task_read_only_region_replacement_start(details: &serde_json::Value) { match serde_json::from_value::( details.clone(), diff --git a/dev-tools/omdb/tests/env.out b/dev-tools/omdb/tests/env.out index 499aa8d2d09..c817a6e40ee 100644 --- a/dev-tools/omdb/tests/env.out +++ b/dev-tools/omdb/tests/env.out @@ -34,6 +34,10 @@ task: "alert_dispatcher" dispatches queued alerts to receivers +task: "attached_subnet_manager" + distributes attached subnets to sleds and switch + + task: "bfd_manager" Manages bidirectional fowarding detection (BFD) configuration on rack switches @@ -271,6 +275,10 @@ task: "alert_dispatcher" dispatches queued alerts to receivers +task: "attached_subnet_manager" + distributes attached subnets to sleds and switch + + task: "bfd_manager" Manages bidirectional fowarding detection (BFD) configuration on rack switches @@ -495,6 +503,10 @@ task: "alert_dispatcher" dispatches queued alerts to receivers +task: "attached_subnet_manager" + distributes attached subnets to sleds and switch + + task: "bfd_manager" Manages bidirectional fowarding detection (BFD) configuration on rack switches diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index 109d74823a6..1d7fa22dc88 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -269,6 +269,10 @@ task: "alert_dispatcher" dispatches queued alerts to receivers +task: "attached_subnet_manager" + distributes attached subnets to sleds and switch + + task: "bfd_manager" Manages bidirectional fowarding detection (BFD) configuration on rack switches @@ -573,6 +577,13 @@ task: "alert_dispatcher" alerts dispatched: 0 alerts with no receivers subscribed: 0 +task: "attached_subnet_manager" + configured period: every m + last completed activation: , triggered by + started at (s ago) and ran for ms + no dendrite instances found + no sleds found + task: "bfd_manager" configured period: every s last completed activation: , triggered by @@ -1153,6 +1164,13 @@ task: "alert_dispatcher" alerts dispatched: 0 alerts with no receivers subscribed: 0 +task: "attached_subnet_manager" + configured period: every m + last completed activation: , triggered by + started at (s ago) and ran for ms + no dendrite instances found + no sleds found + task: "bfd_manager" configured period: every s last completed activation: , triggered by diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index d9defb6c1bc..77e11c8bd39 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -433,6 +433,8 @@ pub struct BackgroundTaskConfig { pub multicast_reconciler: MulticastGroupReconcilerConfig, /// configuration for trust quorum manager task pub trust_quorum: TrustQuorumConfig, + /// configuration for the attached subnet manager + pub attached_subnet_manager: AttachedSubnetManagerConfig, } #[serde_as] @@ -1016,6 +1018,15 @@ pub struct PackageConfig { pub default_region_allocation_strategy: RegionAllocationStrategy, } +#[serde_as] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AttachedSubnetManagerConfig { + /// period (in seconds) for periodic activations of the background task that + /// pushes attached subnets to the switches and sleds. + #[serde_as(as = "DurationSeconds")] + pub period_secs: Duration, +} + // Re-export SchemeName from nexus-types for use in config parsing. pub use nexus_types::authn::SchemeName; @@ -1241,6 +1252,7 @@ mod test { probe_distributor.period_secs = 50 multicast_reconciler.period_secs = 60 trust_quorum.period_secs = 60 + attached_subnet_manager.period_secs = 60 [default_region_allocation_strategy] type = "random" seed = 0 @@ -1501,6 +1513,9 @@ mod test { trust_quorum: TrustQuorumConfig { period_secs: Duration::from_secs(60), }, + attached_subnet_manager: AttachedSubnetManagerConfig { + period_secs: Duration::from_secs(60), + }, }, multicast: MulticastConfig { enabled: false }, default_region_allocation_strategy: @@ -1605,6 +1620,7 @@ mod test { probe_distributor.period_secs = 47 multicast_reconciler.period_secs = 60 trust_quorum.period_secs = 60 + attached_subnet_manager.period_secs = 60 [default_region_allocation_strategy] type = "random" diff --git a/nexus/background-task-interface/src/init.rs b/nexus/background-task-interface/src/init.rs index 7eb7e3ad11a..23a7899eb9b 100644 --- a/nexus/background-task-interface/src/init.rs +++ b/nexus/background-task-interface/src/init.rs @@ -56,6 +56,7 @@ pub struct BackgroundTasks { pub task_probe_distributor: Activator, pub task_multicast_reconciler: Activator, pub task_trust_quorum_manager: Activator, + pub task_attached_subnet_manager: Activator, // Handles to activate background tasks that do not get used by Nexus // at-large. These background tasks are implementation details as far as diff --git a/nexus/examples/config-second.toml b/nexus/examples/config-second.toml index dfbe17968f5..d182ea2874c 100644 --- a/nexus/examples/config-second.toml +++ b/nexus/examples/config-second.toml @@ -188,6 +188,7 @@ multicast_reconciler.period_secs = 60 # Default: 86400 seconds (24 hours) - refreshed on-demand when validation fails # multicast_reconciler.backplane_cache_ttl_secs = 86400 trust_quorum.period_secs = 60 +attached_subnet_manager.period_secs = 60 [default_region_allocation_strategy] # allocate region on 3 random distinct zpools, on 3 random distinct sleds. diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml index bab6eeb7f5f..9342662cb4d 100644 --- a/nexus/examples/config.toml +++ b/nexus/examples/config.toml @@ -172,6 +172,7 @@ multicast_reconciler.period_secs = 60 # Default: 86400 seconds (24 hours) - refreshed on-demand when validation fails # multicast_reconciler.backplane_cache_ttl_secs = 86400 trust_quorum.period_secs = 60 +attached_subnet_manager.period_secs = 60 [default_region_allocation_strategy] # allocate region on 3 random distinct zpools, on 3 random distinct sleds. diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index 16452d155bd..5282940619c 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -91,6 +91,7 @@ use super::Driver; use super::driver::TaskDefinition; use super::tasks::abandoned_vmm_reaper; use super::tasks::alert_dispatcher::AlertDispatcher; +use super::tasks::attached_subnets; use super::tasks::bfd; use super::tasks::blueprint_execution; use super::tasks::blueprint_load; @@ -264,6 +265,7 @@ impl BackgroundTasksInitializer { task_probe_distributor: Activator::new(), task_multicast_reconciler: Activator::new(), task_trust_quorum_manager: Activator::new(), + task_attached_subnet_manager: Activator::new(), // Handles to activate background tasks that do not get used by Nexus // at-large. These background tasks are implementation details as far as @@ -353,6 +355,7 @@ impl BackgroundTasksInitializer { task_probe_distributor, task_multicast_reconciler, task_trust_quorum_manager, + task_attached_subnet_manager, // Add new background tasks here. Be sure to use this binding in a // call to `Driver::register()` below. That's what actually wires // up the Activator to the corresponding background task. @@ -1091,7 +1094,7 @@ impl BackgroundTasksInitializer { period: config.sp_ereport_ingester.period_secs, task_impl: Box::new(ereport_ingester::SpEreportIngester::new( datastore.clone(), - resolver, + resolver.clone(), nexus_id, config.sp_ereport_ingester.disable, )), @@ -1145,13 +1148,25 @@ impl BackgroundTasksInitializer { description: "Drive trust quorum reconfigurations to completion", period: config.trust_quorum.period_secs, task_impl: Box::new(trust_quorum::TrustQuorumManager::new( - datastore, + datastore.clone(), )), opctx: opctx.child(BTreeMap::new()), watchers: vec![], activator: task_trust_quorum_manager, }); + driver.register(TaskDefinition { + name: "attached_subnet_manager", + description: "distributes attached subnets to sleds and switch", + period: config.attached_subnet_manager.period_secs, + task_impl: Box::new(attached_subnets::Manager::new( + resolver, datastore, + )), + opctx: opctx.child(BTreeMap::new()), + watchers: vec![], + activator: task_attached_subnet_manager, + }); + driver } } diff --git a/nexus/src/app/background/tasks/attached_subnets.rs b/nexus/src/app/background/tasks/attached_subnets.rs new file mode 100644 index 00000000000..1671723ac06 --- /dev/null +++ b/nexus/src/app/background/tasks/attached_subnets.rs @@ -0,0 +1,858 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Background task to push configuration for attached subnets to the switch and +//! OPTE instances. + +use crate::app::background::BackgroundTask; +use crate::app::dpd_clients; +use dpd_client::types::AttachedSubnetEntry; +use dpd_client::types::InstanceTarget; +use futures::FutureExt as _; +use futures::StreamExt; +use futures::TryStreamExt; +use futures::future::BoxFuture; +use internal_dns_resolver::Resolver; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::DataStore; +use nexus_types::internal_api::background::AttachedSubnetManagerStatus; +use nexus_types::internal_api::background::DendriteSubnetDetails; +use nexus_types::internal_api::background::SledSubnetDetails; +use omicron_common::api::external::SwitchLocation; +use omicron_common::api::internal::shared; +use omicron_common::api::internal::shared::AttachedSubnetId; +use omicron_uuid_kinds::PropolisUuid; +use omicron_uuid_kinds::SledUuid; +use oxnet::IpNet; +use serde_json::Value; +use serde_json::json; +use sled_agent_client::types::AttachedSubnet; +use sled_agent_client::types::AttachedSubnets; +use slog::Logger; +use slog::debug; +use slog::error; +use slog::info; +use slog_error_chain::InlineErrorChain; +use std::collections::HashMap; +use std::collections::HashSet; +use std::collections::hash_map::Entry; +use std::net::Ipv6Addr; +use std::sync::Arc; + +/// Background task that pushes attached subnets. +pub struct Manager { + resolver: Resolver, + datastore: Arc, +} + +impl Manager { + pub fn new(resolver: Resolver, datastore: Arc) -> Self { + Self { resolver, datastore } + } + + async fn send_attachments_to_dendrite( + &self, + log: &Logger, + clients: &HashMap, + attachments: &[shared::AttachedSubnet], + ) -> HashMap { + // Dendrite provides an API to list all attached subnets, and to delete / + // put one at a time, rather than putting an entire _set_ of mappings. + // That means we have to do the diff on the client side, to compute the + // set of subnets to remove and add, separately for each Dendrite + // instance. + // + // First, build the full set of expected mappings for all switches. This + // is only derived from the database. + let desired_attachments = attachments + .iter() + .map(|at| { + let shared::AttachedSubnet { + sled_ip, subnet, mac, vni, .. + } = at; + let tgt = InstanceTarget { + internal_ip: *sled_ip, + inner_mac: dpd_client::types::MacAddr { + a: mac.into_array(), + }, + // Safety: We've collected this from the DB, so it has to + // have been a valid 24-bit VNI. + vni: u32::from(*vni).try_into().unwrap(), + }; + (*subnet, tgt) + }) + .collect::>(); + + // Loop over each Dendrite instance, find the subnets it has and the + // diff we need to apply. + let mut res = HashMap::<_, DendriteSubnetDetails>::new(); + for (loc, client) in clients.iter() { + let details = res.entry(*loc).or_default(); + let existing_attachments = match client + .attached_subnet_list_stream(None) + .map(|entry| { + entry.map(|e| { + let AttachedSubnetEntry { subnet, tgt } = e; + (subnet, tgt) + }) + }) + .try_collect::>() + .await + { + Ok(m) => { + details.n_total_subnets = m.len(); + m + } + Err(e) => { + let err = InlineErrorChain::new(&e); + details.errors.push(err.to_string()); + error!( + log, + "failed to list existing attached subnets \ + from switch, it will be skipped this time"; + "switch_location" => %loc, + "error" => err, + ); + continue; + } + }; + + // Set-diff them to figure out the changes we need to apply. + let AttachedSubnetDiff { to_add, to_remove } = + AttachedSubnetDiff::new( + &existing_attachments, + &desired_attachments, + ); + + // Remove any attachments Dendrite has that we no longer want. + for subnet in to_remove.into_iter() { + match client.attached_subnet_delete(subnet).await { + Ok(_) => { + details.n_subnets_removed += 1; + details.n_total_subnets -= 1; + debug!( + log, + "deleted subnet from dendrite"; + "subnet" => %subnet, + "switch" => %loc, + ); + } + Err(e) => { + let err = InlineErrorChain::new(&e); + details.errors.push(err.to_string()); + error!( + log, + "failed to delete subnet from dendrite"; + "subnet" => %subnet, + "switch" => %loc, + "error" => err, + ); + } + } + } + + // Add attachments we do want. + for (subnet, target) in to_add.into_iter() { + match client.attached_subnet_create(subnet, target).await { + Ok(_) => { + details.n_subnets_added += 1; + details.n_total_subnets += 1; + debug!( + log, + "created attached subnet on dendrite"; + "subnet" => %subnet, + "target" => ?target, + "switch" => %loc, + ); + } + Err(e) => { + let err = InlineErrorChain::new(&e); + details.errors.push(err.to_string()); + error!( + log, + "failed to create subnet on dendrite"; + "subnet" => %subnet, + "target" => ?target, + "switch" => %loc, + "error" => err, + ); + } + } + } + } + res + } + + async fn send_attachments_to_sled_agents( + &self, + log: &Logger, + opctx: &OpContext, + attachments: &[shared::AttachedSubnet], + ) -> HashMap { + // Map of all the clients needed to send data to sled agents. Update + // this lazily while we iterate below. + let mut clients = HashMap::new(); + + // Send to one sled at a time, all the attachments for its instances. + // + // We might want to reorder the mapping above to be only by VMM ID. That + // would spread the work out across the different sleds, rather than + // sending each sled all its subnets in a short time. But it's not clear + // there's any meaningful difference at this point. + let attachments_by_sled = + group_attached_subnets_by_sled_and_vmm(attachments); + let mut res = HashMap::<_, SledSubnetDetails>::new(); + 'sleds: for ((sled_id, sled_ip), attachments_by_sled) in + attachments_by_sled.iter() + { + let details = res.entry(*sled_id).or_default(); + + // Look up the client or get it from the datastore if needed. + let client = match clients.entry(*sled_id) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => { + match nexus_networking::sled_client( + &self.datastore, + opctx, + *sled_id, + log, + ) + .await + { + Ok(client) => { + debug!( + log, + "fetched new sled client"; + "sled_id" => %sled_id, + "sled_ip" => %sled_ip, + ); + entry.insert(client) + } + Err(e) => { + let e = InlineErrorChain::new(&e); + let n_vmms = attachments_by_sled.len(); + let n_subnets: usize = attachments_by_sled + .values() + .map(|att| att.subnets.len()) + .sum(); + let message = format!( + "failed to lookup client for sled with attached \ + subnets, sled_id={sled_id}, sled_ip={sled_ip}, \ + n_vmms={n_vmms}, n_subnets={n_subnets}, error={e}" + ); + details.errors.push(message); + error!( + log, + "no client for sled with attached subnets"; + "sled_id" => %sled_id, + "sled_ip" => %sled_ip, + "n_vmms" => n_vmms, + "n_subnets" => n_subnets, + "error" => &e, + ); + continue 'sleds; + } + } + } + }; + + // Send all the attached subnets, per-VMM, and update our counters. + for (vmm_id, attachments) in attachments_by_sled.iter() { + match client.vmm_put_attached_subnets(vmm_id, attachments).await + { + Ok(_) => { + details.n_subnets += attachments.subnets.len(); + debug!( + log, + "sent attached subnets to sled"; + "n_subnets" => attachments.subnets.len(), + "sled_ip" => %sled_ip, + "vmm_id" => %vmm_id, + ); + } + Err(e) => { + let err = InlineErrorChain::new(&e); + details.errors.push(err.to_string()); + error!( + log, + "failed to send attached subnets to sled"; + "sled_ip" => %sled_ip, + "vmm_id" => %vmm_id, + "error" => err, + ); + } + } + } + } + res + } +} + +type VmmAttachedSubnetMap = HashMap; +type SledAttachedSubnetMap = + HashMap<(SledUuid, Ipv6Addr), VmmAttachedSubnetMap>; + +// Organize attached subnets first by sled, then by VMM on the sled. +// +// We really care about the instance ID, but the sled-agent API exposes a +// per-VMM endpoint for replacing attached subnets. +fn group_attached_subnets_by_sled_and_vmm( + attachments: &[shared::AttachedSubnet], +) -> SledAttachedSubnetMap { + let mut attachments_by_sled = SledAttachedSubnetMap::new(); + for attachment in attachments.iter() { + attachments_by_sled + .entry((attachment.sled_id, attachment.sled_ip)) + .or_default() + .entry(attachment.vmm_id) + .or_insert_with(|| AttachedSubnets { subnets: Vec::new() }) + .subnets + .push(AttachedSubnet { + subnet: attachment.subnet, + is_external: matches!( + attachment.subnet_id, + AttachedSubnetId::External(_) + ), + }); + } + attachments_by_sled +} + +// Diff between existing and desired attached subnets on a Dendrite instance. +struct AttachedSubnetDiff<'a> { + to_add: HashMap<&'a IpNet, &'a InstanceTarget>, + to_remove: HashSet<&'a IpNet>, +} + +impl<'a> AttachedSubnetDiff<'a> { + fn new( + existing: &'a HashMap, + desired: &'a HashMap, + ) -> AttachedSubnetDiff<'a> { + let mut to_remove = HashSet::new(); + let mut to_add = HashMap::new(); + + // Add all those in existing, but not desired, to `to_remove` + for (subnet, target) in existing.iter() { + match desired.get(subnet) { + Some(desired_target) if desired_target == target => {} + None | Some(_) => { + let _ = to_remove.insert(subnet); + } + } + } + + // Add all those in desired, but not existing, to `to_add`. + for (subnet, desired_target) in desired.iter() { + match existing.get(subnet) { + Some(target) if desired_target == target => {} + None | Some(_) => { + let _ = to_add.insert(subnet, desired_target); + } + } + } + AttachedSubnetDiff { to_add, to_remove } + } +} + +impl BackgroundTask for Manager { + fn activate<'a>( + &'a mut self, + opctx: &'a OpContext, + ) -> BoxFuture<'a, Value> { + // Do a page at a time of: + // + // - Fetch a page of attached subnets + // - Construct Dendrite requests and push + // - Construct sled-agent requests and push, need an API for this first + async { + let log = &opctx.log; + info!(log, "starting attached subnet manager work"); + let mut out = AttachedSubnetManagerStatus::default(); + + // Fetch Dendrite clients. We will need to move this into the loop + // when we resolve #5201, since we'll look up Dendrite instances per + // rack in that case. + let maybe_dpd_clients = dpd_clients(&self.resolver, &log) + .await + .inspect_err(|e| { + error!( + log, + "failed to lookup Dendrite clients, will \ + not be able to forward attachments to Dendrite"; + "error" => %e + ) + }) + .ok(); + let attachments = match self + .datastore + .list_all_attached_subnets_batched(opctx) + .await + { + Ok(attachments) => { + debug!( + log, + "listed attached subnets"; + "n_subnets" => attachments.len(), + ); + attachments + } + Err(e) => { + let err = InlineErrorChain::new(&e); + error!( + log, + "failed to list attached subnets"; + "error" => &err, + ); + out.db_error = Some(err.to_string()); + return json!(out); + } + }; + + out.dendrite = match &maybe_dpd_clients { + None => HashMap::new(), + Some(clients) => { + self.send_attachments_to_dendrite( + &log, + &clients, + &attachments, + ) + .await + } + }; + out.sled = self + .send_attachments_to_sled_agents(&log, opctx, &attachments) + .await; + json!(out) + } + .boxed() + } +} + +#[cfg(test)] +mod test { + use super::*; + use async_bb8_diesel::AsyncRunQueryDsl as _; + use chrono::Utc; + use diesel::ExpressionMethods as _; + use diesel::QueryDsl as _; + use dpd_client::types::MacAddr; + use dpd_client::types::Vni; + use nexus_db_model::IpAttachState; + use nexus_db_schema::schema::external_subnet::dsl; + use nexus_test_utils::resource_helpers::create_default_ip_pools; + use nexus_test_utils::resource_helpers::create_external_subnet_in_pool; + use nexus_test_utils::resource_helpers::create_instance; + use nexus_test_utils::resource_helpers::create_project; + use nexus_test_utils::resource_helpers::create_subnet_pool; + use nexus_test_utils::resource_helpers::create_subnet_pool_member; + use nexus_test_utils_macros::nexus_test; + use omicron_common::address::IpVersion; + use omicron_common::api::internal::shared; + use omicron_uuid_kinds::GenericUuid; + use std::collections::BTreeSet; + + #[test] + fn attached_subnet_diff_works_with_empty_sets() { + let desired = HashMap::new(); + let existing = HashMap::new(); + let AttachedSubnetDiff { to_add, to_remove } = + AttachedSubnetDiff::new(&existing, &desired); + assert!(to_add.is_empty()); + assert!(to_remove.is_empty()); + } + + #[test] + fn attached_subnet_diff_removes_entries() { + let existing = HashMap::from([( + "10.0.0.0/16".parse().unwrap(), + InstanceTarget { + inner_mac: MacAddr { a: [0xa8, 0x40, 0x25, 0x00, 0x00, 0x00] }, + internal_ip: "fd00::1".parse().unwrap(), + vni: Vni(100), + }, + )]); + let desired = HashMap::new(); + let AttachedSubnetDiff { to_add, to_remove } = + AttachedSubnetDiff::new(&existing, &desired); + assert!(to_add.is_empty()); + assert_eq!(to_remove, existing.keys().collect()); + } + + #[test] + fn attached_subnet_diff_adds_entries() { + let desired = HashMap::from([( + "10.0.0.0/16".parse().unwrap(), + InstanceTarget { + inner_mac: MacAddr { a: [0xa8, 0x40, 0x25, 0x00, 0x00, 0x00] }, + internal_ip: "fd00::1".parse().unwrap(), + vni: Vni(100), + }, + )]); + let existing = HashMap::new(); + let AttachedSubnetDiff { to_add, to_remove } = + AttachedSubnetDiff::new(&existing, &desired); + assert!(to_remove.is_empty()); + assert_eq!(to_add, desired.iter().collect()); + } + + #[test] + fn attached_subnet_leaves_valid_entries() { + let desired = HashMap::from([( + "10.0.0.0/16".parse().unwrap(), + InstanceTarget { + inner_mac: MacAddr { a: [0xa8, 0x40, 0x25, 0x00, 0x00, 0x00] }, + internal_ip: "fd00::1".parse().unwrap(), + vni: Vni(100), + }, + )]); + let existing = desired.clone(); + let AttachedSubnetDiff { to_add, to_remove } = + AttachedSubnetDiff::new(&existing, &desired); + assert!(to_remove.is_empty()); + assert!(to_add.is_empty()); + } + + #[test] + fn attached_subnet_modifies_changed_entries() { + let existing = HashMap::from([( + "10.0.0.0/16".parse().unwrap(), + InstanceTarget { + inner_mac: MacAddr { a: [0xa8, 0x40, 0x25, 0xff, 0xff, 0xff] }, + internal_ip: "fd00::1".parse().unwrap(), + vni: Vni(100), + }, + )]); + let desired = HashMap::from([( + "10.0.0.0/16".parse().unwrap(), + InstanceTarget { + inner_mac: MacAddr { a: [0xa8, 0x40, 0x25, 0x00, 0x00, 0x00] }, + internal_ip: "fd00::1".parse().unwrap(), + vni: Vni(100), + }, + )]); + let AttachedSubnetDiff { to_add, to_remove } = + AttachedSubnetDiff::new(&existing, &desired); + + // We should remove everything, because the mapping we have is wrong. + assert_eq!(to_remove, existing.keys().collect()); + + // And add the entire new mapping. + assert_eq!(to_add, desired.iter().collect()); + } + + type ControlPlaneTestContext = + nexus_test_utils::ControlPlaneTestContext; + + // NOTE: This is also a test of the datastore method + // `list_all_attached_subnets_batched`, but it relies on enough related + // records that it's easier to write here. This task is also the only + // consume right now. + #[nexus_test(server = crate::Server)] + async fn test_attached_subnet_manager(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + let nexus = &cptestctx.server.server_context().nexus; + let datastore = nexus.datastore(); + let opctx = OpContext::for_tests( + cptestctx.logctx.log.clone(), + datastore.clone(), + ); + let mut task = + Manager::new(nexus.resolver().clone(), datastore.clone()); + + // Create a resource hierarchy. + let _subnet_pool = + create_subnet_pool(client, "apple", IpVersion::V6).await; + let _member = create_subnet_pool_member( + client, + "apple", + "2001:db8::/48".parse().unwrap(), + ) + .await; + let (_v4_pool, _v6_pool) = create_default_ip_pools(client).await; + let _project = create_project(client, "banana").await; + + // Now let's create some instances + let n_instances = 6; + let mut instances = Vec::with_capacity(n_instances); + for i in 0..n_instances { + let instance = + create_instance(client, "banana", &format!("mango-{i}")).await; + instances.push(instance); + } + + // And some eternal subnets. + let n_subnets = 4; + let n_to_attach = 2; + let mut subnets = Vec::with_capacity(n_subnets); + for i in 0..n_subnets { + let subnet = create_external_subnet_in_pool( + client, + "apple", + "banana", + &format!("plum-{i}"), + 64, + ) + .await; + subnets.push(subnet); + } + + // To start, nothing should be marked as attached. + let attached = + datastore.list_all_attached_subnets_batched(&opctx).await.unwrap(); + assert!( + attached.is_empty(), + "No subnets should be attached at this point" + ); + + // And the task should report the same thing. + let result = task.activate(&opctx).await; + let result = result.as_object().expect("should be a JSON object"); + assert_task_result_is_empty(&result); + + // Mark a few of them in a transitional state, as if the saga for + // attaching them has partially run. These should still now show up. + for (inst, sub) in + instances.iter().take(n_to_attach).zip(subnets.iter()) + { + let n_rows = + diesel::update(dsl::external_subnet.find(sub.identity.id)) + .set(( + dsl::instance_id.eq(inst.identity.id), + dsl::attach_state.eq(IpAttachState::Attaching), + dsl::time_modified.eq(Utc::now()), + )) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) + .await + .unwrap(); + assert_eq!(n_rows, 1); + } + let attached = + datastore.list_all_attached_subnets_batched(&opctx).await.unwrap(); + assert!( + attached.is_empty(), + "No subnets should be attached at this point" + ); + + // And the task should still report the same thing. + let result = task.activate(&opctx).await; + let result = result.as_object().expect("should be a JSON object"); + assert_task_result_is_empty(result); + + // "Complete" the attachment above, and we should see them + for (_inst, sub) in + instances.iter().take(n_to_attach).zip(subnets.iter()) + { + let n_rows = + diesel::update(dsl::external_subnet.find(sub.identity.id)) + .set(( + dsl::attach_state.eq(IpAttachState::Attached), + dsl::time_modified.eq(Utc::now()), + )) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) + .await + .unwrap(); + assert_eq!(n_rows, 1); + } + let attached = + datastore.list_all_attached_subnets_batched(&opctx).await.unwrap(); + assert_eq!(attached.len(), n_to_attach); + assert_eq!( + attached + .iter() + .map(|att| att.instance_id.into_untyped_uuid()) + .collect::>(), + instances + .iter() + .take(n_to_attach) + .map(|inst| inst.identity.id) + .collect::>(), + "Attached subnets aren't attached to the expected instances" + ); + + // The task should also report sending the same items. + let result = task.activate(&opctx).await; + let result = result.as_object().expect("should be a JSON object"); + assert_task_result_has(result, &attached, &attached, &[]); + + // And the sled agent itself should have records for these subnets. + { + let sa = &cptestctx.sled_agents[0]; + assert!( + attached.iter().all(|att| att.sled_id == sa.sled_agent_id()) + ); + let sa_subnets = sa.sled_agent().attached_subnets.lock().unwrap(); + assert_eq!(sa_subnets.len(), attached.len()); + for att in attached.iter() { + let on_sled = sa_subnets.get(&att.vmm_id).unwrap(); + assert_eq!(on_sled.len(), 1); + let attached_on_sled = on_sled.iter().next().unwrap(); + assert!(attached_on_sled.is_external); + assert_eq!(attached_on_sled.subnet, att.subnet); + } + } + + // Now detach the one on the first instance, and check that the + // sled-agent no longer has it. + let removed = attached + .into_iter() + .find(|att| { + att.instance_id.into_untyped_uuid() == instances[0].identity.id + }) + .unwrap(); + let AttachedSubnetId::External(removed_id) = &removed.subnet_id else { + panic!("All subnets are external right now"); + }; + let n_rows = diesel::update( + dsl::external_subnet.find(removed_id.into_untyped_uuid()), + ) + .set(( + dsl::attach_state.eq(IpAttachState::Detached), + dsl::instance_id.eq(Option::::None), + dsl::time_modified.eq(Utc::now()), + )) + .execute_async(&*datastore.pool_connection_for_tests().await.unwrap()) + .await + .unwrap(); + assert_eq!(n_rows, 1); + let attached = + datastore.list_all_attached_subnets_batched(&opctx).await.unwrap(); + assert_eq!(attached.len(), 1); + assert_eq!( + attached[0].instance_id.into_untyped_uuid(), + instances[1].identity.id, + "Attached subnets aren't attached to the expected instances" + ); + + // We'd like to test that the sled-agent has exactly what we send it. + // That's not really possible, nor necessary. Attached subnets get + // removed from the sled-agent when they're detached in two scenarios: + // + // - The subnet detach saga runs + // - An instance delete saga runs + // + // In both of those cases, the attachment will go away. It's also not + // possible for the saga to succeed and the sled-agent to retain the + // attachment. (Absent a bug!) If we fail to run the part of the saga + // that deletes the attachment, the whole saga would fail and unwind. + // + // So we can simulate the subnet-detach operation above we just ran by + // deleting the relevant attachment from the simulated sled agent. + { + cptestctx.sled_agents[0] + .sled_agent() + .attached_subnets + .lock() + .unwrap() + .get_mut(&removed.vmm_id) + .expect("Should still have mapping for this VMM") + .remove(&removed.subnet) + .expect("Should have removed an actual mapping"); + } + + // The task should also report sending the same items. + let result = task.activate(&opctx).await; + let result = result.as_object().expect("should be a JSON object"); + assert_task_result_has(result, &attached, &[], &[removed]); + + // And the sled agent itself should have a record for this one attached + // subnet. + { + let sa = &cptestctx.sled_agents[0]; + assert!( + attached.iter().all(|att| att.sled_id == sa.sled_agent_id()) + ); + let sa_subnets = sa.sled_agent().attached_subnets.lock().unwrap(); + + // It should still have a mapping for the _instance_ we detached the + // subnet from, but that should be empty. + assert_eq!(sa_subnets.len(), attached.len() + 1); + assert!(sa_subnets.get(&removed.vmm_id).unwrap().is_empty()); + + // All the attached subnets should still be there. + for att in attached.iter() { + let on_sled = sa_subnets.get(&att.vmm_id).unwrap(); + assert_eq!(on_sled.len(), 1); + let attached_on_sled = on_sled.iter().next().unwrap(); + assert!(attached_on_sled.is_external); + assert_eq!(attached_on_sled.subnet, att.subnet); + } + } + } + + fn assert_task_result_has( + result: &serde_json::Map, + attached: &[shared::AttachedSubnet], + added: &[shared::AttachedSubnet], + removed: &[shared::AttachedSubnet], + ) { + // No errors + assert!(result.contains_key("db_error")); + assert!(result["db_error"].is_null()); + + // Dendrite should have added all the mappings. + assert!(result.contains_key("dendrite")); + let dendrite = result["dendrite"].as_object().unwrap(); + let switch0 = dendrite["switch0"].as_object().unwrap(); + assert!(switch0["errors"].as_array().unwrap().is_empty()); + assert_eq!( + switch0["n_subnets_added"].as_number().unwrap().as_u64().unwrap(), + u64::try_from(added.len()).unwrap() + ); + assert_eq!( + switch0["n_subnets_removed"].as_number().unwrap().as_u64().unwrap(), + u64::try_from(removed.len()).unwrap(), + ); + assert_eq!( + switch0["n_total_subnets"].as_number().unwrap().as_u64().unwrap(), + u64::try_from(attached.len()).unwrap(), + ); + + // And we should have one sled, with `n_to_attach` attached subnets. + assert!(result.contains_key("sled")); + let sled = result["sled"].as_object().unwrap(); + let sled_id = attached[0].sled_id.to_string(); + assert!(sled.contains_key(&sled_id)); + let this_sled = sled[&sled_id].as_object().unwrap(); + assert!(this_sled["errors"].as_array().unwrap().is_empty()); + assert_eq!( + this_sled["n_subnets"].as_number().unwrap().as_u64().unwrap(), + u64::try_from(attached.len()).unwrap() + ); + } + + fn assert_task_result_is_empty( + result: &serde_json::Map, + ) { + // No errors. + assert!(result.contains_key("db_error")); + assert!(result["db_error"].is_null()); + + // We have a dendrite key with a switch, but there are no mappings. + assert!(result.contains_key("dendrite")); + let dendrite = result["dendrite"].as_object().unwrap(); + let switch0 = dendrite["switch0"].as_object().unwrap(); + assert!(switch0["errors"].as_array().unwrap().is_empty()); + assert_eq!( + switch0["n_subnets_added"].as_number().unwrap().as_u64().unwrap(), + 0 + ); + assert_eq!( + switch0["n_subnets_removed"].as_number().unwrap().as_u64().unwrap(), + 0 + ); + assert_eq!( + switch0["n_total_subnets"].as_number().unwrap().as_u64().unwrap(), + 0 + ); + + // We have no sleds, because there are no active VMMs. + assert!(result.contains_key("sled")); + assert!(result["sled"].as_object().unwrap().is_empty()); + } +} diff --git a/nexus/src/app/background/tasks/mod.rs b/nexus/src/app/background/tasks/mod.rs index 8eef0f2aeab..b79e1ea356d 100644 --- a/nexus/src/app/background/tasks/mod.rs +++ b/nexus/src/app/background/tasks/mod.rs @@ -6,6 +6,7 @@ pub mod abandoned_vmm_reaper; pub mod alert_dispatcher; +pub mod attached_subnets; pub mod bfd; pub mod blueprint_execution; pub mod blueprint_load; diff --git a/nexus/types/src/internal_api/background.rs b/nexus/types/src/internal_api/background.rs index 26b9edba6e9..9fdf5c3c87a 100644 --- a/nexus/types/src/internal_api/background.rs +++ b/nexus/types/src/internal_api/background.rs @@ -11,6 +11,7 @@ use iddqd::IdOrdItem; use iddqd::IdOrdMap; use iddqd::id_upcast; use omicron_common::api::external::Generation; +use omicron_common::api::external::SwitchLocation; use omicron_uuid_kinds::AlertReceiverUuid; use omicron_uuid_kinds::AlertUuid; use omicron_uuid_kinds::BlueprintUuid; @@ -939,6 +940,39 @@ pub enum TrustQuorumManagerStatus { Error(String), } +#[derive(Default, Deserialize, Serialize)] +pub struct AttachedSubnetManagerStatus { + /// Error reaching the database to fetch attached subnets. + pub db_error: Option, + /// Details about attached subnets sent to Dendrite instances. + pub dendrite: HashMap, + /// Details about attached subnets sent to sleds. + pub sled: HashMap, +} + +/// Details about attached subnets sent to a single Dendrite instance. +#[derive(Default, Deserialize, Serialize)] +pub struct DendriteSubnetDetails { + /// Number of new subnets added. + pub n_subnets_added: usize, + /// Number of existing subnets removed. + pub n_subnets_removed: usize, + /// Total number of subnets on the instance after the operation is + /// completed. + pub n_total_subnets: usize, + /// Errors encountered when sending attached subnets. + pub errors: Vec, +} + +/// Details about attached subnets sent to a single sled. +#[derive(Default, Deserialize, Serialize)] +pub struct SledSubnetDetails { + /// Total number of subnets, across all instances on the sled. + pub n_subnets: usize, + /// Errors encountered when sending attached subnets. + pub errors: Vec, +} + #[cfg(test)] mod test { use super::TufRepoInfo; diff --git a/smf/nexus/multi-sled/config-partial.toml b/smf/nexus/multi-sled/config-partial.toml index 96b0281db1d..3d25fb03f47 100644 --- a/smf/nexus/multi-sled/config-partial.toml +++ b/smf/nexus/multi-sled/config-partial.toml @@ -116,6 +116,7 @@ trust_quorum.period_secs = 60 # TTL for backplane topology cache (static platform configuration) # Default: 86400 seconds (24 hours) - refreshed on-demand when validation fails # multicast_reconciler.backplane_cache_ttl_secs = 86400 +attached_subnet_manager.period_secs = 60 [default_region_allocation_strategy] # by default, allocate across 3 distinct sleds diff --git a/smf/nexus/single-sled/config-partial.toml b/smf/nexus/single-sled/config-partial.toml index 07d847b8ae0..e856deb2920 100644 --- a/smf/nexus/single-sled/config-partial.toml +++ b/smf/nexus/single-sled/config-partial.toml @@ -116,6 +116,7 @@ multicast_reconciler.period_secs = 60 # TTL for backplane topology cache (static platform configuration) # Default: 86400 seconds (24 hours) - refreshed on-demand when validation fails # multicast_reconciler.backplane_cache_ttl_secs = 86400 +attached_subnet_manager.period_secs = 60 [default_region_allocation_strategy] # by default, allocate without requirement for distinct sleds.