feat: Switch (NVSwitch) state machine

book/src/SUMMARY.md

@@ -21,6 +21,7 @@
     - [NIC and Port selection](architecture/infiniband/nic_selection.md)
 - [State Machines]()
     - [Managed Host](architecture/state_machines/managedhost.md)
+    - [Switch](architecture/state_machines/switch.md)
 
 # Manuals
 

book/src/architecture/state_machines/switch.md

@@ -0,0 +1,81 @@
+# Switch State Diagram
+
+This document describes the Finite State Machine (FSM) for Switches in Carbide: lifecycle from creation through configuration, validation, ready, optional reprovisioning, and deletion.
+
+## High-Level Overview
+
+The main flow shows the primary states and transitions:
+
+<div style="width: 180%; background: white; margin-left: -40%;">
+
+```plantuml
+@startuml
+skinparam state {
+  BackgroundColor White
+}
+
+state "Initializing" as Initializing
+state "Configuring\n(RotateOsPassword)" as Configuring
+state "Validating" as Validating
+state "BomValidating" as BomValidating
+state "Ready" as Ready
+state "ReProvisioning\n(Start → WaitFirmware)" as ReProvisioning
+state "Error" as Error
+state "Deleting" as Deleting
+
+[*] --> Initializing : Switch created
+
+Initializing --> Configuring : init complete
+Configuring --> Validating : rotate password done
+Validating --> BomValidating : validation complete
+BomValidating --> Ready : BOM validation complete
+
+Ready --> Deleting : marked for deletion
+Ready --> ReProvisioning : reprovision requested
+
+ReProvisioning --> Ready : firmware upgrade Completed
+ReProvisioning --> Error : firmware upgrade Failed
+
+Error --> Deleting : marked for deletion
+
+Deleting --> [*] : final delete
+@enduml
+```
+
+</div>
+
+## States
+
+| State | Description |
+|-------|-------------|
+| **Initializing** | Switch is created in Carbide; controller performs initial setup. |
+| **Configuring** | Switch is being configured (rotate OS password). Sub-state: `RotateOsPassword`. |
+| **Validating** | Switch is being validated. Sub-state: `ValidateComplete`. |
+| **BomValidating** | BOM (Bill of Materials) validation. Sub-state: `BomValidateComplete`. |
+| **Ready** | Switch is ready for use. From here it can be deleted, or reprovisioning can be requested. |
+| **ReProvisioning** | Reprovisioning (e.g. firmware update) in progress. Sub-states: `Start`, `WaitFirmwareUpdateCompletion`. Completion is driven by `firmware_upgrade_status` (Completed → Ready, Failed → Error). |
+| **Error** | Switch is in error (e.g. firmware upgrade failed). Can transition to Deleting if marked for deletion; otherwise waits for manual intervention. |
+| **Deleting** | Switch is being removed; ends in final delete (terminal). |
+
+## Transitions (by trigger)
+
+| From | To | Trigger / Condition |
+|------|-----|----------------------|
+| *(create)* | Initializing | Switch created |
+| Initializing | Configuring (RotateOsPassword) | Initialization complete |
+| Configuring (RotateOsPassword) | Validating (ValidateComplete) | OS password rotated |
+| Validating (ValidateComplete) | BomValidating (BomValidateComplete) | Validation complete |
+| BomValidating (BomValidateComplete) | Ready | BOM validation complete |
+| Ready | Deleting | `deleted` set (marked for deletion) |
+| Ready | ReProvisioning (Start) | `switch_reprovisioning_requested` is set |
+| ReProvisioning (Start) | ReProvisioning (WaitFirmwareUpdateCompletion) | Reprovision triggered |
+| ReProvisioning (WaitFirmwareUpdateCompletion) | Ready | `firmware_upgrade_status == Completed` |
+| ReProvisioning (WaitFirmwareUpdateCompletion) | Error | `firmware_upgrade_status == Failed { cause }` |
+| Error | Deleting | `deleted` set (marked for deletion) |
+| Deleting | *(end)* | Final delete committed |
+
+## Implementation
+
+- **State type**: `SwitchControllerState` in `crates/api-model/src/switch/mod.rs`.
+- **Handlers**: `crates/api/src/state_controller/switch/` — one module per top-level state (`initializing`, `configuring`, `validating`, `bom_validating`, `ready`, `reprovisioning`, `error_state`, `deleting`).
+- **Orchestration**: `SwitchStateHandler` in `handler.rs` delegates to the handler for the current `controller_state`.

crates/admin-cli/src/expected_switch/add/args.rs

@@ -37,6 +37,8 @@ pub struct Args {
     )]
     pub switch_serial_number: String,
 
+    #[clap(long, help = "NVOS MAC address of the expected switch")]
+    pub nvos_mac_address: Option<MacAddress>,
     #[clap(long, help = "NVOS username of the expected switch")]
     pub nvos_username: Option<String>,
     #[clap(long, help = "NVOS password of the expected switch")]
@@ -89,6 +91,7 @@ impl From<Args> for rpc::forge::ExpectedSwitch {
             switch_serial_number: value.switch_serial_number,
             metadata: Some(metadata),
             rack_id: value.rack_id,
+            nvos_mac_address: value.nvos_mac_address.map(|m| m.to_string()),
             nvos_username: value.nvos_username,
             nvos_password: value.nvos_password,
         }

crates/admin-cli/src/expected_switch/common.rs

@@ -25,6 +25,7 @@ pub struct ExpectedSwitchJson {
     pub bmc_username: String,
     pub bmc_password: String,
     pub switch_serial_number: String,
+    pub nvos_mac_address: Option<MacAddress>,
     pub nvos_username: Option<String>,
     pub nvos_password: Option<String>,
     #[serde(default)]

crates/admin-cli/src/expected_switch/show/cmd.rs

@@ -100,6 +100,7 @@ fn convert_and_print_into_nice_table(
     table.set_titles(row![
         "Serial Number",
         "BMC Mac",
+        "NVOS Mac",
         "Interface IP",
         "Associated Machine",
         "Name",
@@ -141,6 +142,7 @@ fn convert_and_print_into_nice_table(
         table.add_row(row![
             expected_switch.switch_serial_number,
             expected_switch.bmc_mac_address,
+            expected_switch.nvos_mac_address.as_deref().unwrap_or_default(),
             machine_interface
                 .map(|x| x.address.join("\n"))
                 .unwrap_or("Undiscovered".to_string()),

crates/admin-cli/src/expected_switch/update/args.rs

@@ -27,6 +27,9 @@ use uuid::Uuid;
 "bmc_username",
 "bmc_password",
 "switch_serial_number",
+"nvos_mac_address",
+"nvos_username",
+"nvos_password",
 ])))]
 pub struct Args {
     #[clap(short = 'a', long, help = "BMC MAC Address of the expected switch")]
@@ -59,6 +62,12 @@ pub struct Args {
     )]
     pub switch_serial_number: Option<String>,
 
+    #[clap(
+        long,
+        group = "group",
+        help = "NVOS MAC address of the expected switch"
+    )]
+    pub nvos_mac_address: Option<MacAddress>,
     #[clap(long, group = "group", help = "NVOS username of the expected switch")]
     pub nvos_username: Option<String>,
     #[clap(long, group = "group", help = "NVOS password of the expected switch")]
@@ -140,6 +149,7 @@ impl TryFrom<Args> for rpc::forge::ExpectedSwitch {
                 labels: crate::metadata::parse_rpc_labels(args.labels.unwrap_or_default()),
             }),
             rack_id: args.rack_id,
+            nvos_mac_address: args.nvos_mac_address.map(|m| m.to_string()),
         })
     }
 }

crates/admin-cli/src/rpc.rs

@@ -564,7 +564,6 @@ impl ApiClient {
 
         Ok(self.0.update_expected_machine(request).await?)
     }
-
     pub async fn replace_all_expected_machines(
         &self,
         expected_machine_list: Vec<ExpectedMachineJson>,
@@ -635,6 +634,7 @@ impl ApiClient {
                     bmc_username: switch.bmc_username,
                     bmc_password: switch.bmc_password,
                     switch_serial_number: switch.switch_serial_number,
+                    nvos_mac_address: switch.nvos_mac_address.map(|m| m.to_string()),
                     nvos_username: switch.nvos_username,
                     nvos_password: switch.nvos_password,
                     metadata: switch.metadata,

crates/api-db/migrations/20260316120000_switch_reprovisioning_requested.sql

@@ -0,0 +1,6 @@
+-- Add switch_reprovisioning_requested and firmware_upgrade_status columns to switches table.
+-- switch_reprovisioning_requested: when set by an external entity, the state controller (when switch is Ready) transitions to ReProvisioning::Start.
+-- firmware_upgrade_status: used during ReProvisioning (WaitFirmwareUpdateCompletion): Started, InProgress, Completed, Failed.
+ALTER TABLE switches
+    ADD COLUMN switch_reprovisioning_requested JSONB,
+    ADD COLUMN firmware_upgrade_status JSONB;

crates/api-db/migrations/20260316120002_expected_switches_nvos_mac_address.sql

@@ -0,0 +1,3 @@
+-- Add nvos_mac_address column to expected_switches table (NVOS host MAC, similar to bmc_mac_address).
+ALTER TABLE expected_switches
+    ADD COLUMN nvos_mac_address macaddr;

crates/api-db/src/expected_switch.rs

@@ -165,9 +165,9 @@ pub async fn create(
 ) -> DatabaseResult<ExpectedSwitch> {
     let id = switch.expected_switch_id.unwrap_or_else(Uuid::new_v4);
     let query = "INSERT INTO expected_switches
-             (expected_switch_id, bmc_mac_address, bmc_username, bmc_password, serial_number, metadata_name, metadata_description, rack_id, metadata_labels, nvos_username, nvos_password)
+             (expected_switch_id, bmc_mac_address, bmc_username, bmc_password, serial_number, metadata_name, metadata_description, rack_id, metadata_labels, nvos_username, nvos_password, nvos_mac_address)
              VALUES
-             ($1::uuid, $2::macaddr, $3::varchar, $4::varchar, $5::varchar, $6::varchar, $7::varchar, $8::varchar, $9::jsonb, $10::varchar, $11::varchar) RETURNING *";
+             ($1::uuid, $2::macaddr, $3::varchar, $4::varchar, $5::varchar, $6::varchar, $7::varchar, $8::varchar, $9::jsonb, $10::varchar, $11::varchar, $12::macaddr) RETURNING *";
 
     sqlx::query_as(query)
         .bind(id)
@@ -181,6 +181,7 @@ pub async fn create(
         .bind(sqlx::types::Json(&switch.metadata.labels))
         .bind(&switch.nvos_username)
         .bind(&switch.nvos_password)
+        .bind(switch.nvos_mac_address)
         .fetch_one(txn)
         .await
         .map_err(|err: sqlx::Error| match err {

crates/api-db/src/switch.rs

@@ -22,7 +22,9 @@ use chrono::prelude::*;
 use config_version::{ConfigVersion, Versioned};
 use futures::StreamExt;
 use model::controller_outcome::PersistentStateHandlerOutcome;
-use model::switch::{NewSwitch, Switch, SwitchControllerState};
+use model::switch::{
+    FirmwareUpgradeStatus, NewSwitch, Switch, SwitchControllerState, SwitchReprovisionRequest,
+};
 use sqlx::PgConnection;
 
 use crate::{
@@ -81,6 +83,8 @@ pub async fn create(txn: &mut PgConnection, new_switch: &NewSwitch) -> DatabaseR
             version,
         },
         controller_state_outcome: None,
+        switch_reprovisioning_requested: None,
+        firmware_upgrade_status: None,
     })
 }
 
@@ -128,6 +132,18 @@ pub async fn find_by_id(txn: &mut PgConnection, id: &SwitchId) -> DatabaseResult
     }
 }
 
+pub async fn find_by_host_mac_address(
+    txn: &mut PgConnection,
+    host_mac_address: &MacAddress,
+) -> DatabaseResult<Option<Switch>> {
+    let query = sqlx::query_as::<_, Switch>("SELECT * FROM switches WHERE host_mac_address = $1");
+    query
+        .bind(host_mac_address)
+        .fetch_optional(txn)
+        .await
+        .map_err(|e| DatabaseError::new("find_by_host_mac_address", e))
+}
+
 pub async fn find_all(txn: &mut PgConnection) -> DatabaseResult<Vec<SwitchId>> {
     let query = sqlx::query_as::<_, SwitchId>("SELECT id FROM switches WHERE deleted IS NULL");
 
@@ -207,6 +223,62 @@ pub async fn update_controller_state_outcome(
     Ok(())
 }
 
+/// Sets switch_reprovisioning_requested on the switch. Can be called from any state machine or
+/// service. When the switch is in Ready state, the switch state controller will observe the flag
+/// and transition to ReProvisioning::Start.
+pub async fn set_switch_reprovisioning_requested(
+    txn: &mut PgConnection,
+    switch_id: SwitchId,
+    initiator: &str,
+) -> DatabaseResult<()> {
+    let req = SwitchReprovisionRequest {
+        requested_at: Utc::now(),
+        initiator: initiator.to_string(),
+    };
+    let query =
+        "UPDATE switches SET switch_reprovisioning_requested = $1 WHERE id = $2 RETURNING id";
+    sqlx::query_as::<_, SwitchId>(query)
+        .bind(sqlx::types::Json(req))
+        .bind(switch_id)
+        .fetch_optional(txn)
+        .await
+        .map_err(|e| DatabaseError::new("set_switch_reprovisioning_requested", e))?;
+    Ok(())
+}
+
+/// Clears switch_reprovisioning_requested. Typically called when reprovisioning completes or is
+/// cancelled.
+pub async fn clear_switch_reprovisioning_requested(
+    txn: &mut PgConnection,
+    switch_id: SwitchId,
+) -> DatabaseResult<()> {
+    let query =
+        "UPDATE switches SET switch_reprovisioning_requested = NULL WHERE id = $1 RETURNING id";
+    sqlx::query_as::<_, SwitchId>(query)
+        .bind(switch_id)
+        .fetch_optional(txn)
+        .await
+        .map_err(|e| DatabaseError::new("clear_switch_reprovisioning_requested", e))?;
+    Ok(())
+}
+
+/// Sets firmware_upgrade_status on the switch. Call from any state machine or service to report
+/// upgrade progress. WaitFirmwareUpdateCompletion reads this: Completed → Ready, Failed → Error.
+pub async fn update_firmware_upgrade_status(
+    txn: &mut PgConnection,
+    switch_id: SwitchId,
+    status: Option<&FirmwareUpgradeStatus>,
+) -> DatabaseResult<()> {
+    let query = "UPDATE switches SET firmware_upgrade_status = $1 WHERE id = $2 RETURNING id";
+    sqlx::query_as::<_, SwitchId>(query)
+        .bind(status.map(|s| sqlx::types::Json(s.clone())))
+        .bind(switch_id)
+        .fetch_optional(txn)
+        .await
+        .map_err(|e| DatabaseError::new("update_firmware_upgrade_status", e))?;
+    Ok(())
+}
+
 pub async fn mark_as_deleted<'a>(
     switch: &'a mut Switch,
     txn: &mut PgConnection,

crates/api-model/src/expected_switch.rs

@@ -28,11 +28,13 @@ use uuid::Uuid;
 
 use crate::metadata::{Metadata, default_metadata_for_deserializer};
 
-#[derive(Debug, Clone, Default, Deserialize)]
+#[derive(Default, Debug, Clone, Deserialize)]
+#[serde(default)]
 pub struct ExpectedSwitch {
     #[serde(default)]
     pub expected_switch_id: Option<Uuid>,
     pub bmc_mac_address: MacAddress,
+    pub nvos_mac_address: Option<MacAddress>,
     pub bmc_username: String,
     pub serial_number: String,
     pub bmc_password: String,
@@ -52,9 +54,12 @@ impl<'r> FromRow<'r, PgRow> for ExpectedSwitch {
             labels: labels.0,
         };
 
+        let nvos_mac_address: Option<MacAddress> = row.try_get("nvos_mac_address").ok();
+
         Ok(ExpectedSwitch {
             expected_switch_id: row.try_get("expected_switch_id")?,
             bmc_mac_address: row.try_get("bmc_mac_address")?,
+            nvos_mac_address,
             bmc_username: row.try_get("bmc_username")?,
             serial_number: row.try_get("serial_number")?,
             bmc_password: row.try_get("bmc_password")?,
@@ -75,6 +80,7 @@ impl From<ExpectedSwitch> for rpc::forge::ExpectedSwitch {
                     value: u.to_string(),
                 }),
             bmc_mac_address: expected_switch.bmc_mac_address.to_string(),
+            nvos_mac_address: expected_switch.nvos_mac_address.map(|m| m.to_string()),
             bmc_username: expected_switch.bmc_username,
             bmc_password: expected_switch.bmc_password,
             switch_serial_number: expected_switch.serial_number,
@@ -92,6 +98,15 @@ impl TryFrom<rpc::forge::ExpectedSwitch> for ExpectedSwitch {
     fn try_from(rpc: rpc::forge::ExpectedSwitch) -> Result<Self, Self::Error> {
         let bmc_mac_address = MacAddress::try_from(rpc.bmc_mac_address.as_str())
             .map_err(|_| RpcDataConversionError::InvalidMacAddress(rpc.bmc_mac_address.clone()))?;
+        let nvos_mac_address = if rpc.nvos_mac_address.is_none() {
+            None
+        } else {
+            let mac_address = rpc.nvos_mac_address.unwrap();
+            Some(
+                MacAddress::try_from(mac_address.as_str())
+                    .map_err(|_| RpcDataConversionError::InvalidMacAddress(mac_address))?,
+            )
+        };
         let expected_switch_id = rpc
             .expected_switch_id
             .map(|u| {
@@ -111,6 +126,7 @@ impl TryFrom<rpc::forge::ExpectedSwitch> for ExpectedSwitch {
             nvos_password: rpc.nvos_password,
             metadata,
             rack_id: rpc.rack_id,
+            nvos_mac_address,
         })
     }
 }