test(conformance): cover relay reconnect recovery

architecture/build.md

@@ -271,6 +271,10 @@ for explicit publication.
 CLI conformance runs after target provisioning and operates only through the
 configured OpenShell CLI. The smoke scenario verifies the black-box sandbox
 lifecycle by creating, inspecting, executing in, and deleting a sandbox. The
+dedicated sandbox-lifecycle scenarios verify stop/start persistence, a bounded
+two-sandbox startup burst across the supervisor's initial reconnect, and a
+deterministic in-flight relay across a forced reconnect when the driver supplies
+a helper. The guest runs at most two archive tests concurrently. The
 file-transfer scenario verifies portable upload and download behavior, Git-aware
 filtering, and sandbox workspace path safety.
 Feature suites use the same disposable guest but may provision isolated
@@ -321,6 +325,9 @@ with musl, and the gateway and supervisor with GNU. Image assembly stages
 the gateway, sandbox, and supervisor as separate binaries for their respective
 Dockerfiles. The helpers stage binaries under `artifacts/binaries` so local and
 CI builds expose the same inputs to tmachine and image assembly. The Ubuntu
+tmachine conformance tests consume the staged nextest archive, so local source
+changes to those tests require `nix run .#build-artifacts-test-archives` before
+running tmachine. The Ubuntu
 Docker and Fedora Podman environments import both local runtime images and
 configure the gateway to use them. The Ubuntu `deb` installer consumes
 `artifacts/packages/openshell.deb`; the `binaries` installer remains available

architecture/gateway.md

@@ -410,6 +410,10 @@ the serving gateway retries ownership lookup until the normal relay wait
 deadline. Each retry re-reads the owner record, so a supervisor reconnect or
 heartbeat can surface a new owner; if no fresh reachable owner appears before
 the deadline, the client operation fails rather than electing an owner itself.
+The owning gateway replays unclaimed relay opens when a supervisor reconnects,
+including when the previous session ended before the new one registered. Relay
+delivery is tracked by session ID so an open queued during session setup is not
+sent twice to the same supervisor connection.
 Provider-readiness reports, endpoint-status reports, and provider-status reads
 also follow the durable owner record through unary peer RPCs. The owning replica
 validates the current supervisor session and keeps the in-memory evidence; a

crates/openshell-conformance/src/lib.rs

@@ -26,8 +26,9 @@ use self::executor::{CliExecutionError, CliExecutor, ProcessCli};
 pub use scenarios::{
     FILE_TRANSFER_GIT_FILTERING_SCENARIO, FILE_TRANSFER_PATH_SAFETY_SCENARIO,
     FILE_TRANSFER_ROUND_TRIP_SCENARIO, FILE_TRANSFER_SCENARIO, MECHANISTIC_PROPOSAL_SCENARIO,
-    NEW_HOSTNAME_PROPOSAL_SCENARIO, POLICY_LOCAL_SCENARIO, SANDBOX_LIFECYCLE_SCENARIO,
-    SMOKE_SCENARIO,
+    NEW_HOSTNAME_PROPOSAL_SCENARIO, POLICY_LOCAL_SCENARIO,
+    SANDBOX_LIFECYCLE_RELAY_READINESS_SCENARIO, SANDBOX_LIFECYCLE_RELAY_RECONNECT_SCENARIO,
+    SANDBOX_LIFECYCLE_SCENARIO, SANDBOX_LIFECYCLE_STATE_TRANSITIONS_SCENARIO, SMOKE_SCENARIO,
 };
 
 /// An installed conformance scenario.

crates/openshell-conformance/src/scenarios/mod.rs

@@ -15,5 +15,8 @@ pub use file_transfer::{
 pub use policy_behavior::{
     MECHANISTIC_PROPOSAL_SCENARIO, NEW_HOSTNAME_PROPOSAL_SCENARIO, POLICY_LOCAL_SCENARIO,
 };
-pub use sandbox_lifecycle::SANDBOX_LIFECYCLE_SCENARIO;
+pub use sandbox_lifecycle::{
+    SANDBOX_LIFECYCLE_RELAY_READINESS_SCENARIO, SANDBOX_LIFECYCLE_RELAY_RECONNECT_SCENARIO,
+    SANDBOX_LIFECYCLE_SCENARIO, SANDBOX_LIFECYCLE_STATE_TRANSITIONS_SCENARIO,
+};
 pub use smoke::SMOKE_SCENARIO;

crates/openshell-conformance/src/scenarios/sandbox_lifecycle.rs

@@ -13,6 +13,9 @@ const CREATE_TIMEOUT: Duration = Duration::from_mins(10);
 const COMMAND_TIMEOUT: Duration = Duration::from_mins(2);
 const TRANSITION_TIMEOUT: Duration = Duration::from_mins(4);
 const TRANSITION_INTERVAL: Duration = Duration::from_secs(2);
+const RELAY_ATTACHMENT_TIMEOUT: Duration = Duration::from_secs(30);
+const RELAY_SANDBOX_ATTEMPTS: usize = 6;
+const RELAY_SCENARIO_CONCURRENCY: usize = 2;
 
 #[derive(Debug, Deserialize)]
 struct SandboxState {
@@ -23,17 +26,248 @@ struct SandboxState {
 /// Certify sandbox stop, start, and deletion lifecycle behavior.
 pub const SANDBOX_LIFECYCLE_SCENARIO: Scenario = Scenario {
     name: "sandbox-lifecycle",
-    description: "Verify sandbox stop, start, and deletion lifecycle behavior.",
+    description: "Verify sandbox state transitions and relay readiness across reconnects.",
     run: run_sandbox_lifecycle,
 };
 
+/// Certify sandbox stop, start, workspace preservation, and deletion behavior.
+pub const SANDBOX_LIFECYCLE_STATE_TRANSITIONS_SCENARIO: Scenario = Scenario {
+    name: "sandbox-lifecycle/state-transitions",
+    description: "Verify sandbox stop, start, workspace preservation, and deletion behavior.",
+    run: run_state_transitions,
+};
+
+/// Certify that relay-backed commands remain available across startup reconnects.
+pub const SANDBOX_LIFECYCLE_RELAY_READINESS_SCENARIO: Scenario = Scenario {
+    name: "sandbox-lifecycle/relay-readiness",
+    description: "Verify attachments survive supervisor reconnects during sandbox startup.",
+    run: run_relay_readiness,
+};
+
+/// Certify that an in-flight relay is replayed after a forced supervisor reconnect.
+pub const SANDBOX_LIFECYCLE_RELAY_RECONNECT_SCENARIO: Scenario = Scenario {
+    name: "sandbox-lifecycle/relay-reconnect",
+    description: "Verify an in-flight attachment survives a forced supervisor reconnect.",
+    run: run_relay_reconnect,
+};
+
 fn run_sandbox_lifecycle(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> {
+    Box::pin(async move {
+        SANDBOX_LIFECYCLE_STATE_TRANSITIONS_SCENARIO
+            .run(runner)
+            .await?;
+        SANDBOX_LIFECYCLE_RELAY_READINESS_SCENARIO
+            .run(runner)
+            .await?;
+        SANDBOX_LIFECYCLE_RELAY_RECONNECT_SCENARIO.run(runner).await
+    })
+}
+
+fn run_state_transitions(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> {
     Box::pin(async move {
         stop_start_preserves_workspace(runner).await?;
         stopped_can_be_deleted(runner).await
     })
 }
 
+fn run_relay_readiness(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> {
+    Box::pin(async move { relay_readiness_survives_reconnects(runner).await })
+}
+
+fn run_relay_reconnect(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> {
+    Box::pin(async move {
+        let Some(helper) = std::env::var_os("OPENSHELL_RELAY_RECONNECT_HELPER")
+            .filter(|helper| !helper.is_empty())
+        else {
+            return Ok(());
+        };
+        relay_reconnect(runner, &helper).await
+    })
+}
+
+async fn relay_reconnect(
+    runner: &mut OpenShellRunner,
+    helper: &std::ffi::OsStr,
+) -> Result<(), String> {
+    let sandbox_name = format!("ct-{}-rr-fi", runner.id());
+    let marker = format!("relay-reconnected-{}", runner.id());
+    create_running_sandbox(
+        runner,
+        &sandbox_name,
+        "exec sleep infinity",
+        "relay-reconnect/create",
+    )
+    .await?;
+
+    run_reconnect_helper(helper, "pause", &sandbox_name).await?;
+
+    let relay_command = runner
+        .step("relay-reconnect/exec")
+        .description(format!(
+            "in-flight attachment reaches sandbox '{sandbox_name}' after its supervisor reconnects"
+        ))
+        .with_timeout(RELAY_ATTACHMENT_TIMEOUT);
+    let relay_args = [
+        "sandbox",
+        "exec",
+        "--name",
+        &sandbox_name,
+        "--no-tty",
+        "--",
+        "printf",
+        "%s\\n",
+        &marker,
+    ];
+    let relay = relay_command.run(&relay_args);
+    let reconnect = async {
+        // Allow the gateway to queue RelayOpen while the helper blocks its
+        // delivery, then sever only the control connection. The same process
+        // and workload remain alive while the supervisor reconnects.
+        tokio::time::sleep(Duration::from_millis(500)).await;
+        run_reconnect_helper(helper, "disconnect-resume", &sandbox_name).await
+    };
+    let (relay, reconnect) = tokio::join!(relay, reconnect);
+    reconnect?;
+    let relay = relay.map_err(|error| error.to_string())?;
+    relay.require_success()?;
+    if !relay.stdout().lines().any(|line| line == marker) {
+        return Err(relay.failure_diagnostic(&format!("stdout contains marker {marker:?}")));
+    }
+
+    run_lifecycle_command(runner, "delete", &sandbox_name, "relay-reconnect/delete").await?;
+    wait_for_absence(runner, &sandbox_name, "relay-reconnect/deleted").await?;
+    runner.forget_sandbox(&sandbox_name);
+    Ok(())
+}
+
+async fn run_reconnect_helper(
+    helper: &std::ffi::OsStr,
+    action: &str,
+    sandbox_name: &str,
+) -> Result<(), String> {
+    let output = tokio::process::Command::new(helper)
+        .args([action, sandbox_name])
+        .output()
+        .await
+        .map_err(|error| format!("failed to run relay reconnect helper: {error}"))?;
+    if output.status.success() {
+        return Ok(());
+    }
+    Err(format!(
+        "relay reconnect helper {action:?} failed with {}\nstdout:\n{}\nstderr:\n{}",
+        output.status,
+        String::from_utf8_lossy(&output.stdout),
+        String::from_utf8_lossy(&output.stderr),
+    ))
+}
+
+async fn relay_readiness_survives_reconnects(runner: &mut OpenShellRunner) -> Result<(), String> {
+    for first_attempt in (1..=RELAY_SANDBOX_ATTEMPTS).step_by(RELAY_SCENARIO_CONCURRENCY) {
+        let second_attempt = first_attempt + 1;
+        let first_name = relay_sandbox_name(runner, first_attempt);
+        let second_name = relay_sandbox_name(runner, second_attempt);
+        runner.track_sandbox(&first_name);
+        runner.track_sandbox(&second_name);
+
+        let (first, second) = tokio::join!(
+            relay_readiness_attempt(runner, first_attempt, &first_name),
+            relay_readiness_attempt(runner, second_attempt, &second_name),
+        );
+        first?;
+        second?;
+        wait_for_absence(
+            runner,
+            &first_name,
+            &format!("relay-readiness/attempt-{first_attempt}/deleted"),
+        )
+        .await?;
+        wait_for_absence(
+            runner,
+            &second_name,
+            &format!("relay-readiness/attempt-{second_attempt}/deleted"),
+        )
+        .await?;
+        runner.forget_sandbox(&first_name);
+        runner.forget_sandbox(&second_name);
+    }
+    Ok(())
+}
+
+fn relay_sandbox_name(runner: &OpenShellRunner, attempt: usize) -> String {
+    format!("ct-{}-rr-{attempt}", runner.id())
+}
+
+async fn relay_readiness_attempt(
+    runner: &OpenShellRunner,
+    attempt: usize,
+    sandbox_name: &str,
+) -> Result<(), String> {
+    let marker = format!("relay-ready-{}-{attempt}", runner.id());
+    // Match the ordering that exposed the original race: detached creation
+    // returns while the sandbox is still starting, then the attachment opens
+    // against the first supervisor session before its startup reconnect.
+    let main = format!("printf '%s\\n' '{marker}'; sleep 5");
+    let create_command = runner
+        .step(format!("relay-readiness/attempt-{attempt}/create"))
+        .description(format!("sandbox '{sandbox_name}' is created"))
+        .with_timeout(CREATE_TIMEOUT);
+    let create_args = [
+        "sandbox",
+        "create",
+        "--name",
+        sandbox_name,
+        "--detach",
+        "--",
+        "sh",
+        "-c",
+        &main,
+    ];
+    create_command
+        .run(&create_args)
+        .await
+        .map_err(|error| error.to_string())?
+        .require_success()?;
+    relay_connect(
+        runner,
+        sandbox_name,
+        &format!("attempt-{attempt}/initial"),
+        &marker,
+    )
+    .await?;
+
+    run_lifecycle_command(
+        runner,
+        "delete",
+        sandbox_name,
+        &format!("relay-readiness/attempt-{attempt}/delete"),
+    )
+    .await?;
+    Ok(())
+}
+
+async fn relay_connect(
+    runner: &OpenShellRunner,
+    sandbox_name: &str,
+    cycle: &str,
+    marker: &str,
+) -> Result<(), String> {
+    let result = runner
+        .step(format!("relay-readiness/{cycle}/connect"))
+        .description(format!(
+            "attachment reaches sandbox '{sandbox_name}' during its startup reconnect window"
+        ))
+        .with_timeout(RELAY_ATTACHMENT_TIMEOUT)
+        .run(&["sandbox", "connect", sandbox_name])
+        .await
+        .map_err(|error| error.to_string())?;
+    result.require_success()?;
+    if result.stdout().lines().any(|line| line == marker) {
+        Ok(())
+    } else {
+        Err(result.failure_diagnostic(&format!("stdout contains marker {marker:?}")))
+    }
+}
+
 async fn stop_start_preserves_workspace(runner: &mut OpenShellRunner) -> Result<(), String> {
     let sandbox_name = format!("ct-{}-ss", runner.id());
     let sentinel = format!("openshell-stop-start-{}", runner.id());