From eb619eb224b937df6233142ec8ee1045c9bd13a1 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Thu, 22 Oct 2026 09:57:52 -0400 Subject: [PATCH 01/10] test: derive load-sensitive harness budgets from named constants Fixed harness budgets failed under host load (M5 at load 15-40, M3 while another suite runs) although each case passes alone: - runtime-core native runtime tests gave fake-backend graph runs 1-5 s; journal writes on a loaded host exceeded that with graph_timeout. They now take RUN_BUDGET (120 s, under the product's 600 s cap). Passing runs never wait on it. The log-expiry case keeps its own 1 s deadline. - native-compose-adoption-generation cases spawn the Bun stub engine for several hundred probes per retained dispatch; 20 s caller deadlines and 20-30 s case timeouts expired. Case timeouts and non-deadline caller deadlines now derive from testBudgetMs. The 5 s expiry cases and their promptness assertions are product bounds and stay literal. - the process-policy replay child defaulted to 5 s and refused replays ("Initial process-policy replay unavailable"); its child and summary budgets, and the replay cases' timeouts, now derive from testBudgetMs. tests/helpers/test-budget.ts holds the one scale (TEST_BUDGET_SCALE). Refs HACK-1367 Co-Authored-By: Claude Opus 5.5 --- .../graph/native/runtime/exec_tests.rs | 6 +- .../graph/native/runtime/run_tests.rs | 8 +- .../provider/graph/native/runtime/tests.rs | 87 +++++++------ .../native-process-policy-initial-replay.ts | 16 ++- tests/helpers/test-budget.ts | 24 ++++ ...native-compose-adoption-generation.test.ts | 118 ++++++++++-------- ...ative-process-policy-initial-trace.test.ts | 23 +++- 7 files changed, 178 insertions(+), 104 deletions(-) create mode 100644 tests/helpers/test-budget.ts diff --git a/packages/runtime-core/src/provider/graph/native/runtime/exec_tests.rs b/packages/runtime-core/src/provider/graph/native/runtime/exec_tests.rs index fa0c98913..9d5e90115 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime/exec_tests.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime/exec_tests.rs @@ -76,7 +76,7 @@ fn selected() -> ExecSelection { } } fn ready(session: &mut Session<'_, Fake>, graph: &execution::Graph) { - execution::run(graph, session, Duration::from_secs(2)).unwrap(); + execution::run(graph, session, RUN_BUDGET).unwrap(); session .backend .state @@ -105,7 +105,7 @@ fn finite_exec_uses_exact_live_member_and_one_original_source_data_deadline() { data_refusal: false, }; let original = fs::read(session.root.join("state.json")).unwrap(); - let deadline = Instant::now() + Duration::from_secs(2); + let deadline = Instant::now() + RUN_BUDGET; let result = exec::execute_with(&backend, session.receipt.clone(), &selected(), deadline).unwrap(); assert_eq!(result.exit_code, 17); @@ -163,7 +163,7 @@ fn finite_exec_refuses_changed_member_or_data_before_command_and_withholds_resta + if mode == "deadline" { Duration::ZERO } else { - Duration::from_secs(2) + RUN_BUDGET }; assert!( exec::execute_with(&backend, session.receipt.clone(), &request, deadline).is_err(), diff --git a/packages/runtime-core/src/provider/graph/native/runtime/run_tests.rs b/packages/runtime-core/src/provider/graph/native/runtime/run_tests.rs index 8617fe9d2..d633a6a2b 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime/run_tests.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime/run_tests.rs @@ -152,7 +152,7 @@ fn selected() -> run::RunSelection { } } fn ready(session: &mut Session<'_, Fake>, graph: &execution::Graph) { - execution::run(graph, session, Duration::from_secs(2)).unwrap(); + execution::run(graph, session, RUN_BUDGET).unwrap(); let mut state = session.backend.state.borrow_mut(); let member = state.containers.values_mut().next().unwrap(); member["State"]["StartedAt"] = json!("2026-10-09T00:00:00.000000000Z"); @@ -204,7 +204,7 @@ fn finite_one_off_derives_the_exact_member_and_discards_its_record_after_proven_ one_offs: RefCell::new(BTreeMap::new()), }; let original = fs::read(session.root.join("state.json")).unwrap(); - let deadline = Instant::now() + Duration::from_secs(2); + let deadline = Instant::now() + RUN_BUDGET; let result = run::execute_with( &backend, session.receipt.clone(), @@ -290,7 +290,7 @@ fn finite_one_off_refuses_changed_member_mounts_or_data_before_create_and_retain + if mode == "deadline" { Duration::ZERO } else { - Duration::from_secs(2) + RUN_BUDGET }; assert!( run::execute_with( @@ -337,7 +337,7 @@ fn cleanup_reaps_a_recorded_one_off_before_stopping_members_and_refuses_foreign_ session.receipt.clone(), &session.root, &selected(), - Instant::now() + Duration::from_secs(2) + Instant::now() + RUN_BUDGET ) .is_err() ); diff --git a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs index 6d11b969e..334aeea27 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs @@ -14,6 +14,15 @@ use std::{ const OWNER: &str = "cccccccccccccccccccccccccccccccc"; const BOOT: &str = "12345678-abcd-abcd-abcd-123456789abc"; const RUN: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +/// Harness budget for each fake-backend graph run, selection, exec, run or log read here. +/// +/// The fake backend is ready as soon as each effect is applied, so a passing call returns +/// without waiting on this budget; it only bounds a call that would otherwise hang. It is a +/// test budget, not the product bound: `execution::run` still refuses budgets above its own +/// 600 s limit, and cases that exercise expiry pass their own short or zero deadline. Fixed +/// 1-5 s budgets failed with `graph_timeout` under host load (journal writes and syncs on a +/// loaded host), so every such call takes its budget from this one constant. +const RUN_BUDGET: Duration = Duration::from_secs(120); struct Fixture { root: PathBuf, project: PathBuf, @@ -59,7 +68,7 @@ impl Fixture { profiles: &[], explicit_overlay: None, metadata, - deadline: Instant::now() + Duration::from_secs(60), + deadline: Instant::now() + RUN_BUDGET, }, ) .unwrap() @@ -140,7 +149,7 @@ fn two_owned_bridges_keep_per_network_members_and_aliases_through_cleanup() { serde_json::to_value(&session.receipt).unwrap()["version"], 5 ); - execution::run(&graph, &mut session, Duration::from_secs(2)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); assert_eq!(session.receipt.phase, Phase::ReadyObserved); assert_eq!(session.backend.state.borrow().networks.len(), 2); snapshot(&session.backend, session.receipt.clone()).unwrap(); @@ -222,7 +231,7 @@ fn two_bridge_cancellation_before_second_create_retains_exact_first_bridge_for_c session.startup_guard = Some(&guard); session.backend.state.borrow_mut().cancel_after_network = Some(canceled.clone()); assert_eq!( - execution::run(&graph, &mut session, Duration::from_secs(2)) + execution::run(&graph, &mut session, RUN_BUDGET) .unwrap_err() .code, "native_graph_canceled" @@ -240,7 +249,7 @@ fn two_bridge_cleanup_retries_only_the_retained_second_bridge_after_first_delete let fixture = Fixture::new(two_bridges()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"db":{},"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(2)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); session .backend .state @@ -648,7 +657,7 @@ fn foreground_cancellation_after_network_creation_prevents_container_effects_and session.startup_guard = Some(&guard); session.backend.state.borrow_mut().cancel_after_network = Some(canceled.clone()); assert_eq!( - execution::run(&graph, &mut session, Duration::from_secs(1)) + execution::run(&graph, &mut session, RUN_BUDGET) .unwrap_err() .code, "native_graph_canceled" @@ -659,7 +668,7 @@ fn foreground_cancellation_after_network_creation_prevents_container_effects_and assert_eq!(retained.resources["network:default"].phase, "created"); assert!(retained.resources["network:default"].id.is_some()); canceled.set(false); - assert!(execution::run(&graph, &mut session, Duration::from_secs(1)).is_err()); + assert!(execution::run(&graph, &mut session, RUN_BUDGET).is_err()); assert_eq!(session.backend.state.borrow().effects, ["create:network"]); cleanup_using(&session.backend, &mut session.receipt, &session.root).unwrap(); assert_eq!(session.receipt.phase, Phase::Removed); @@ -1083,7 +1092,7 @@ fn live_source_controller_preserves_edits_and_cleans_only_original_owned_resourc let fixture = source_fixture(); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); assert_eq!( serde_json::to_value(&session.receipt).unwrap()["version"], 3 @@ -1135,7 +1144,7 @@ fn selected_source_replacement_before_effect_or_after_create_refuses_without_sta fs::rename(&path, fixture.project.join("original")).unwrap(); fs::DirBuilder::new().mode(0o700).create(&path).unwrap(); } - assert!(execution::run(&graph, &mut session, Duration::from_secs(5)).is_err()); + assert!(execution::run(&graph, &mut session, RUN_BUDGET).is_err()); assert!( !session .backend @@ -1155,7 +1164,7 @@ fn selected_source_replacement_before_effect_or_after_create_refuses_without_sta } else { assert!(session.backend.state.borrow().effects.is_empty()); } - assert!(execution::run(&graph, &mut session, Duration::from_secs(5)).is_err()); + assert!(execution::run(&graph, &mut session, RUN_BUDGET).is_err()); } } @@ -1165,7 +1174,7 @@ fn retained_source_bind_or_approved_share_drift_refuses_cleanup_before_mutation( let fixture = source_fixture(); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); if changed_share { session.backend.source.as_mut().unwrap().share.inode += 1; } else { @@ -1184,7 +1193,7 @@ fn source_cleanup_does_not_recapture_a_moved_or_deleted_host_project() { let fixture = source_fixture(); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let moved = fixture.root.join("moved-project"); if deleted { fs::remove_dir_all(&fixture.project).unwrap(); @@ -1215,7 +1224,7 @@ fn real_compiler_job_dependency_and_readiness_drive_durable_native_effects_and_c let fixture = Fixture::new(project); let prepared = fixture.prepared(json!({"web":{},"z.seed":{}}), &BTreeMap::new()); let (graph, mut session) = fixture.session(prepared); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); assert_eq!(session.receipt.phase, Phase::ReadyObserved); assert_eq!( session.backend.state.borrow().effects, @@ -1259,12 +1268,12 @@ fn create_and_start_uncertainty_are_retained_once_without_replay_or_adoption() { fake.fail_create = !fail_start; fake.fail_start = fail_start; } - assert!(execution::run(&graph, &mut session, Duration::from_secs(5)).is_err()); + assert!(execution::run(&graph, &mut session, RUN_BUDGET).is_err()); let (retained, _) = journal::load(&fixture.candidate, RUN, OWNER, BOOT).unwrap(); assert_eq!(retained.resources["container:web"].phase, "uncertain"); assert_eq!(retained.resources["container:web"].id.is_some(), fail_start); let count = session.backend.state.borrow().effects.len(); - assert!(execution::run(&graph, &mut session, Duration::from_secs(5)).is_err()); + assert!(execution::run(&graph, &mut session, RUN_BUDGET).is_err()); assert_eq!(session.backend.state.borrow().effects.len(), count); } } @@ -1277,12 +1286,12 @@ fn project_network_create_uncertainty_is_durable_and_never_replayed() { fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); session.backend.state.borrow_mut().fail_network_create = !applied; session.backend.state.borrow_mut().fail_network_after_apply = applied; - assert!(execution::run(&graph, &mut session, Duration::from_secs(5)).is_err()); + assert!(execution::run(&graph, &mut session, RUN_BUDGET).is_err()); let (retained, _) = journal::load(&fixture.candidate, RUN, OWNER, BOOT).unwrap(); assert_eq!(retained.resources["network:default"].phase, "create-intent"); assert!(retained.resources["network:default"].id.is_none()); assert_eq!(retained.resources["container:web"].phase, "reserved"); - assert!(execution::run(&graph, &mut session, Duration::from_secs(5)).is_err()); + assert!(execution::run(&graph, &mut session, RUN_BUDGET).is_err()); assert_eq!( session.backend.state.borrow().effects, vec!["create:network"] @@ -1303,7 +1312,7 @@ fn foreign_or_changed_project_networks_refuse_cleanup_before_any_stop_or_delete( let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let name = session.receipt.resources["network:default"].name.clone(); { let mut state = session.backend.state.borrow_mut(); @@ -1336,7 +1345,7 @@ fn changed_network_ids_aliases_extra_attachments_and_endpoint_addresses_refuse_o let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let name = session.receipt.resources["container:web"].name.clone(); let network = session.receipt.resources["network:default"].name.clone(); { @@ -1389,13 +1398,13 @@ fn authored_change_and_foreign_container_identity_refuse_before_effects() { let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); fs::write(fixture.project.join(".hack/hack.project.json"), "changed").unwrap(); - assert!(execution::run(&graph, &mut session, Duration::from_secs(5)).is_err()); + assert!(execution::run(&graph, &mut session, RUN_BUDGET).is_err()); assert!(session.backend.state.borrow().effects.is_empty()); let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let name = session.receipt.resources["container:web"].name.clone(); session .backend @@ -1441,7 +1450,7 @@ fn native_reservations_count_against_the_common_budget_and_uncertainty_blocks_ad let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); for requested in [31, 32] { let mut budget = admission::Budget::default(); for _ in 0..requested { @@ -1495,7 +1504,7 @@ fn private_native_input_crosses_typed_stage_intent_without_entering_public_journ &values, ); let (graph, mut session) = fixture.session(prepared); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); assert_eq!(session.backend.state.borrow().staged, vec!["web"]); assert!( !fs::read_to_string(session.root.join("state.json")) @@ -1542,7 +1551,7 @@ fn cleanup_retry_keeps_observed_stop_request_for_the_same_immutable_id() { let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); session.backend.state.borrow_mut().fail_delete = true; assert!(cleanup_using(&session.backend, &mut session.receipt, &session.root).is_err()); let (mut retained, _) = journal::load(&fixture.candidate, RUN, OWNER, BOOT).unwrap(); @@ -1568,7 +1577,7 @@ fn committed_removal_retry_does_not_repeat_stop_or_regress_the_receipt_phase() { let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); session.backend.state.borrow_mut().fail_delete = true; assert!(cleanup_using(&session.backend, &mut session.receipt, &session.root).is_err()); let (mut retained, _) = journal::load(&fixture.candidate, RUN, OWNER, BOOT).unwrap(); @@ -1597,7 +1606,7 @@ fn stopped_cleanup_resumes_only_terminal_instances_without_another_stop_batch() let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); session.backend.state.borrow_mut().fail_delete = true; assert!(cleanup_using(&session.backend, &mut session.receipt, &session.root).is_err()); // Select the durable Stopped boundary with actual matching terminal observations. @@ -1623,7 +1632,7 @@ fn cleanup_of_removed_inventory_is_read_only_and_keeps_the_committed_bytes() { let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); cleanup_using(&session.backend, &mut session.receipt, &session.root).unwrap(); let before = fs::read(session.root.join("state.json")).unwrap(); let effects = session.backend.state.borrow().effects.clone(); @@ -1648,7 +1657,7 @@ fn stopped_or_removing_inventory_refuses_a_restarted_instance_before_effects() { let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); session.backend.state.borrow_mut().fail_delete = true; assert!(cleanup_using(&session.backend, &mut session.receipt, &session.root).is_err()); session.receipt.phase = phase; @@ -1677,7 +1686,7 @@ fn removed_inventory_refuses_even_matching_resource_reappearance_without_effects let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let containers = session.backend.state.borrow().containers.clone(); let networks = session.backend.state.borrow().networks.clone(); cleanup_using(&session.backend, &mut session.receipt, &session.root).unwrap(); @@ -1712,7 +1721,7 @@ fn cleanup_owner_guard_refuses_before_provider_connection_or_receipt_mutation() assert!(!fixture.candidate.state_root.join("run/smolvm").exists()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let before = session.backend.state.borrow().effects.clone(); assert!( cleanup_using_guarded( @@ -1750,7 +1759,7 @@ fn cleanup_guard_loss_during_first_observation_prevents_next_request_and_stop_in let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let changed = Cell::new(false); let backend = Changed { backend: &session.backend, @@ -1830,7 +1839,7 @@ fn native_stop_failure_keeps_reservation_and_maps_only_owned_ids_to_workload_nam let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); session.backend.state.borrow_mut().fail_stop = true; let error = cleanup_using(&session.backend, &mut session.receipt, &session.root).unwrap_err(); let detail = error.stop_failures.as_ref().unwrap(); @@ -1865,7 +1874,7 @@ fn cleanup_owner_guard_fences_delete_after_stop_and_network_delete_after_contain let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let changed = Rc::new(Cell::new(false)); { let mut state = session.backend.state.borrow_mut(); @@ -1948,7 +1957,7 @@ fn owner_change_during_final_terminal_inspection_refuses_container_delete() { let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let changed = Rc::new(Cell::new(false)); session .backend @@ -2004,7 +2013,7 @@ fn old_boot_removed_history_releases_capacity_only_after_strict_owner_and_remova let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); cleanup_using(&session.backend, &mut session.receipt, &session.root).unwrap(); let new_boot = "87654321-abcd-abcd-abcd-123456789abc"; let check = || { @@ -2845,7 +2854,7 @@ fn finite_native_logs_bind_exact_member_tail_and_restart_generation_without_effe let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(2)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let name = session.receipt.resources["container:web"].name.clone(); let id = session.receipt.resources["container:web"] .id @@ -2859,7 +2868,7 @@ fn finite_native_logs_bind_exact_member_tail_and_restart_generation_without_effe .get_mut(&name) .unwrap()["State"]["StartedAt"] = json!("2026-10-09T00:00:00.000000000Z"); let effects = session.backend.state.borrow().effects.clone(); - let deadline = Instant::now() + Duration::from_secs(2); + let deadline = Instant::now() + RUN_BUDGET; let logs = logs_with( &session.backend, session.receipt.clone(), @@ -2932,7 +2941,7 @@ fn finite_native_logs_share_one_deadline_with_all_source_and_data_checks() { let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(2)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let name = session.receipt.resources["container:web"].name.clone(); session .backend @@ -2947,7 +2956,7 @@ fn finite_native_logs_share_one_deadline_with_all_source_and_data_checks() { .borrow_mut() .log_data_deadlines .clear(); - let deadline = Instant::now() + Duration::from_secs(2); + let deadline = Instant::now() + RUN_BUDGET; logs_with( &session.backend, session.receipt.clone(), @@ -2968,6 +2977,8 @@ fn finite_native_logs_share_one_deadline_with_all_source_and_data_checks() { state.log_data_deadlines.clear(); state.expire_source_check = 4; } + // The expiry under test: the fake sleeps past this deadline at the fourth source check, + // so it stays short instead of using `RUN_BUDGET`. let deadline = Instant::now() + Duration::from_secs(1); assert!( logs_with( diff --git a/tests/e2e/native-process-policy-initial-replay.ts b/tests/e2e/native-process-policy-initial-replay.ts index 18daa785c..afe454047 100644 --- a/tests/e2e/native-process-policy-initial-replay.ts +++ b/tests/e2e/native-process-policy-initial-replay.ts @@ -10,6 +10,7 @@ import type { } from "../../src/lib/native-compose-ownership.ts"; import { isNativeComposeTopologyPredicate } from "../../src/lib/native-compose-ownership.ts"; import { exec } from "../../src/lib/shell.ts"; +import { testBudgetMs } from "../helpers/test-budget.ts"; import { type ProcessPolicyInitialTraceQuery, readProcessPolicyInitialTrace, @@ -20,6 +21,15 @@ import { } from "./native-process-policy-startup-diagnostic.ts"; const REFUSAL = "Initial process-policy replay unavailable; values omitted"; +/** + * Harness budget for one replay child: a fresh Bun process that transpiles the + * ownership policy and spawns one stub engine per recorded reply. This bounds + * the replay itself; the ownership policy's own probe bounds run unchanged + * inside it. A fixed 5 s refused replays on loaded hosts. + */ +const REPLAY_CHILD_BUDGET_MS = testBudgetMs(5000); +/** Harness budget for every replay child of one trace summary together. */ +const REPLAY_SUMMARY_BUDGET_MS = testBudgetMs(30_000); const SERVICES = ["forced", "graceful", "reaper", "retry"] as const; const CODES = [ "E_NATIVE_COMPOSE_OWNERSHIP", @@ -173,7 +183,7 @@ process.stdout.write(JSON.stringify({outcome,code,reason,topologyPredicate,consu cwd: opts.directory, env: { PATH: opts.directory }, stdin: "ignore", - timeoutMs: opts.timeoutMs ?? 5000, + timeoutMs: opts.timeoutMs ?? REPLAY_CHILD_BUDGET_MS, } ); if (result.exitCode !== 0 || result.stdout.length > 1024) { @@ -222,13 +232,13 @@ export async function summarizeProcessPolicyInitialTrace(opts: { readonly projectRoot: string; readonly replayRoot: string; }) { - const deadline = Date.now() + 30_000; + const deadline = Date.now() + REPLAY_SUMMARY_BUDGET_MS; const remaining = (): number => { const budget = Math.floor(deadline - Date.now()); if (budget <= 0) { throw new Error(REFUSAL); } - return Math.min(budget, 5000); + return Math.min(budget, REPLAY_CHILD_BUDGET_MS); }; const trace = await readProcessPolicyInitialTrace(opts.directory); const store = await openNativeComposeGenerationStore({ diff --git a/tests/helpers/test-budget.ts b/tests/helpers/test-budget.ts new file mode 100644 index 000000000..0df399a75 --- /dev/null +++ b/tests/helpers/test-budget.ts @@ -0,0 +1,24 @@ +/** + * Harness budgets for load-sensitive tests. + * + * A harness budget bounds how long a test waits for its own fixture work: + * spawning stub engines and compilers, transpiling a child program, or a + * stub-backed owner operation. It is not a product bound. A passing case + * finishes as soon as that work does, so a larger budget only lengthens how + * long a hung case takes to fail. Product bounds under test (an expiring + * caller deadline, a cancellation latency) keep their own literal values. + * + * Each call site names the time its fixture work needs on an unloaded host; + * `TEST_BUDGET_SCALE` is the headroom for a loaded one. On the M5 at load + * 15-40, and on the M3 while another suite ran, these fixtures took three to + * four times their unloaded time and failed at the former literal budgets. + */ +export const TEST_BUDGET_SCALE = 4; + +/** Scale the unloaded time a fixture needs to the budget a loaded host needs. */ +export function testBudgetMs(unloadedMs: number): number { + if (!(Number.isSafeInteger(unloadedMs) && unloadedMs > 0)) { + throw new Error("Test budget refused a non-positive or fractional time."); + } + return unloadedMs * TEST_BUDGET_SCALE; +} diff --git a/tests/native-compose-adoption-generation.test.ts b/tests/native-compose-adoption-generation.test.ts index a674fc7d5..e9e943cf8 100644 --- a/tests/native-compose-adoption-generation.test.ts +++ b/tests/native-compose-adoption-generation.test.ts @@ -69,10 +69,20 @@ import { createCompletedJobFixtureSettlement } from "./e2e/scenarios/native-comp import { restoreEnv } from "./helpers/env.ts"; import { managedEnvCompilerFixture } from "./helpers/managed-env-compiler.ts"; import { privateCliEnv } from "./helpers/private-cli-env.ts"; +import { testBudgetMs } from "./helpers/test-budget.ts"; -// Each owner workflow performs multiple bounded child probes; allow their cumulative work on shared CI hosts. +/* + * Harness budgets. Every engine probe below spawns the Bun stub engine, and one + * retained dispatch issues several hundred probes, so an owner operation's wall + * time follows host load. Case timeouts and the caller deadlines of cases that + * are not about the deadline derive from `testBudgetMs`. The short deadlines in + * the expiry cases (5 s) and their promptness assertions are product bounds + * under test and stay literal. + */ +/** Caller deadline for one owner operation in a case that is not about the deadline. */ +const OPERATION_DEADLINE_MS = testBudgetMs(20_000); const test = (name: string, run: () => Promise) => - boundedTest(name, run, 20_000); + boundedTest(name, run, testBudgetMs(20_000)); const CANARY = "synthetic-private-adoption-canary"; const ID = "a".repeat(64); const NETWORK = "b".repeat(64); @@ -720,7 +730,7 @@ boundedTest.each(["no", ""] as const)( }); }, // Cover prepare/publish and separate bounded up/down operations without changing their deadlines. - 60_000 + testBudgetMs(60_000) ); boundedTest.each([ @@ -781,7 +791,7 @@ boundedTest.each([ } }); }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -835,7 +845,7 @@ boundedTest( } }); }, - 30_000 + testBudgetMs(30_000) ); boundedTest.each(["start", "restart", "stop"] as const)( @@ -854,7 +864,7 @@ boundedTest.each(["start", "restart", "stop"] as const)( fixture.states = { [ID]: true, [app]: true, [job]: false }; await save(); } - const deadline = Date.now() + 20_000; + const deadline = Date.now() + OPERATION_DEADLINE_MS; expect( await store.withMutation({ generation, @@ -913,7 +923,7 @@ boundedTest.each(["start", "restart", "stop"] as const)( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -930,7 +940,7 @@ boundedTest( binary, operation: "start", services: [], - deadline: Date.now() + 20_000, + deadline: Date.now() + OPERATION_DEADLINE_MS, run: async () => { fixture.states = { [ID]: true, [app]: true, [job]: false }; await save(); @@ -940,7 +950,7 @@ boundedTest( ).rejects.toThrow("values omitted"); expect((await readReceipt()).pendingOperation.operation).toBe("start"); expect(await mutationCommands()).toEqual([]); - const deadline = Date.now() + 20_000; + const deadline = Date.now() + OPERATION_DEADLINE_MS; expect( await store.withMutation({ generation, @@ -968,7 +978,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -984,7 +994,7 @@ boundedTest( try { const binary = await compiler(); await store.publish({ generation, binary }); - const deadline = Date.now() + 20_000; + const deadline = Date.now() + OPERATION_DEADLINE_MS; await expect( store.withMutation({ generation, @@ -1011,7 +1021,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -1023,7 +1033,7 @@ boundedTest( const binary = await compiler(); await store.publish({ generation, binary }); let previous: LegacyComposeRetainedOutcome | undefined; - const deadline = Date.now() + 20_000; + const deadline = Date.now() + OPERATION_DEADLINE_MS; expect( await store.withMutation({ generation, @@ -1050,7 +1060,7 @@ boundedTest( binary, operation: "start", services: [], - deadline: Date.now() + 20_000, + deadline: Date.now() + OPERATION_DEADLINE_MS, run: async () => { if (!previous) { throw new Error("missing completion"); @@ -1065,7 +1075,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -1076,7 +1086,7 @@ boundedTest( try { const binary = await compiler(); await store.publish({ generation, binary }); - const deadline = Date.now() + 20_000; + const deadline = Date.now() + OPERATION_DEADLINE_MS; await expect( store.withMutation({ generation, @@ -1106,7 +1116,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -1121,7 +1131,7 @@ boundedTest( join(projectRoot, ".hack/hack.project.json"), "utf8" ); - const deadline = Date.now() + 20_000; + const deadline = Date.now() + OPERATION_DEADLINE_MS; await refusal( store.withMutation({ generation, @@ -1155,7 +1165,7 @@ boundedTest( fixture.jobFinalRace = false; await save(); await writeFile(join(projectRoot, ".hack/hack.project.json"), original); - const recoveryDeadline = Date.now() + 20_000; + const recoveryDeadline = Date.now() + OPERATION_DEADLINE_MS; expect( await store.withMutation({ generation, @@ -1177,7 +1187,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -1188,7 +1198,7 @@ boundedTest( try { const binary = await compiler(); await store.publish({ generation, binary }); - const deadline = Date.now() + 20_000; + const deadline = Date.now() + OPERATION_DEADLINE_MS; await expect( store.withMutation({ generation, @@ -1224,7 +1234,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -1251,7 +1261,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -1264,7 +1274,7 @@ boundedTest( const controller = new AbortController(); try { const binary = await compiler(), - deadline = Date.now() + 20_000; + deadline = Date.now() + OPERATION_DEADLINE_MS; expect( await store.withPreparationStop({ generation, @@ -1294,7 +1304,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -1326,7 +1336,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -1367,7 +1377,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); for (const linked of [false, true]) { @@ -1451,7 +1461,7 @@ for (const linked of [false, true]) { stage("closed"); } }, - 30_000 + testBudgetMs(30_000) ); } @@ -1506,7 +1516,7 @@ for (const stage of ["per-effect freshness", "final observation"] as const) { await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); } @@ -1531,7 +1541,7 @@ boundedTest( expect(await mutationCommands()).toEqual([]); expect(fixture.volume[0]?.createdAt).toBe(CREATED); }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -1564,7 +1574,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -1625,7 +1635,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -1639,7 +1649,7 @@ boundedTest( let calls = 0; await refusal( store.withMutation({ - deadline: Date.now() + 20_000, + deadline: Date.now() + OPERATION_DEADLINE_MS, generation, binary, operation: "start", @@ -1655,7 +1665,7 @@ boundedTest( expect((await readReceipt()).pendingOperation).toBeNull(); let retained: (() => Promise) | undefined; await store.withMutation({ - deadline: Date.now() + 20_000, + deadline: Date.now() + OPERATION_DEADLINE_MS, generation, binary, operation: "stop", @@ -1678,7 +1688,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -1728,7 +1738,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -1741,7 +1751,7 @@ boundedTest( await store.publish({ generation, binary }); await refusal( store.withMutation({ - deadline: Date.now() + 20_000, + deadline: Date.now() + OPERATION_DEADLINE_MS, generation, binary, operation: "start", @@ -1764,7 +1774,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -1783,7 +1793,7 @@ boundedTest( let effects = 0; await refusal( store.withMutation({ - deadline: Date.now() + 20_000, + deadline: Date.now() + OPERATION_DEADLINE_MS, generation, binary, operation: "start", @@ -1804,7 +1814,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -1822,7 +1832,7 @@ boundedTest( await store.publish({ generation, binary }); fixture.hangMutation = true; await save(); - const deadline = Date.now() + 20_000; + const deadline = Date.now() + OPERATION_DEADLINE_MS; let effectStartedAt = 0; await refusal( store.withMutation({ @@ -1838,7 +1848,7 @@ boundedTest( deadline, signal: controller.signal, }); - const readyUntil = Date.now() + 4000; + const readyUntil = Date.now() + testBudgetMs(4000); while ( !(await Bun.file(join(root, "effect-started")).exists()) && Date.now() < readyUntil @@ -1871,7 +1881,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest.skipIf(!process.env.HACK_TEST_OLD_ADOPTION_BINARY)( @@ -1899,7 +1909,7 @@ boundedTest.skipIf(!process.env.HACK_TEST_OLD_ADOPTION_BINARY)( stdin: "ignore", stdout: "pipe", stderr: "pipe", - timeout: 5000, + timeout: testBudgetMs(5000), } ); const [stdout, stderr, code] = await Promise.all([ @@ -1915,7 +1925,7 @@ boundedTest.skipIf(!process.env.HACK_TEST_OLD_ADOPTION_BINARY)( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); /** Current writers keep references at rest; "literal" models a file an earlier @@ -2271,7 +2281,7 @@ boundedTest( "active" ); }, - 90_000 + testBudgetMs(90_000) ); boundedTest( @@ -2356,7 +2366,7 @@ boundedTest( }) ).toMatchObject({ removed: true, adoptionBound: false }); }, - 90_000 + testBudgetMs(90_000) ); test("v3 value-free env override binds the engine config hash computed with the values (HACK-1273)", async () => { @@ -2564,7 +2574,7 @@ test("v4 typed locals preserve generated bytes, exact original IDs and key-free stdin: "ignore", stdout: "pipe", stderr: "pipe", - timeout: 5000, + timeout: testBudgetMs(5000), } ); const [stdout, stderr, code] = await Promise.all([ @@ -2744,7 +2754,7 @@ boundedTest( await store.close(); } }, - 240_000 + testBudgetMs(240_000) ); boundedTest( @@ -2807,7 +2817,7 @@ boundedTest( await store.close(); } }, - 60_000 + testBudgetMs(60_000) ); async function artifactPath(name: string) { const receipt = JSON.parse( @@ -3412,7 +3422,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); test("v10 saved bridge and health owner rejects other contract receipts", async () => { @@ -3500,7 +3510,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); boundedTest( @@ -3535,7 +3545,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); test("rolled-back v10 can reprepare the same owned bridge without health under v6", async () => { @@ -3705,7 +3715,7 @@ boundedTest( await store.close(); } }, - 30_000 + testBudgetMs(30_000) ); test("v6 rollback permits a later plain owner without inheriting topology authority", async () => { diff --git a/tests/native-process-policy-initial-trace.test.ts b/tests/native-process-policy-initial-trace.test.ts index 9439adfc3..5ea5b22b9 100644 --- a/tests/native-process-policy-initial-trace.test.ts +++ b/tests/native-process-policy-initial-trace.test.ts @@ -1,4 +1,4 @@ -import { expect, test } from "bun:test"; +import { test as bunTest, expect } from "bun:test"; import { chmod, mkdtemp, @@ -33,6 +33,25 @@ import { classifyProcessPolicyMemberKeys, } from "./e2e/native-process-policy-startup-diagnostic.ts"; import { restoreEnv } from "./helpers/env.ts"; +import { testBudgetMs } from "./helpers/test-budget.ts"; + +/** + * A case whose last argument is the time its replay children need on an + * unloaded host. The Bun timeout is that time scaled by `testBudgetMs`, since + * each replay child cold-starts Bun and one stub engine per recorded reply. + * Cases without one keep Bun's default. + */ +function test( + name: string, + run: () => void | Promise, + unloadedTimeoutMs?: number +): void { + if (unloadedTimeoutMs === undefined) { + bunTest(name, run); + return; + } + bunTest(name, run, testBudgetMs(unloadedTimeoutMs)); +} const CANARY = "synthetic-unknown-docker-stdout-stderr-canary"; const EMPTY_MEMBER_KEYS = { hex64: 0, epPrefixed: 0, other: 0 } as const; @@ -114,7 +133,7 @@ const FORWARDED_DOCKER_EXEC_TIMEOUT_MS = 30_000; /** Four forwarded commands at their exec bound, plus trace reads and rewrites. */ const INITIAL_FORWARDING_TEST_TIMEOUT_MS = 150_000; -test( +bunTest( "initial forwarding preserves bytes/exits but records no unknown output, args or stderr", async () => { const root = await mkdtemp(join(tmpdir(), "process-initial-forwarding-")); From 7f76651d10bbfa6e96ef0ba21322054b967ccdfc Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Thu, 22 Oct 2026 17:06:43 -0400 Subject: [PATCH 02/10] test(native): publish the hanging status PID atomically and scale authored budgets Under host load, "cancel during the runtime status check refuses before the startup intent" failed settlement after its assertions passed. The stand-in status request appended its PID with appendFile, and the case cancelled as soon as the request was logged. The product's SIGKILL could land after the file was created but before the PID was written. Settlement reads an empty PID file as a write still in progress, so it waited out its budget. - The stand-in now stages its PID and renames it into place, so the record is either absent or complete. - The case waits for the published PID (awaitPidFile) before it cancels, instead of the call log. - The file's settlement, startup and stop harness budgets now derive from testBudgetMs. The 50 ms deadline cases and their wall bound stay literal. Refs HACK-1367 Co-Authored-By: Claude Opus 5.5 --- tests/native-authored-project-start.test.ts | 44 ++++++++++----------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/native-authored-project-start.test.ts b/tests/native-authored-project-start.test.ts index 28c68b2a2..9271eb2f5 100644 --- a/tests/native-authored-project-start.test.ts +++ b/tests/native-authored-project-start.test.ts @@ -41,23 +41,29 @@ import { evaluateComposeServiceEnvFreshness, } from "../src/lib/compose-env-fingerprint.ts"; import { writeLifecycleMuxFixture } from "./helpers/lifecycle-mux-fixture.ts"; +import { awaitPidFile } from "./helpers/native-authored-settlement.ts"; import { createNativeFixtureCases, type NativeFixtureCase, } from "./helpers/native-fixture-cases.ts"; +import { testBudgetMs } from "./helpers/test-budget.ts"; const run = "a".repeat(32); const CANARY = "synthetic-private-native-start-canary"; -/** Allowance for a case's owners, groups and stop endpoints to be absent afterwards. */ -const SETTLEMENT_BUDGET_MS = 15_000; +/** + * Allowance for a case's owners, groups and stop endpoints to be absent + * afterwards. Harness budget: a passing case settles as soon as they exit. + */ +const SETTLEMENT_BUDGET_MS = testBudgetMs(5000); /** * Caller startup deadline for cases that are not about the deadline itself. * Each start spawns several Bun children (compiler, planner, owner, hooks); on * a loaded host that takes well over the 10-20 s these cases used to allow. + * Cases that test the deadline pass their own literal product deadline. */ -const STARTUP_TIMEOUT_MS = 60_000; +const STARTUP_TIMEOUT_MS = testBudgetMs(15_000); /** Caller stop deadline for ordinary `down` in the hook and live-stop cases. */ -const STOP_TIMEOUT_MS = 60_000; +const STOP_TIMEOUT_MS = testBudgetMs(15_000); /** * Body budget for every case: a full start plus a stop, with slack. The guard * adds the settlement budget on top, so cases never fall back to Bun's 5 s. @@ -464,6 +470,9 @@ console.log(JSON.stringify(result)); group: true, }); // A hanging status request must be killed by cancellation, not outlive the case. + // The stand-in publishes its PID by rename, so the file is either absent or + // complete: a cancel that lands mid-write must not leave an empty record that + // settlement reads as a write still in progress. const statusPid = join(root, "status.pid"); owner.settlement.record({ kind: "pid-file", @@ -475,11 +484,11 @@ console.log(JSON.stringify(result)); await writeFile( binary, `#!${process.execPath} -import {appendFile,readFile,readdir,unlink,writeFile} from 'node:fs/promises'; +import {appendFile,readFile,readdir,rename,unlink,writeFile} from 'node:fs/promises'; import {createHash} from 'node:crypto';import {join} from 'node:path'; const args=process.argv.slice(2);const action=args[4];await appendFile(${JSON.stringify(calls)},JSON.stringify(args)+'\\n'); if(${options.runtimeStopped === true}&&args[2]==='runtime'&&args[3]==='status'){console.log(JSON.stringify({phase:'stopped',process_alive:false}));process.exit(0)} -if(${options.runtimeStatusHangs === true}&&args[2]==='runtime'&&args[3]==='status'){await appendFile(${JSON.stringify(statusPid)},process.pid+'\\n');await new Promise((resolve)=>setTimeout(resolve,${2 * CASE_TIMEOUT_MS}));process.exit(0)} +if(${options.runtimeStatusHangs === true}&&args[2]==='runtime'&&args[3]==='status'){const staged=${JSON.stringify(statusPid)}+'.'+process.pid;await writeFile(staged,process.pid+'\\n');await rename(staged,${JSON.stringify(statusPid)});await new Promise((resolve)=>setTimeout(resolve,${2 * CASE_TIMEOUT_MS}));process.exit(0)} if(${options.runtimeStatusWrappedStop === true}&&args[2]==='runtime'&&args[3]==='status'){console.error(JSON.stringify({code:'native_graph_foreground',cause_code:'runtime_not_running',message:${JSON.stringify(CANARY)}}));process.exit(2)} const journal=${JSON.stringify(journal)}; const root=${JSON.stringify(join(projectDir, ".internal", "native-authored-runs"))}; @@ -578,6 +587,7 @@ if(['plan','serve','frontend-plan','frontend-serve'].includes(action)){ calls, delivery, sources, + statusPid, }; } @@ -887,22 +897,12 @@ macTest( signal: signal.signal, startupTimeoutMs: STARTUP_TIMEOUT_MS, }); - // Cancel once the status request is in flight. The check treats the - // aborted request as inconclusive; the start must still not reserve. - // The stand-in request hangs past the case budget, so this waits on the - // request itself rather than racing a shorter fixed window. - const deadline = performance.now() + CASE_TIMEOUT_MS; - while ( - !( - (await Bun.file(selected.calls).exists()) && - (await Bun.file(selected.calls).text()).includes('"status"') - ) - ) { - if (performance.now() > deadline) { - throw new Error("runtime status was never requested"); - } - await Bun.sleep(20); - } + // Cancel once the status request is in flight and has published its PID. + // The check treats the aborted request as inconclusive; the start must + // still not reserve. The stand-in request hangs past the case budget, so + // this waits on the request's own identity rather than a fixed window, and + // the cancel cannot land while the stand-in is still recording it. + await awaitPidFile(selected.statusPid, { budgetMs: CASE_TIMEOUT_MS }); signal.abort(); const error = await failure(started, "not-started"); expect(error).toHaveProperty("canceled", true); From 1686c1d0704197062305f3355329dd8d4ed10e6e Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Sat, 24 Oct 2026 10:19:19 -0400 Subject: [PATCH 03/10] test(native): widen load budgets for authored starts, adoption dispatch and replay cases Raise TEST_BUDGET_SCALE from 4 to 10. With a second suite plus churn workers on the M5 (1-min load 28-66), Source5 authored starts expired the 60 s startup deadline and v7 retained dispatches expired their 80 s caller deadline; a v7 dispatch case took 146 s at load 22-50. - Adoption: public dispatch takes its caller deadline from the Compose startup budget (90 s default). Inject the harness operation deadline through HACK_COMPOSE_STARTUP_TIMEOUT_MS in beforeEach and restore it. - Authored: the planner-refusal case uses STARTUP_TIMEOUT_MS instead of a literal 10 s; the two-cycle down.after case gets two case budgets, and the stand-in owner cap outlasts it. - Replay trace: cases that name no time now get Bun's 5 s default scaled by testBudgetMs, and the multicall/UTF-8 cases use the forwarded-exec bound instead of literal 3 s exec and probe deadlines. Refs HACK-1367 Co-Authored-By: Claude Opus 5.5 --- tests/helpers/test-budget.ts | 13 +++++--- tests/native-authored-project-start.test.ts | 10 +++--- ...native-compose-adoption-generation.test.ts | 9 ++++++ ...ative-process-policy-initial-trace.test.ts | 32 ++++++++++++------- 4 files changed, 44 insertions(+), 20 deletions(-) diff --git a/tests/helpers/test-budget.ts b/tests/helpers/test-budget.ts index 0df399a75..5b3cd8b29 100644 --- a/tests/helpers/test-budget.ts +++ b/tests/helpers/test-budget.ts @@ -9,11 +9,16 @@ * caller deadline, a cancellation latency) keep their own literal values. * * Each call site names the time its fixture work needs on an unloaded host; - * `TEST_BUDGET_SCALE` is the headroom for a loaded one. On the M5 at load - * 15-40, and on the M3 while another suite ran, these fixtures took three to - * four times their unloaded time and failed at the former literal budgets. + * `TEST_BUDGET_SCALE` is the headroom for a loaded one. These fixtures are + * spawn-bound (a retained adoption dispatch runs about 750 stub-engine + * processes; an authored start runs the compiler several times), so their wall + * time grows faster than the load average. On the M5 at 1-min load 15-40, and + * on the M3 while another suite ran, they took three to four times their + * unloaded time. With a second suite plus churn workers (load 28-66) a scale + * of 4 still expired Source5 authored starts and v7 retained dispatches, and + * one v7 dispatch case took 146 s at load 22-50. */ -export const TEST_BUDGET_SCALE = 4; +export const TEST_BUDGET_SCALE = 10; /** Scale the unloaded time a fixture needs to the budget a loaded host needs. */ export function testBudgetMs(unloadedMs: number): number { diff --git a/tests/native-authored-project-start.test.ts b/tests/native-authored-project-start.test.ts index 9271eb2f5..f2cf6a536 100644 --- a/tests/native-authored-project-start.test.ts +++ b/tests/native-authored-project-start.test.ts @@ -74,9 +74,10 @@ const DEADLINE_WALL_BOUND_MS = 1000; /** * Upper bound for a stand-in owner that never receives its release or SIGTERM. * This bounds the orphan a broken case can leave rather than preventing it: the - * cap outlasts the case's Bun timeout. Passing cases release it explicitly. + * cap outlasts the longest case's Bun timeout (two start and stop cycles). + * Passing cases release it explicitly. */ -const OWNER_RELEASE_CAP_MS = 2 * CASE_TIMEOUT_MS; +const OWNER_RELEASE_CAP_MS = 3 * CASE_TIMEOUT_MS; const cases = createNativeFixtureCases({ label: "Startup fixture", defaultTimeoutMs: CASE_TIMEOUT_MS, @@ -1069,7 +1070,7 @@ macTest( serveNativeAuthoredProject({ ...selected, run, - startupTimeoutMs: 10_000, + startupTimeoutMs: STARTUP_TIMEOUT_MS, }), "not-started" ); @@ -1776,7 +1777,8 @@ macTest( 2 ); }, - CASE_TIMEOUT_MS + // Two full start and stop cycles. + 2 * CASE_TIMEOUT_MS ); macTest( diff --git a/tests/native-compose-adoption-generation.test.ts b/tests/native-compose-adoption-generation.test.ts index 4e58d53ae..1b664e658 100644 --- a/tests/native-compose-adoption-generation.test.ts +++ b/tests/native-compose-adoption-generation.test.ts @@ -39,6 +39,7 @@ import { renderComposeEnvReferenceOverride, renderManagedComposeEnvOverride, } from "../src/lib/compose-managed-env.ts"; +import { COMPOSE_STARTUP_TIMEOUT_ENV } from "../src/lib/compose-startup-budget.ts"; import { acquireLegacyComposeAdoptionPreparationBinding } from "../src/lib/native-compose-adoption-binding.ts"; import { tryLegacyComposeAdoptedCommand } from "../src/lib/native-compose-adoption-command.ts"; import { @@ -160,6 +161,7 @@ let priorPath: string | undefined; let priorCI: string | undefined; let priorExecutionMode: string | undefined; let priorRuntimeBackend: string | undefined; +let priorDispatchDeadline: string | undefined; let fixture: Fixture; let publicStatic7Commands = 0; const publicStatic7Settlement = createCompletedJobFixtureSettlement(); @@ -170,7 +172,13 @@ beforeEach(async () => { priorCI = process.env.CI; priorExecutionMode = process.env.HACK_EXECUTION_MODE; priorRuntimeBackend = process.env.HACK_RUNTIME_BACKEND; + priorDispatchDeadline = process.env[COMPOSE_STARTUP_TIMEOUT_ENV]; process.env.HACK_RUNTIME_BACKEND = "compose"; + // Public dispatch (`tryLegacyComposeAdoptedCommand`) takes its caller + // deadline from the Compose startup budget, 90 s by default. No case here + // tests that default, so inject the direct operations' harness deadline + // through the product's own setting. + process.env[COMPOSE_STARTUP_TIMEOUT_ENV] = String(OPERATION_DEADLINE_MS); root = await realpath( await mkdtemp(join(tmpdir(), "native-adoption-binding-")) ); @@ -332,6 +340,7 @@ afterEach(async () => { restoreEnv("CI", priorCI); restoreEnv("HACK_EXECUTION_MODE", priorExecutionMode); restoreEnv("HACK_RUNTIME_BACKEND", priorRuntimeBackend); + restoreEnv(COMPOSE_STARTUP_TIMEOUT_ENV, priorDispatchDeadline); await rm(root, { recursive: true, force: true }); }); async function save() { diff --git a/tests/native-process-policy-initial-trace.test.ts b/tests/native-process-policy-initial-trace.test.ts index 5ea5b22b9..124f30a6d 100644 --- a/tests/native-process-policy-initial-trace.test.ts +++ b/tests/native-process-policy-initial-trace.test.ts @@ -35,21 +35,21 @@ import { import { restoreEnv } from "./helpers/env.ts"; import { testBudgetMs } from "./helpers/test-budget.ts"; +/** Bun's own default case timeout, used as the unloaded time of a case that names none. */ +const BUN_DEFAULT_CASE_TIMEOUT_MS = 5000; + /** - * A case whose last argument is the time its replay children need on an - * unloaded host. The Bun timeout is that time scaled by `testBudgetMs`, since - * each replay child cold-starts Bun and one stub engine per recorded reply. - * Cases without one keep Bun's default. + * A case whose last argument is the time it needs on an unloaded host. The Bun + * timeout is that time scaled by `testBudgetMs`, since these cases cold-start + * Bun children: trace forwarders, stub engines and replay children. A case + * that names no time gets Bun's default, scaled the same way, so no case here + * keeps an unscaled 5 s budget. */ function test( name: string, run: () => void | Promise, - unloadedTimeoutMs?: number + unloadedTimeoutMs = BUN_DEFAULT_CASE_TIMEOUT_MS ): void { - if (unloadedTimeoutMs === undefined) { - bunTest(name, run); - return; - } bunTest(name, run, testBudgetMs(unloadedTimeoutMs)); } @@ -271,7 +271,12 @@ test("initial forwarding preserves a multicall Docker entry and refuses its repl await symlink(target, entry); process.env.PATH = root; expect( - (await exec([target], { stdin: "ignore", timeoutMs: 3000 })).exitCode + ( + await exec([target], { + stdin: "ignore", + timeoutMs: FORWARDED_DOCKER_EXEC_TIMEOUT_MS, + }) + ).exitCode ).toBe(79); const prepared = await prepareProcessPolicyInitialTrace({ directory: traceRoot, @@ -280,7 +285,7 @@ test("initial forwarding preserves a multicall Docker entry and refuses its repl await exec([join(traceRoot, "docker"), "version"], { env: { PATH: prepared.path }, stdin: "ignore", - timeoutMs: 3000, + timeoutMs: FORWARDED_DOCKER_EXEC_TIMEOUT_MS, }); expect(await run()).toEqual({ exitCode: 0, @@ -331,7 +336,10 @@ test("malformed UTF-8 keeps the caller's exact bytes/exit and makes replay unava expect(await new Response(child.stderr).text()).toBe(""); process.env.PATH = prepared.path; await expect( - assertNativeComposeOwned({ ...selection, timeoutMs: 3000 }) + assertNativeComposeOwned({ + ...selection, + timeoutMs: FORWARDED_DOCKER_EXEC_TIMEOUT_MS, + }) ).rejects.toMatchObject({ code: "E_NATIVE_COMPOSE_PROBE" }); await expect( readProcessPolicyInitialTrace(prepared.directory) From f0352ce31ae611e8fe9cc5ae9bd7eda6b55ff0cd Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Sat, 24 Oct 2026 11:28:29 -0400 Subject: [PATCH 04/10] test(native): scale load budgets by 20 and cap authored deadlines at the product ceiling At scale 10 the adoption retained-dispatch subset still expired its 200 s caller deadlines on the M5 while the authored suite and 8 churn workers ran (1-min load 44-91): one public retained up outlasted 200 s and v7 start/restart refused with E_LEGACY_ADOPTION_DEADLINE. Raise TEST_BUDGET_SCALE to 20. The authored startup deadline is capped at 300 s, the largest value serveNativeAuthoredProject accepts, and the stop deadline uses the same value. Refs HACK-1367 Co-Authored-By: Claude Opus 5.5 --- tests/helpers/test-budget.ts | 10 ++++++---- tests/native-authored-project-start.test.ts | 12 +++++++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/helpers/test-budget.ts b/tests/helpers/test-budget.ts index 5b3cd8b29..3e6268c8b 100644 --- a/tests/helpers/test-budget.ts +++ b/tests/helpers/test-budget.ts @@ -14,11 +14,13 @@ * processes; an authored start runs the compiler several times), so their wall * time grows faster than the load average. On the M5 at 1-min load 15-40, and * on the M3 while another suite ran, they took three to four times their - * unloaded time. With a second suite plus churn workers (load 28-66) a scale - * of 4 still expired Source5 authored starts and v7 retained dispatches, and - * one v7 dispatch case took 146 s at load 22-50. + * unloaded time. With a second suite plus churn workers, the M5 at load 28-90 + * took more than ten times: one public retained `up` dispatch outlasted a + * 200 s deadline, and a v7 dispatch case took 146 s even at load 22-50. + * A deadline a case does not test must not expire first, so the scale keeps + * twice that headroom. */ -export const TEST_BUDGET_SCALE = 10; +export const TEST_BUDGET_SCALE = 20; /** Scale the unloaded time a fixture needs to the budget a loaded host needs. */ export function testBudgetMs(unloadedMs: number): number { diff --git a/tests/native-authored-project-start.test.ts b/tests/native-authored-project-start.test.ts index f2cf6a536..3e2c82550 100644 --- a/tests/native-authored-project-start.test.ts +++ b/tests/native-authored-project-start.test.ts @@ -55,15 +55,21 @@ const CANARY = "synthetic-private-native-start-canary"; * afterwards. Harness budget: a passing case settles as soon as they exit. */ const SETTLEMENT_BUDGET_MS = testBudgetMs(5000); +/** Largest caller startup deadline `serveNativeAuthoredProject` accepts. */ +const MAX_STARTUP_TIMEOUT_MS = 300_000; /** * Caller startup deadline for cases that are not about the deadline itself. * Each start spawns several Bun children (compiler, planner, owner, hooks); on * a loaded host that takes well over the 10-20 s these cases used to allow. - * Cases that test the deadline pass their own literal product deadline. + * The scaled budget is capped at the product's own ceiling. Cases that test + * the deadline pass their own literal product deadline. */ -const STARTUP_TIMEOUT_MS = testBudgetMs(15_000); +const STARTUP_TIMEOUT_MS = Math.min( + testBudgetMs(15_000), + MAX_STARTUP_TIMEOUT_MS +); /** Caller stop deadline for ordinary `down` in the hook and live-stop cases. */ -const STOP_TIMEOUT_MS = testBudgetMs(15_000); +const STOP_TIMEOUT_MS = STARTUP_TIMEOUT_MS; /** * Body budget for every case: a full start plus a stop, with slack. The guard * adds the settlement budget on top, so cases never fall back to Bun's 5 s. From fa5b36d7d783864f18693efcea8ad939c2a8c3c4 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Sat, 24 Oct 2026 11:57:10 -0400 Subject: [PATCH 05/10] test(native): open adoption stores with the largest probe-phase bound One store probe carries a single aggregate 15 s clock across a whole resource inspection, dozens of stub-engine processes. Under load that expired as E_NATIVE_COMPOSE_PROBE_TIMEOUT, reported as E_LEGACY_ADOPTION_STATE. Stores the cases open now pass timeoutMs 60 s, the largest the product accepts, unless a case names its own. The two expiry cases used a 5 s caller deadline that expired before the callback at load 30-45. They now use 45 s, under the 60 s phase bound so the caller deadline still ends the held probe; the 1.5 s promptness assertion is unchanged. Refs HACK-1367 Co-Authored-By: Claude Opus 5.5 --- ...native-compose-adoption-generation.test.ts | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/tests/native-compose-adoption-generation.test.ts b/tests/native-compose-adoption-generation.test.ts index 1b664e658..c77ee3b15 100644 --- a/tests/native-compose-adoption-generation.test.ts +++ b/tests/native-compose-adoption-generation.test.ts @@ -48,7 +48,7 @@ import { } from "../src/lib/native-compose-adoption-execution.ts"; import { LegacyComposeAdoptedGenerationError, - openLegacyComposeAdoptedGenerationStore, + openLegacyComposeAdoptedGenerationStore as openAdoptedGenerationStore, } from "../src/lib/native-compose-adoption-generation.ts"; import { inspectLegacyComposeAdoptionSelection } from "../src/lib/native-compose-adoption-marker.ts"; import { @@ -77,14 +77,41 @@ import { testBudgetMs } from "./helpers/test-budget.ts"; * Harness budgets. Every engine probe below spawns the Bun stub engine, and one * retained dispatch issues several hundred probes, so an owner operation's wall * time follows host load. Case timeouts and the caller deadlines of cases that - * are not about the deadline derive from `testBudgetMs`. The short deadlines in - * the expiry cases (5 s) and their promptness assertions are product bounds - * under test and stay literal. + * are not about the deadline derive from `testBudgetMs`. The expiry cases' + * promptness assertion (1.5 s past the deadline) is a product bound under test + * and stays literal. */ /** Caller deadline for one owner operation in a case that is not about the deadline. */ const OPERATION_DEADLINE_MS = testBudgetMs(20_000); const test = (name: string, run: () => Promise) => boundedTest(name, run, testBudgetMs(20_000)); +/** + * Probe-phase bound for stores these cases open. One store probe carries a + * single aggregate clock across a whole inspection (dozens of stub-engine + * processes), 15 s by default. That default is a product bound no case here + * tests, so stores open with the largest phase bound the product accepts. + * Public dispatch opens its own store and keeps the default (see the PR gaps). + */ +const PROBE_PHASE_BUDGET_MS = 60_000; +/** + * Caller deadline in the cases that test its expiry after the callback. The + * product behavior under test (expiry ends a held probe or an overlong callback + * with `E_LEGACY_ADOPTION_DEADLINE`, keeps the operation pending, and returns + * within 1.5 s of the deadline) does not depend on its length, but the work + * before the callback must finish first: a 5 s deadline expired before the + * callback at load 30-45. It stays under the probe-phase bound, so the caller + * deadline, not a probe's own phase clock, ends a held probe. + */ +const EXPIRY_DEADLINE_MS = PROBE_PHASE_BUDGET_MS - 15_000; +/** The product store opener, with the harness probe-phase bound unless a case names one. */ +function openLegacyComposeAdoptedGenerationStore( + input: Parameters[0] +): ReturnType { + return openAdoptedGenerationStore({ + timeoutMs: PROBE_PHASE_BUDGET_MS, + ...input, + }); +} const CANARY = "synthetic-private-adoption-canary"; const ID = "a".repeat(64); const NETWORK = "b".repeat(64); @@ -1325,7 +1352,7 @@ boundedTest( try { const binary = await compiler(); await store.publish({ generation, binary }); - const deadline = Date.now() + 5000; + const deadline = Date.now() + EXPIRY_DEADLINE_MS; await refusal( store.withMutation({ generation, @@ -1484,7 +1511,7 @@ for (const stage of ["per-effect freshness", "final observation"] as const) { try { const binary = await compiler(); await store.publish({ generation, binary }); - const deadline = Date.now() + 5000; + const deadline = Date.now() + EXPIRY_DEADLINE_MS; let reachedCallback = false; await refusal( store.withMutation({ From 073041c1935e8309753d54d3d219c9fd0458cf5f Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Sat, 24 Oct 2026 13:21:13 -0400 Subject: [PATCH 06/10] test(e2e): run the replayed ownership policy with its largest probe bound The replay child runs the real ownership policy against recorded replies, one stub engine per reply under a single 15 s probe clock. Under a 36-worker churn (load 100-120) that clock expired with E_NATIVE_COMPOSE_PROBE_TIMEOUT and changed the replayed outcome of 'trace replay keeps a member whose placeholder first appears in the recheck scan unready'. The replay classifies replies and never tests that clock, so the child now passes timeoutMs 60 s, the largest the policy accepts, unless the recorded selection names one. The child budget still outlasts it. Refs HACK-1367 Co-Authored-By: Claude Opus 5.5 --- .../native-process-policy-initial-replay.ts | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/e2e/native-process-policy-initial-replay.ts b/tests/e2e/native-process-policy-initial-replay.ts index afe454047..87f2a7ce7 100644 --- a/tests/e2e/native-process-policy-initial-replay.ts +++ b/tests/e2e/native-process-policy-initial-replay.ts @@ -21,13 +21,25 @@ import { } from "./native-process-policy-startup-diagnostic.ts"; const REFUSAL = "Initial process-policy replay unavailable; values omitted"; +/** + * Aggregate probe bound the replayed ownership policy runs with, unless the + * recorded selection names one: the largest the policy accepts. One query + * spawns a stub engine per recorded reply under a single clock, 15 s by + * default. The replay classifies recorded replies and never tests that clock; + * at the default, a 36-worker churn (load 100-120) expired it + * (`E_NATIVE_COMPOSE_PROBE_TIMEOUT`) and changed the replayed outcome. + */ +const REPLAY_PROBE_BUDGET_MS = 60_000; /** * Harness budget for one replay child: a fresh Bun process that transpiles the - * ownership policy and spawns one stub engine per recorded reply. This bounds - * the replay itself; the ownership policy's own probe bounds run unchanged - * inside it. A fixed 5 s refused replays on loaded hosts. + * ownership policy and spawns one stub engine per recorded reply. It outlasts + * the policy's own probe bound inside it. A fixed 5 s refused replays on + * loaded hosts. */ -const REPLAY_CHILD_BUDGET_MS = testBudgetMs(5000); +const REPLAY_CHILD_BUDGET_MS = Math.max( + testBudgetMs(5000), + REPLAY_PROBE_BUDGET_MS + 10_000 +); /** Harness budget for every replay child of one trace summary together. */ const REPLAY_SUMMARY_BUDGET_MS = testBudgetMs(30_000); const SERVICES = ["forced", "graceful", "reaper", "retry"] as const; @@ -170,7 +182,8 @@ await Bun.write(Bun.stdout,row.stdout);process.exit(row.exitCode); await chmod(docker, 0o700); const program = ` import {assertNativeComposeOwned,observeNativeComposeStartupOwned,NativeComposeOwnershipError,nativeComposeOwnershipRefusal,nativeComposeTopologyPredicate} from ${JSON.stringify(join(import.meta.dir, "../../src/lib/native-compose-ownership.ts"))}; -const {selection}=await Bun.file(${JSON.stringify(inputs)}).json(); +const {selection:recorded}=await Bun.file(${JSON.stringify(inputs)}).json(); +const selection={timeoutMs:${REPLAY_PROBE_BUDGET_MS},...recorded}; let outcome="refused",code=null,reason=null,topologyPredicate=null; try {const value=${REPLAY_CALLS[opts.mode]};outcome=value===null?"unready":"owned";} catch(error){code=error instanceof NativeComposeOwnershipError?error.code:null;reason=nativeComposeOwnershipRefusal(error)??null;topologyPredicate=nativeComposeTopologyPredicate(error)??null;} From 7fbeb501b2110014bfe8da7ffd686788d9fa7e55 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Sat, 24 Oct 2026 17:17:04 -0400 Subject: [PATCH 07/10] test(runtime-core): take persistent-data enrollment deadlines from one test budget The witnessed, enrollment and engine persistent-data tests passed literal 5-10 s deadlines to fake-backend enrollments and retained reads. Move the runtime tests' 120 s RUN_BUDGET to a crate-level TEST_RUN_BUDGET and use it for those deadlines too. Passing calls never wait on it; expiry cases keep their own zero deadline. Refs HACK-1367 Co-Authored-By: Claude Opus 5.5 --- packages/runtime-core/src/lib.rs | 4 ++++ .../graph/native/persistent_data/engine/tests.rs | 5 ++--- .../native/persistent_data/enrollment/tests.rs | 8 ++++---- .../persistent_data/enrollment/witnessed/tests.rs | 6 +++--- .../src/provider/graph/native/runtime/tests.rs | 12 +++--------- packages/runtime-core/src/test_budget.rs | 14 ++++++++++++++ 6 files changed, 30 insertions(+), 19 deletions(-) create mode 100644 packages/runtime-core/src/test_budget.rs diff --git a/packages/runtime-core/src/lib.rs b/packages/runtime-core/src/lib.rs index c5fc6b8fb..07c33c9d0 100644 --- a/packages/runtime-core/src/lib.rs +++ b/packages/runtime-core/src/lib.rs @@ -19,8 +19,12 @@ pub const CANDIDATE_VERSION: &str = "5.0.0-dev.6"; mod error; mod error_stop_diagnostics; pub use error::CandidateError; +#[cfg(all(test, feature = "native-config-plan"))] +mod test_budget; #[cfg(test)] mod test_scratch; +#[cfg(all(test, feature = "native-config-plan"))] +pub(crate) use test_budget::TEST_RUN_BUDGET; #[cfg(test)] pub(crate) use test_scratch::next_test_scratch; diff --git a/packages/runtime-core/src/provider/graph/native/persistent_data/engine/tests.rs b/packages/runtime-core/src/provider/graph/native/persistent_data/engine/tests.rs index 208e8285f..009e63b53 100644 --- a/packages/runtime-core/src/provider/graph/native/persistent_data/engine/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/persistent_data/engine/tests.rs @@ -6,7 +6,6 @@ use std::{ os::unix::fs::{DirBuilderExt, MetadataExt}, path::PathBuf, rc::Rc, - time::Duration, }; struct Fixture { @@ -169,7 +168,7 @@ fn enroll(fixture: &Fixture, endpoint: &mut Endpoint<'_>) -> Result( state_root: &fixture.0, binding, intent: "dddddddddddddddddddddddddddddddd", - deadline: Instant::now() + Duration::from_secs(10), + deadline: Instant::now() + crate::TEST_RUN_BUDGET, cancelled, } } @@ -184,7 +184,7 @@ fn read(fixture: &Fixture, binding: &Binding, fake: &mut Fake) -> Result( state_root: &f.0, binding: b, intent: "f0000000000000000000000000000000", - deadline: Instant::now() + Duration::from_secs(5), + deadline: Instant::now() + crate::TEST_RUN_BUDGET, cancelled, }, witness: w, @@ -203,7 +203,7 @@ fn read_options<'a>(f: &'a Fixture, b: &'a Binding, c: &'a AtomicBool) -> ReadOp ReadOptions { state_root: &f.0, binding: b, - deadline: Instant::now() + Duration::from_secs(5), + deadline: Instant::now() + crate::TEST_RUN_BUDGET, cancelled: c, } } diff --git a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs index 334aeea27..26c4f7468 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs @@ -14,15 +14,9 @@ use std::{ const OWNER: &str = "cccccccccccccccccccccccccccccccc"; const BOOT: &str = "12345678-abcd-abcd-abcd-123456789abc"; const RUN: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; -/// Harness budget for each fake-backend graph run, selection, exec, run or log read here. -/// -/// The fake backend is ready as soon as each effect is applied, so a passing call returns -/// without waiting on this budget; it only bounds a call that would otherwise hang. It is a -/// test budget, not the product bound: `execution::run` still refuses budgets above its own -/// 600 s limit, and cases that exercise expiry pass their own short or zero deadline. Fixed -/// 1-5 s budgets failed with `graph_timeout` under host load (journal writes and syncs on a -/// loaded host), so every such call takes its budget from this one constant. -const RUN_BUDGET: Duration = Duration::from_secs(120); +/// Harness budget for each fake-backend graph run, selection, exec, run or log read here +/// (see `crate::TEST_RUN_BUDGET`). +const RUN_BUDGET: Duration = crate::TEST_RUN_BUDGET; struct Fixture { root: PathBuf, project: PathBuf, diff --git a/packages/runtime-core/src/test_budget.rs b/packages/runtime-core/src/test_budget.rs new file mode 100644 index 000000000..20dffb751 --- /dev/null +++ b/packages/runtime-core/src/test_budget.rs @@ -0,0 +1,14 @@ +//! Harness budgets for tests that drive fake backends. + +use std::time::Duration; + +/// Harness deadline for one fake-backend operation in a test: a graph run, selection, +/// exec, log read, or a persistent-data enrollment or retained read. +/// +/// The fakes answer at once, so a passing call returns without waiting on this budget; +/// it only bounds a call that would otherwise hang. It is a test budget, not a product +/// bound: `execution::run` still refuses budgets above its own 600 s limit, and cases +/// that exercise expiry pass their own short or zero deadline. Fixed 1-5 s deadlines +/// failed with `graph_timeout` under host load (journal writes and syncs on a loaded +/// host), so these calls take their deadline from this one constant. +pub(crate) const TEST_RUN_BUDGET: Duration = Duration::from_secs(120); From 5afe58aaf693ab12f1b3932c146af9b1378b4add Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Sat, 24 Oct 2026 17:17:21 -0400 Subject: [PATCH 08/10] test: keep load budgets inside the hosted shard's per-file limit testBudgetMs now stops at CASE_TIMEOUT_CEILING_MS, a third of the shard's 720 s per-file limit, and withinCaseCeilingMs checks composed case timeouts when a file loads. A hung case then times out by name before the shard stops its file. The authored start file derives its case budget from the ceiling minus settlement, caps the startup and stop deadlines at half of it, and bounds its stand-in owner and hanging status request at the ceiling instead of 1800 s and 1200 s. Refs HACK-1367 Co-Authored-By: Claude Opus 5.5 --- tests/helpers/native-fixture-cases.ts | 4 +- tests/helpers/test-budget.ts | 68 +++++++++++++++++++-- tests/native-authored-project-start.test.ts | 52 ++++++++++------ tests/test-budget.test.ts | 44 +++++++++++++ 4 files changed, 142 insertions(+), 26 deletions(-) create mode 100644 tests/test-budget.test.ts diff --git a/tests/helpers/native-fixture-cases.ts b/tests/helpers/native-fixture-cases.ts index 75c715a29..76ad6ebaa 100644 --- a/tests/helpers/native-fixture-cases.ts +++ b/tests/helpers/native-fixture-cases.ts @@ -53,7 +53,7 @@ export type NativeFixtureCases = { }; /** Slack between the case's own deadline plus settlement and Bun's timeout. */ -const BUN_TIMEOUT_MARGIN_MS = 5000; +export const NATIVE_FIXTURE_BUN_MARGIN_MS = 5000; function message(error: unknown): string { return error instanceof Error ? error.message : String(error); @@ -190,7 +190,7 @@ export function createNativeFixtureCases(opts: { bunTest( name, () => runCase(body, timeoutMs), - timeoutMs + opts.settlementBudgetMs + BUN_TIMEOUT_MARGIN_MS + timeoutMs + opts.settlementBudgetMs + NATIVE_FIXTURE_BUN_MARGIN_MS ); }, }; diff --git a/tests/helpers/test-budget.ts b/tests/helpers/test-budget.ts index 3e6268c8b..760ecc236 100644 --- a/tests/helpers/test-budget.ts +++ b/tests/helpers/test-budget.ts @@ -1,3 +1,5 @@ +import { DEFAULT_FILE_TIMEOUT_SECONDS } from "../../scripts/test-shard.ts"; + /** * Harness budgets for load-sensitive tests. * @@ -19,13 +21,69 @@ * 200 s deadline, and a v7 dispatch case took 146 s even at load 22-50. * A deadline a case does not test must not expire first, so the scale keeps * twice that headroom. + * + * The scale stops at `CASE_TIMEOUT_CEILING_MS`. Hosted CI runs each file under + * the shard's per-file limit; a case whose budget outlives that limit is + * reported as a file timeout instead of by name. So the headroom is real only + * up to the ceiling, and a host loaded past it fails cases by name. */ export const TEST_BUDGET_SCALE = 20; -/** Scale the unloaded time a fixture needs to the budget a loaded host needs. */ -export function testBudgetMs(unloadedMs: number): number { - if (!(Number.isSafeInteger(unloadedMs) && unloadedMs > 0)) { - throw new Error("Test budget refused a non-positive or fractional time."); +/** Per-file limit of the hosted test shard (`scripts/test-shard.ts`). */ +export const SHARD_FILE_TIMEOUT_MS = DEFAULT_FILE_TIMEOUT_SECONDS * 1000; + +/** + * Largest Bun timeout one case may have, its settlement or drain included: a + * third of the shard's per-file limit. The slowest hosted file (adoption + * generation, 4m55s-6m57s) plus one case hung for this long still ends inside + * the 720 s limit, so Bun prints that case's own timeout before the shard + * stops the file. + */ +export const CASE_TIMEOUT_CEILING_MS = Math.floor(SHARD_FILE_TIMEOUT_MS / 3); + +/** + * Scale the unloaded time a fixture needs to the budget a loaded host needs, + * never above `ceilingMs` (the case ceiling by default). A file whose Bun + * timeout adds settlement or drain time to a body budget passes the body's + * share of the ceiling. + */ +export function testBudgetMs( + unloadedMs: number, + opts: { readonly ceilingMs?: number } = {} +): number { + const ceilingMs = opts.ceilingMs ?? CASE_TIMEOUT_CEILING_MS; + if ( + !( + Number.isSafeInteger(unloadedMs) && + unloadedMs > 0 && + Number.isSafeInteger(ceilingMs) && + ceilingMs > 0 && + ceilingMs <= CASE_TIMEOUT_CEILING_MS + ) + ) { + throw new Error( + "E_TEST_BUDGET_INVALID: a test budget must be a positive whole number of milliseconds within the case ceiling." + ); + } + return Math.min(unloadedMs * TEST_BUDGET_SCALE, ceilingMs); +} + +/** + * Check a composed Bun timeout (a body budget plus settlement, drain or + * margin) against the case ceiling when the file loads, so a file cannot + * register a case that would outlive the shard's per-file limit. + */ +export function withinCaseCeilingMs(timeoutMs: number): number { + if ( + !( + Number.isSafeInteger(timeoutMs) && + timeoutMs > 0 && + timeoutMs <= CASE_TIMEOUT_CEILING_MS + ) + ) { + throw new Error( + "E_TEST_BUDGET_CEILING: a case timeout must be a positive whole number of milliseconds within the case ceiling." + ); } - return unloadedMs * TEST_BUDGET_SCALE; + return timeoutMs; } diff --git a/tests/native-authored-project-start.test.ts b/tests/native-authored-project-start.test.ts index 3e2c82550..664a69d0e 100644 --- a/tests/native-authored-project-start.test.ts +++ b/tests/native-authored-project-start.test.ts @@ -44,9 +44,14 @@ import { writeLifecycleMuxFixture } from "./helpers/lifecycle-mux-fixture.ts"; import { awaitPidFile } from "./helpers/native-authored-settlement.ts"; import { createNativeFixtureCases, + NATIVE_FIXTURE_BUN_MARGIN_MS, type NativeFixtureCase, } from "./helpers/native-fixture-cases.ts"; -import { testBudgetMs } from "./helpers/test-budget.ts"; +import { + CASE_TIMEOUT_CEILING_MS, + testBudgetMs, + withinCaseCeilingMs, +} from "./helpers/test-budget.ts"; const run = "a".repeat(32); const CANARY = "synthetic-private-native-start-canary"; @@ -54,36 +59,44 @@ const CANARY = "synthetic-private-native-start-canary"; * Allowance for a case's owners, groups and stop endpoints to be absent * afterwards. Harness budget: a passing case settles as soon as they exit. */ -const SETTLEMENT_BUDGET_MS = testBudgetMs(5000); +const SETTLEMENT_BUDGET_MS = testBudgetMs(1500); /** Largest caller startup deadline `serveNativeAuthoredProject` accepts. */ const MAX_STARTUP_TIMEOUT_MS = 300_000; +/** + * Body budget for every case, the two-cycle case included: what the case + * ceiling leaves after the settlement budget and the guard's margin. A case + * that overruns it fails by name, inside the hosted shard's per-file limit. + */ +const CASE_TIMEOUT_MS = + CASE_TIMEOUT_CEILING_MS - SETTLEMENT_BUDGET_MS - NATIVE_FIXTURE_BUN_MARGIN_MS; +withinCaseCeilingMs( + CASE_TIMEOUT_MS + SETTLEMENT_BUDGET_MS + NATIVE_FIXTURE_BUN_MARGIN_MS +); /** * Caller startup deadline for cases that are not about the deadline itself. * Each start spawns several Bun children (compiler, planner, owner, hooks); on * a loaded host that takes well over the 10-20 s these cases used to allow. - * The scaled budget is capped at the product's own ceiling. Cases that test - * the deadline pass their own literal product deadline. + * The scaled budget is capped at the product's own ceiling and at half the + * case budget, so a start and its stop fit in one case. Cases that test the + * deadline pass their own literal product deadline. */ const STARTUP_TIMEOUT_MS = Math.min( testBudgetMs(15_000), - MAX_STARTUP_TIMEOUT_MS + MAX_STARTUP_TIMEOUT_MS, + Math.floor(CASE_TIMEOUT_MS / 2) ); /** Caller stop deadline for ordinary `down` in the hook and live-stop cases. */ const STOP_TIMEOUT_MS = STARTUP_TIMEOUT_MS; -/** - * Body budget for every case: a full start plus a stop, with slack. The guard - * adds the settlement budget on top, so cases never fall back to Bun's 5 s. - */ -const CASE_TIMEOUT_MS = 2 * STARTUP_TIMEOUT_MS; /** A caller's 50 ms startup deadline must return the frontend well inside this wall bound. */ const DEADLINE_WALL_BOUND_MS = 1000; /** - * Upper bound for a stand-in owner that never receives its release or SIGTERM. - * This bounds the orphan a broken case can leave rather than preventing it: the - * cap outlasts the longest case's Bun timeout (two start and stop cycles). - * Passing cases release it explicitly. + * Upper bound for a stand-in owner that never receives its release or SIGTERM, + * and for a stand-in status request that hangs. This bounds the orphan a broken + * case can leave rather than preventing it: it outlasts every case's Bun + * timeout, which the case ceiling bounds, and ends well inside the hosted + * shard's per-file limit. Passing cases release or cancel them explicitly. */ -const OWNER_RELEASE_CAP_MS = 3 * CASE_TIMEOUT_MS; +const STAND_IN_CAP_MS = CASE_TIMEOUT_CEILING_MS; const cases = createNativeFixtureCases({ label: "Startup fixture", defaultTimeoutMs: CASE_TIMEOUT_MS, @@ -495,7 +508,7 @@ import {appendFile,readFile,readdir,rename,unlink,writeFile} from 'node:fs/promi import {createHash} from 'node:crypto';import {join} from 'node:path'; const args=process.argv.slice(2);const action=args[4];await appendFile(${JSON.stringify(calls)},JSON.stringify(args)+'\\n'); if(${options.runtimeStopped === true}&&args[2]==='runtime'&&args[3]==='status'){console.log(JSON.stringify({phase:'stopped',process_alive:false}));process.exit(0)} -if(${options.runtimeStatusHangs === true}&&args[2]==='runtime'&&args[3]==='status'){const staged=${JSON.stringify(statusPid)}+'.'+process.pid;await writeFile(staged,process.pid+'\\n');await rename(staged,${JSON.stringify(statusPid)});await new Promise((resolve)=>setTimeout(resolve,${2 * CASE_TIMEOUT_MS}));process.exit(0)} +if(${options.runtimeStatusHangs === true}&&args[2]==='runtime'&&args[3]==='status'){const staged=${JSON.stringify(statusPid)}+'.'+process.pid;await writeFile(staged,process.pid+'\\n');await rename(staged,${JSON.stringify(statusPid)});await new Promise((resolve)=>setTimeout(resolve,${STAND_IN_CAP_MS}));process.exit(0)} if(${options.runtimeStatusWrappedStop === true}&&args[2]==='runtime'&&args[3]==='status'){console.error(JSON.stringify({code:'native_graph_foreground',cause_code:'runtime_not_running',message:${JSON.stringify(CANARY)}}));process.exit(2)} const journal=${JSON.stringify(journal)}; const root=${JSON.stringify(join(projectDir, ".internal", "native-authored-runs"))}; @@ -563,7 +576,7 @@ if(['plan','serve','frontend-plan','frontend-serve'].includes(action)){ // Handshake, not a wall-clock window: retire only once the frontend has // finished ready publication and the test released this run (or on SIGTERM). const release=${JSON.stringify(ownerRelease(root, ""))}+input.run; - const cap=performance.now()+${OWNER_RELEASE_CAP_MS}; + const cap=performance.now()+${STAND_IN_CAP_MS}; while(!(await Bun.file(release).exists())&&performance.now()undefined); await finish(); @@ -1783,8 +1796,9 @@ macTest( 2 ); }, - // Two full start and stop cycles. - 2 * CASE_TIMEOUT_MS + // Two full start and stop cycles. The case ceiling leaves no room for a + // second case budget; a passing cycle takes a small part of one. + CASE_TIMEOUT_MS ); macTest( diff --git a/tests/test-budget.test.ts b/tests/test-budget.test.ts new file mode 100644 index 000000000..c0026608a --- /dev/null +++ b/tests/test-budget.test.ts @@ -0,0 +1,44 @@ +import { expect, test } from "bun:test"; +import { DEFAULT_FILE_TIMEOUT_SECONDS } from "../scripts/test-shard.ts"; +import { + CASE_TIMEOUT_CEILING_MS, + SHARD_FILE_TIMEOUT_MS, + TEST_BUDGET_SCALE, + testBudgetMs, + withinCaseCeilingMs, +} from "./helpers/test-budget.ts"; + +test("the case ceiling follows the hosted shard's per-file limit", () => { + expect(SHARD_FILE_TIMEOUT_MS).toBe(DEFAULT_FILE_TIMEOUT_SECONDS * 1000); + // The slowest hosted file took up to 417 s; it plus one hung case must end + // before the shard stops the file, so Bun names that case. + expect(417_000 + CASE_TIMEOUT_CEILING_MS).toBeLessThan(SHARD_FILE_TIMEOUT_MS); +}); + +test("test budgets scale the unloaded time and stop at the ceiling", () => { + expect(testBudgetMs(250)).toBe(250 * TEST_BUDGET_SCALE); + expect(testBudgetMs(1_000_000)).toBe(CASE_TIMEOUT_CEILING_MS); + expect(testBudgetMs(1_000_000, { ceilingMs: 10_000 })).toBe(10_000); + expect(testBudgetMs(100, { ceilingMs: 10_000 })).toBe( + Math.min(100 * TEST_BUDGET_SCALE, 10_000) + ); +}); + +test("test budgets refuse invalid times and ceilings above the case ceiling", () => { + for (const unloadedMs of [0, -1, 1.5, Number.NaN]) { + expect(() => testBudgetMs(unloadedMs)).toThrow(/^E_TEST_BUDGET_INVALID/); + } + expect(() => + testBudgetMs(1000, { ceilingMs: CASE_TIMEOUT_CEILING_MS + 1 }) + ).toThrow(/^E_TEST_BUDGET_INVALID/); +}); + +test("a composed case timeout must fit the case ceiling", () => { + expect(withinCaseCeilingMs(CASE_TIMEOUT_CEILING_MS)).toBe( + CASE_TIMEOUT_CEILING_MS + ); + expect(() => withinCaseCeilingMs(CASE_TIMEOUT_CEILING_MS + 1)).toThrow( + /^E_TEST_BUDGET_CEILING/ + ); + expect(() => withinCaseCeilingMs(0)).toThrow(/^E_TEST_BUDGET_CEILING/); +}); From 6e3ec5e183567374acb2cced020458fbc5ba35de Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Sat, 24 Oct 2026 17:17:26 -0400 Subject: [PATCH 09/10] test(native): replace the adoption file latch with a per-case drain guard Bun's timeout does not cancel a case body, so the adoption file latched after one abandoned public static7 case and failed every later case. Every case now runs under createCaseDrain: an overrun fails that case by name, the body gets a drain (the operation deadline plus slack) to settle, and only a body still running after that reserves the shared roots and env until it settles. Case budgets, the operation deadline and the drain fit the case ceiling. The three expiry cases no longer wait out a 45 s deadline: once the callback runs they move Date.now (the product's only clock) to just before or past the deadline, so the held probe still ends by the timeout the product derives from the remaining time. Refs HACK-1367 Co-Authored-By: Claude Opus 5.5 --- tests/case-drain.test.ts | 104 ++++ tests/helpers/case-drain.ts | 125 ++++ ...native-compose-adoption-generation.test.ts | 568 ++++++++++-------- 3 files changed, 552 insertions(+), 245 deletions(-) create mode 100644 tests/case-drain.test.ts create mode 100644 tests/helpers/case-drain.ts diff --git a/tests/case-drain.test.ts b/tests/case-drain.test.ts new file mode 100644 index 000000000..92b5f2648 --- /dev/null +++ b/tests/case-drain.test.ts @@ -0,0 +1,104 @@ +import { expect, test } from "bun:test"; +import { + CASE_DRAIN_BUN_MARGIN_MS, + createCaseDrain, +} from "./helpers/case-drain.ts"; + +const BUDGET_MS = 40; +const DRAIN_MS = 200; + +test("a case that overruns its budget but settles within the drain fails once and leaves later cases free", async () => { + const cases = createCaseDrain({ label: "Fixture", drainMs: DRAIN_MS }); + let finished = false; + await expect( + cases.run( + "slow", + async () => { + await Bun.sleep(BUDGET_MS + 60); + finished = true; + }, + BUDGET_MS + ) + ).rejects.toThrow(/^E_TEST_CASE_BUDGET: Fixture case exceeded .*settled/); + // The run returns only after the overrunning body settled. + expect(finished).toBe(true); + expect(cases.reservedBy()).toBeUndefined(); + expect(() => cases.requireUnreserved()).not.toThrow(); + let ran = false; + await cases.run( + "next", + () => { + ran = true; + }, + BUDGET_MS + ); + expect(ran).toBe(true); +}); + +test("a case still running past its drain reserves shared state only until its body settles", async () => { + const cases = createCaseDrain({ label: "Fixture", drainMs: DRAIN_MS }); + const release = Promise.withResolvers(); + await expect( + cases.run("hung", () => release.promise, BUDGET_MS) + ).rejects.toThrow( + /^E_TEST_CASE_BUDGET: Fixture case exceeded .*stays reserved/ + ); + expect(cases.reservedBy()).toBe("hung"); + expect(() => cases.requireUnreserved()).toThrow( + /^E_TEST_CASE_RESERVED: Fixture case "hung" still runs/ + ); + let ran = false; + await expect( + cases.run( + "while-hung", + () => { + ran = true; + }, + BUDGET_MS + ) + ).rejects.toThrow(/^E_TEST_CASE_RESERVED/); + expect(ran).toBe(false); + release.resolve(); + await Bun.sleep(0); + expect(cases.reservedBy()).toBeUndefined(); + await cases.run( + "after", + () => { + ran = true; + }, + BUDGET_MS + ); + expect(ran).toBe(true); +}); + +test("a case that fails within its budget reports its own error and reserves nothing", async () => { + const cases = createCaseDrain({ label: "Fixture", drainMs: DRAIN_MS }); + await expect( + cases.run( + "fails", + () => { + throw new Error("own failure"); + }, + BUDGET_MS + ) + ).rejects.toThrow("own failure"); + expect(cases.reservedBy()).toBeUndefined(); +}); + +test("a hung body that later rejects still clears its reservation", async () => { + const cases = createCaseDrain({ label: "Fixture", drainMs: DRAIN_MS }); + const release = Promise.withResolvers(); + await expect( + cases.run("hung", () => release.promise, BUDGET_MS) + ).rejects.toThrow(/^E_TEST_CASE_BUDGET/); + release.reject(new Error("late failure")); + await Bun.sleep(0); + expect(cases.reservedBy()).toBeUndefined(); +}); + +test("the Bun timeout covers the budget, the drain and the margin", () => { + const cases = createCaseDrain({ label: "Fixture", drainMs: DRAIN_MS }); + expect(cases.bunTimeoutMs(BUDGET_MS)).toBe( + BUDGET_MS + DRAIN_MS + CASE_DRAIN_BUN_MARGIN_MS + ); +}); diff --git a/tests/helpers/case-drain.ts b/tests/helpers/case-drain.ts new file mode 100644 index 000000000..2cb652d41 --- /dev/null +++ b/tests/helpers/case-drain.ts @@ -0,0 +1,125 @@ +import { test as bunTest } from "bun:test"; + +/** + * Case budgets for a test file whose cases share module-level fixture state + * (temp roots, a stub engine on `PATH`, process environment). + * + * Bun's own timeout does not cancel a case body. An abandoned body keeps + * running, and while the next case runs it can still read the shared globals, + * spawn through the shared `PATH` and restore environment it saved. Such files + * used to latch after the first abandoned case and fail every later case. + * + * This guard runs each body under its own budget instead, so an overrun fails + * that case by name. The overrunning body then gets `drainMs` (inside the same + * case's Bun timeout) to settle; bodies normally end within it, because each + * owner operation they await has its own caller deadline below `drainMs`. + * + * - Settled: the case still fails once with its budget refusal, and the next + * case starts normally. + * - Still running: the shared state stays reserved for that body. `reservedBy` + * names it, so the file's `beforeEach` refuses with a closed reason instead + * of reusing state the body may still touch, and `afterEach` keeps its roots. + * The reservation clears by itself when the body settles, and later cases + * run normally again. + */ +export type CaseDrain = { + /** Register a case: `budgetMs` bounds its body; the Bun timeout adds the drain. */ + readonly test: ( + name: string, + body: () => void | Promise, + budgetMs: number + ) => void; + /** Run one case body under `budgetMs`, then the drain. Rejects with the case's own outcome. */ + readonly run: ( + name: string, + body: () => void | Promise, + budgetMs: number + ) => Promise; + /** The Bun timeout for a body budget: budget, drain and margin. */ + readonly bunTimeoutMs: (budgetMs: number) => number; + /** The case whose body still runs past its budget and drain, if any. */ + readonly reservedBy: () => string | undefined; + /** Refuse, with a closed reason, while an earlier case's body still runs. */ + readonly requireUnreserved: () => void; +}; + +/** Slack between a case's budget plus drain and Bun's own timeout. */ +export const CASE_DRAIN_BUN_MARGIN_MS = 5000; + +type Outcome = "settled" | "overrun"; + +async function within(running: Promise, ms: number): Promise { + let timer: ReturnType | undefined; + const overrun = new Promise((resolve) => { + timer = setTimeout(() => resolve("overrun"), ms); + }); + try { + return await Promise.race([ + running.then( + (): Outcome => "settled", + (): Outcome => "settled" + ), + overrun, + ]); + } finally { + clearTimeout(timer); + } +} + +export function createCaseDrain(opts: { + /** Prefix for refusals, e.g. "Adoption fixture". */ + readonly label: string; + /** Time an overrunning body gets to settle before its state stays reserved. */ + readonly drainMs: number; +}): CaseDrain { + let reservation: { readonly name: string } | undefined; + const bunTimeoutMs = (budgetMs: number) => + budgetMs + opts.drainMs + CASE_DRAIN_BUN_MARGIN_MS; + const requireUnreserved = () => { + if (reservation !== undefined) { + throw new Error( + `E_TEST_CASE_RESERVED: ${opts.label} case "${reservation.name}" still runs past its budget and drain, so this case did not start; cases start again once it settles.` + ); + } + }; + const run = async ( + name: string, + body: () => void | Promise, + budgetMs: number + ): Promise => { + requireUnreserved(); + const running = (async () => { + await body(); + })(); + if ((await within(running, budgetMs)) === "settled") { + await running; + return; + } + const drained = await within(running, opts.drainMs); + if (drained === "overrun") { + const held = { name }; + reservation = held; + running + .catch(() => undefined) + .finally(() => { + if (reservation === held) { + reservation = undefined; + } + }); + } + throw new Error( + drained === "settled" + ? `E_TEST_CASE_BUDGET: ${opts.label} case exceeded its ${budgetMs} ms budget; its body settled within the ${opts.drainMs} ms drain, so later cases start normally.` + : `E_TEST_CASE_BUDGET: ${opts.label} case exceeded its ${budgetMs} ms budget and its ${opts.drainMs} ms drain; its state stays reserved until its body settles.` + ); + }; + return { + test: (name, body, budgetMs) => { + bunTest(name, () => run(name, body, budgetMs), bunTimeoutMs(budgetMs)); + }, + run, + bunTimeoutMs, + reservedBy: () => reservation?.name, + requireUnreserved, + }; +} diff --git a/tests/native-compose-adoption-generation.test.ts b/tests/native-compose-adoption-generation.test.ts index c77ee3b15..5faeec446 100644 --- a/tests/native-compose-adoption-generation.test.ts +++ b/tests/native-compose-adoption-generation.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, - test as boundedTest, + test as bunTest, expect, spyOn, } from "bun:test"; @@ -67,30 +67,88 @@ import { import { buildRuntimeHostMetadataOverride } from "../src/lib/runtime-host-metadata.ts"; import { captureAdoptionDependencyFirstPrepare } from "./e2e/scenarios/native-compose-adoption-dependency-staged-read.ts"; import { completedJobFixtureSources } from "./e2e/scenarios/native-compose-adoption-job-inputs.ts"; -import { createCompletedJobFixtureSettlement } from "./e2e/scenarios/native-compose-adoption-job-worktrees.ts"; +import { + CASE_DRAIN_BUN_MARGIN_MS, + createCaseDrain, +} from "./helpers/case-drain.ts"; import { restoreEnv } from "./helpers/env.ts"; import { managedEnvCompilerFixture } from "./helpers/managed-env-compiler.ts"; import { privateCliEnv } from "./helpers/private-cli-env.ts"; -import { testBudgetMs } from "./helpers/test-budget.ts"; +import { + CASE_TIMEOUT_CEILING_MS, + testBudgetMs, + withinCaseCeilingMs, +} from "./helpers/test-budget.ts"; /* * Harness budgets. Every engine probe below spawns the Bun stub engine, and one * retained dispatch issues several hundred probes, so an owner operation's wall - * time follows host load. Case timeouts and the caller deadlines of cases that - * are not about the deadline derive from `testBudgetMs`. The expiry cases' + * time follows host load. Case budgets and the caller deadlines of cases that + * are not about the deadline derive from `testBudgetMs`, within the case + * ceiling that the hosted shard's per-file limit allows. The expiry cases' * promptness assertion (1.5 s past the deadline) is a product bound under test * and stays literal. */ /** Caller deadline for one owner operation in a case that is not about the deadline. */ -const OPERATION_DEADLINE_MS = testBudgetMs(20_000); -const test = (name: string, run: () => Promise) => - boundedTest(name, run, testBudgetMs(20_000)); +const OPERATION_DEADLINE_MS = testBudgetMs(4000); +/** + * Time a case body that overran its budget gets to settle. Such a body is + * normally inside one owner operation, which ends by its own caller deadline. + */ +const CASE_DRAIN_MS = OPERATION_DEADLINE_MS + testBudgetMs(250); +/** + * Every case shares this file's roots, stub engine and environment, and Bun's + * timeout does not cancel a body, so cases run under the drain guard: an + * overrun fails only that case, by name (see `tests/helpers/case-drain.ts`). + */ +const cases = createCaseDrain({ + label: "Adoption fixture", + drainMs: CASE_DRAIN_MS, +}); +/** Largest body budget whose Bun timeout (budget, drain and margin) fits the case ceiling. */ +const CASE_BODY_CEILING_MS = + CASE_TIMEOUT_CEILING_MS - CASE_DRAIN_MS - CASE_DRAIN_BUN_MARGIN_MS; +withinCaseCeilingMs(cases.bunTimeoutMs(CASE_BODY_CEILING_MS)); +/** Body budget for a case that needs `unloadedMs` on an unloaded host. */ +function caseBudgetMs(unloadedMs: number): number { + return testBudgetMs(unloadedMs, { ceilingMs: CASE_BODY_CEILING_MS }); +} +type CaseBody = () => void | Promise; +/** Bun's `test`, `test.each` and `test.skipIf`, each under the drain guard. */ +const boundedTest = Object.assign( + (name: string, body: CaseBody, budgetMs: number) => + cases.test(name, body, budgetMs), + { + each: + (values: readonly T[]) => + ( + name: string, + body: (value: T) => void | Promise, + budgetMs: number + ) => + bunTest.each(values as T[])( + name, + (value: T) => cases.run(name, () => body(value), budgetMs), + cases.bunTimeoutMs(budgetMs) + ), + skipIf: + (condition: boolean) => + (name: string, body: CaseBody, budgetMs: number) => + bunTest.skipIf(condition)( + name, + () => cases.run(name, body, budgetMs), + cases.bunTimeoutMs(budgetMs) + ), + } +); +const test = (name: string, run: CaseBody) => + boundedTest(name, run, caseBudgetMs(20_000)); /** * Probe-phase bound for stores these cases open. One store probe carries a * single aggregate clock across a whole inspection (dozens of stub-engine * processes), 15 s by default. That default is a product bound no case here * tests, so stores open with the largest phase bound the product accepts. - * Public dispatch opens its own store and keeps the default (see the PR gaps). + * Public dispatch opens its own store and keeps the default (HACK-1458). */ const PROBE_PHASE_BUDGET_MS = 60_000; /** @@ -100,9 +158,36 @@ const PROBE_PHASE_BUDGET_MS = 60_000; * within 1.5 s of the deadline) does not depend on its length, but the work * before the callback must finish first: a 5 s deadline expired before the * callback at load 30-45. It stays under the probe-phase bound, so the caller - * deadline, not a probe's own phase clock, ends a held probe. + * deadline, not a probe's own phase clock, ends a held probe. The cases do not + * wait it out: once the callback is reached they move the product's clock to + * just before the deadline (`shiftProductClock`). */ const EXPIRY_DEADLINE_MS = PROBE_PHASE_BUDGET_MS - 15_000; +/** + * Clock time an expiry case leaves before the deadline when its callback runs. + * It must cover the work before the held probe and the stub engine recording + * its PID, which the case then proves was killed. For per-effect freshness the + * callback's own probe is the held one (0.3 s from the callback at 1-min load + * 45 on the M5); for the final observation the product first runs its other + * final probes (5.7 s at the same load). + */ +const EXPIRY_LEAD_MS = { + "per-effect freshness": testBudgetMs(250), + "final observation": testBudgetMs(1000), +} as const; +/** + * Move the clock the product reads (`Date.now`) forward by `byMs` for the rest + * of the case; it keeps advancing at the real rate. The product takes the + * caller deadline as a `Date.now` timestamp and has no clock seam, so this + * stands in for one. Timers are not moved: a held probe still ends by the + * timeout the product derives from the remaining clock time. Returns the + * restore function; call it in `finally`. + */ +function shiftProductClock(byMs: number): () => void { + const realNow = Date.now.bind(Date); + const clock = spyOn(Date, "now").mockImplementation(() => realNow() + byMs); + return () => clock.mockRestore(); +} /** The product store opener, with the harness probe-phase bound unless a case names one. */ function openLegacyComposeAdoptedGenerationStore( input: Parameters[0] @@ -184,22 +269,23 @@ type Fixture = { }; let root: string; let projectRoot: string; -let priorPath: string | undefined; -let priorCI: string | undefined; -let priorExecutionMode: string | undefined; -let priorRuntimeBackend: string | undefined; -let priorDispatchDeadline: string | undefined; let fixture: Fixture; -let publicStatic7Commands = 0; -const publicStatic7Settlement = createCompletedJobFixtureSettlement(); +/** + * Environment the cases change, as it was when the file loaded. Every case + * restores these values, so a case that settles late cannot leave its own + * values behind as a later case's "prior" ones. + */ +const FILE_ENV = Object.freeze({ + PATH: process.env.PATH, + CI: process.env.CI, + HACK_EXECUTION_MODE: process.env.HACK_EXECUTION_MODE, + HACK_RUNTIME_BACKEND: process.env.HACK_RUNTIME_BACKEND, + [COMPOSE_STARTUP_TIMEOUT_ENV]: process.env[COMPOSE_STARTUP_TIMEOUT_ENV], +}); beforeEach(async () => { - // An outer test timeout does not cancel an awaited owner. Never recycle its globals or roots. - publicStatic7Settlement.assertConfirmed(); - priorPath = process.env.PATH; - priorCI = process.env.CI; - priorExecutionMode = process.env.HACK_EXECUTION_MODE; - priorRuntimeBackend = process.env.HACK_RUNTIME_BACKEND; - priorDispatchDeadline = process.env[COMPOSE_STARTUP_TIMEOUT_ENV]; + // A body still running past its budget and drain owns the shared roots, + // stub engine and environment. Do not recycle them under it. + cases.requireUnreserved(); process.env.HACK_RUNTIME_BACKEND = "compose"; // Public dispatch (`tryLegacyComposeAdoptedCommand`) takes its caller // deadline from the Compose startup budget, 90 s by default. No case here @@ -354,20 +440,16 @@ if (fixture.mode === "inventory-change" && kind === "container" && action === "l await save(); }); afterEach(async () => { - if (publicStatic7Commands !== 0) { - publicStatic7Settlement.markUnconfirmed(); - } - try { - publicStatic7Settlement.assertConfirmed(); - } catch { - // Retain the still-owned fixture; a late callback cannot restore teardown authority. + if (cases.reservedBy() !== undefined) { + // The body still runs and owns this root and environment; keep both. + console.error( + "Adoption fixture root retained for a case still running past its budget and drain." + ); return; } - restoreEnv("PATH", priorPath); - restoreEnv("CI", priorCI); - restoreEnv("HACK_EXECUTION_MODE", priorExecutionMode); - restoreEnv("HACK_RUNTIME_BACKEND", priorRuntimeBackend); - restoreEnv(COMPOSE_STARTUP_TIMEOUT_ENV, priorDispatchDeadline); + for (const [key, value] of Object.entries(FILE_ENV)) { + restoreEnv(key, value); + } await rm(root, { recursive: true, force: true }); }); async function save() { @@ -653,15 +735,6 @@ else { return binary; } -async function publicStatic7Case(run: () => Promise) { - publicStatic7Commands += 1; - try { - await run(); - } finally { - publicStatic7Commands -= 1; - } -} - function publicStatic7Stages() { const started = Date.now(); return ( @@ -675,29 +748,61 @@ function publicStatic7Stages() { } test("retained Compose fixture admits its explicit backend and preserves opposite-backend refusal before probes", async () => { - await publicStatic7Case(async () => { - expect(process.env.HACK_RUNTIME_BACKEND).toBe("compose"); - const binary = await compiler(); + expect(process.env.HACK_RUNTIME_BACKEND).toBe("compose"); + const binary = await compiler(); + const { store, generation } = await prepared(binary); + const priorCompiler = process.env.HACK_CONFIG_COMPILER_BINARY; + try { + process.env.HACK_CONFIG_COMPILER_BINARY = binary; + await store.publish({ generation, binary }); + const commands = await readFile(join(root, "commands"), "utf8"); + process.env.HACK_RUNTIME_BACKEND = "native"; + try { + await expect( + tryLegacyComposeAdoptedCommand({ + cwd: projectRoot, + operation: "up", + detach: true, + }) + ).rejects.toMatchObject({ code: "E_NATIVE_PROJECT_UNSUPPORTED" }); + expect(await readFile(join(root, "commands"), "utf8")).toBe(commands); + expect((await readReceipt()).pendingOperation).toBeNull(); + } finally { + process.env.HACK_RUNTIME_BACKEND = "compose"; + } + expect( + await tryLegacyComposeAdoptedCommand({ + cwd: projectRoot, + operation: "up", + detach: true, + }) + ).toBe(0); + expect(await mutationCommands()).toEqual([["container", "start", ID]]); + } finally { + restoreEnv("HACK_CONFIG_COMPILER_BINARY", priorCompiler); + await store.close(); + } +}); + +boundedTest.each(["no", ""] as const)( + "public retained up dispatches the maintained static7 family with disabled policy %j", + async (jobRestartPolicy) => { + const stages = publicStatic7Stages(); + stages("prepare", "enter"); + const { app, job } = await maintainedCompletedJobFixture(); + const binary = await maintainedJobCompiler(); const { store, generation } = await prepared(binary); + stages("prepare", "settled"); const priorCompiler = process.env.HACK_CONFIG_COMPILER_BINARY; try { process.env.HACK_CONFIG_COMPILER_BINARY = binary; + stages("publish", "enter"); await store.publish({ generation, binary }); - const commands = await readFile(join(root, "commands"), "utf8"); - process.env.HACK_RUNTIME_BACKEND = "native"; - try { - await expect( - tryLegacyComposeAdoptedCommand({ - cwd: projectRoot, - operation: "up", - detach: true, - }) - ).rejects.toMatchObject({ code: "E_NATIVE_PROJECT_UNSUPPORTED" }); - expect(await readFile(join(root, "commands"), "utf8")).toBe(commands); - expect((await readReceipt()).pendingOperation).toBeNull(); - } finally { - process.env.HACK_RUNTIME_BACKEND = "compose"; - } + stages("publish", "settled"); + fixture.jobRestartPolicy = jobRestartPolicy; + await save(); + expect(generation.report.adoption_generation_version).toBe(7); + stages("start", "enter"); expect( await tryLegacyComposeAdoptedCommand({ cwd: projectRoot, @@ -705,69 +810,33 @@ test("retained Compose fixture admits its explicit backend and preserves opposit detach: true, }) ).toBe(0); - expect(await mutationCommands()).toEqual([["container", "start", ID]]); + stages("start", "settled"); + expect(await mutationCommands()).toEqual([ + ["container", "start", ID], + ["container", "start", job], + ["container", "start", app], + ]); + expect((await readReceipt()).pendingOperation).toBeNull(); + stages("stop", "enter"); + expect( + await tryLegacyComposeAdoptedCommand({ + cwd: projectRoot, + operation: "down", + }) + ).toBe(0); + stages("stop", "settled"); + expect((await mutationCommands()).slice(-3)).toEqual([ + ["container", "stop", app], + ["container", "stop", job], + ["container", "stop", ID], + ]); } finally { restoreEnv("HACK_CONFIG_COMPILER_BINARY", priorCompiler); await store.close(); } - }); -}); - -boundedTest.each(["no", ""] as const)( - "public retained up dispatches the maintained static7 family with disabled policy %j", - async (jobRestartPolicy) => { - await publicStatic7Case(async () => { - const stages = publicStatic7Stages(); - stages("prepare", "enter"); - const { app, job } = await maintainedCompletedJobFixture(); - const binary = await maintainedJobCompiler(); - const { store, generation } = await prepared(binary); - stages("prepare", "settled"); - const priorCompiler = process.env.HACK_CONFIG_COMPILER_BINARY; - try { - process.env.HACK_CONFIG_COMPILER_BINARY = binary; - stages("publish", "enter"); - await store.publish({ generation, binary }); - stages("publish", "settled"); - fixture.jobRestartPolicy = jobRestartPolicy; - await save(); - expect(generation.report.adoption_generation_version).toBe(7); - stages("start", "enter"); - expect( - await tryLegacyComposeAdoptedCommand({ - cwd: projectRoot, - operation: "up", - detach: true, - }) - ).toBe(0); - stages("start", "settled"); - expect(await mutationCommands()).toEqual([ - ["container", "start", ID], - ["container", "start", job], - ["container", "start", app], - ]); - expect((await readReceipt()).pendingOperation).toBeNull(); - stages("stop", "enter"); - expect( - await tryLegacyComposeAdoptedCommand({ - cwd: projectRoot, - operation: "down", - }) - ).toBe(0); - stages("stop", "settled"); - expect((await mutationCommands()).slice(-3)).toEqual([ - ["container", "stop", app], - ["container", "stop", job], - ["container", "stop", ID], - ]); - } finally { - restoreEnv("HACK_CONFIG_COMPILER_BINARY", priorCompiler); - await store.close(); - } - }); }, // Cover prepare/publish and separate bounded up/down operations without changing their deadlines. - testBudgetMs(60_000) + caseBudgetMs(60_000) ); boundedTest.each([ @@ -779,110 +848,106 @@ boundedTest.each([ ] as const)( "public static7 observer refuses %s with only closed diagnostics and keeps pending", async (reason) => { - await publicStatic7Case(async () => { - await maintainedCompletedJobFixture(); - const binary = await maintainedJobCompiler(); - const { store, generation } = await prepared(binary); - const priorCompiler = process.env.HACK_CONFIG_COMPILER_BINARY; + await maintainedCompletedJobFixture(); + const binary = await maintainedJobCompiler(); + const { store, generation } = await prepared(binary); + const priorCompiler = process.env.HACK_CONFIG_COMPILER_BINARY; + try { + process.env.HACK_CONFIG_COMPILER_BINARY = binary; + await store.publish({ generation, binary }); + fixture.jobObservation = reason; + await save(); + let rejection: unknown; try { - process.env.HACK_CONFIG_COMPILER_BINARY = binary; - await store.publish({ generation, binary }); - fixture.jobObservation = reason; - await save(); - let rejection: unknown; - try { - await tryLegacyComposeAdoptedCommand({ - cwd: projectRoot, - operation: "up", - detach: true, - }); - } catch (error: unknown) { - rejection = error; - } - expect(rejection).toBeInstanceOf(HackCliError); - if (!(rejection instanceof HackCliError)) { - throw new Error("missing public refusal"); - } - expect(rejection.code).toBe("E_CONFIG_INVALID"); - expect(rejection.detail).toEqual({ - legacy_adoption_refusal: { - stage: "ordered-observation", - reason: reason === "probe" ? "probe-child" : reason, - }, + await tryLegacyComposeAdoptedCommand({ + cwd: projectRoot, + operation: "up", + detach: true, }); - expect( - JSON.stringify({ - message: rejection.message, - detail: rejection.detail, - }) - ).not.toContain(CANARY); - expect((await readReceipt()).pendingOperation.operation).toBe("start"); - expect( - (await mutationCommands()).some( - (args) => args[2] === "e".repeat(64) || args[2] === "c".repeat(64) - ) - ).toBe(false); - } finally { - restoreEnv("HACK_CONFIG_COMPILER_BINARY", priorCompiler); - await store.close(); + } catch (error: unknown) { + rejection = error; } - }); + expect(rejection).toBeInstanceOf(HackCliError); + if (!(rejection instanceof HackCliError)) { + throw new Error("missing public refusal"); + } + expect(rejection.code).toBe("E_CONFIG_INVALID"); + expect(rejection.detail).toEqual({ + legacy_adoption_refusal: { + stage: "ordered-observation", + reason: reason === "probe" ? "probe-child" : reason, + }, + }); + expect( + JSON.stringify({ + message: rejection.message, + detail: rejection.detail, + }) + ).not.toContain(CANARY); + expect((await readReceipt()).pendingOperation.operation).toBe("start"); + expect( + (await mutationCommands()).some( + (args) => args[2] === "e".repeat(64) || args[2] === "c".repeat(64) + ) + ).toBe(false); + } finally { + restoreEnv("HACK_CONFIG_COMPILER_BINARY", priorCompiler); + await store.close(); + } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( "public static7 scheduler classifies actual nonzero job exit without starting its dependent", async () => { - await publicStatic7Case(async () => { - const { job, app } = await maintainedCompletedJobFixture(); - const binary = await maintainedJobCompiler(); - const { store, generation } = await prepared(binary); - const priorCompiler = process.env.HACK_CONFIG_COMPILER_BINARY; + const { job, app } = await maintainedCompletedJobFixture(); + const binary = await maintainedJobCompiler(); + const { store, generation } = await prepared(binary); + const priorCompiler = process.env.HACK_CONFIG_COMPILER_BINARY; + try { + process.env.HACK_CONFIG_COMPILER_BINARY = binary; + await store.publish({ generation, binary }); + const jobState = fixture.jobs?.[job]; + if (!jobState) { + throw new Error("missing job state"); + } + jobState.exitCode = 17; + await save(); + let rejection: unknown; try { - process.env.HACK_CONFIG_COMPILER_BINARY = binary; - await store.publish({ generation, binary }); - const jobState = fixture.jobs?.[job]; - if (!jobState) { - throw new Error("missing job state"); - } - jobState.exitCode = 17; - await save(); - let rejection: unknown; - try { - await tryLegacyComposeAdoptedCommand({ - cwd: projectRoot, - operation: "up", - detach: true, - }); - } catch (error: unknown) { - rejection = error; - } - expect(rejection).toBeInstanceOf(HackCliError); - if (!(rejection instanceof HackCliError)) { - throw new Error("missing public refusal"); - } - expect(rejection.detail).toEqual({ - legacy_adoption_refusal: { - stage: "ordered-scheduler", - reason: "job-failed", - }, + await tryLegacyComposeAdoptedCommand({ + cwd: projectRoot, + operation: "up", + detach: true, }); - expect(await mutationCommands()).toEqual([ - ["container", "start", ID], - ["container", "start", job], - ]); - expect((await mutationCommands()).some((args) => args[2] === app)).toBe( - false - ); - expect((await readReceipt()).pendingOperation.operation).toBe("start"); - } finally { - restoreEnv("HACK_CONFIG_COMPILER_BINARY", priorCompiler); - await store.close(); + } catch (error: unknown) { + rejection = error; } - }); + expect(rejection).toBeInstanceOf(HackCliError); + if (!(rejection instanceof HackCliError)) { + throw new Error("missing public refusal"); + } + expect(rejection.detail).toEqual({ + legacy_adoption_refusal: { + stage: "ordered-scheduler", + reason: "job-failed", + }, + }); + expect(await mutationCommands()).toEqual([ + ["container", "start", ID], + ["container", "start", job], + ]); + expect((await mutationCommands()).some((args) => args[2] === app)).toBe( + false + ); + expect((await readReceipt()).pendingOperation.operation).toBe("start"); + } finally { + restoreEnv("HACK_CONFIG_COMPILER_BINARY", priorCompiler); + await store.close(); + } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest.each(["start", "restart", "stop"] as const)( @@ -960,7 +1025,7 @@ boundedTest.each(["start", "restart", "stop"] as const)( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -1015,7 +1080,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -1058,7 +1123,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -1112,7 +1177,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -1153,7 +1218,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -1224,7 +1289,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -1271,7 +1336,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -1298,7 +1363,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -1341,7 +1406,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -1349,6 +1414,7 @@ boundedTest( async () => { await dependencyFixture(); const { store, generation } = await prepared(); + let restoreClock: () => void = () => undefined; try { const binary = await compiler(); await store.publish({ generation, binary }); @@ -1361,7 +1427,11 @@ boundedTest( services: [], deadline, run: async () => { - await Bun.sleep(Math.max(1, deadline - Date.now() + 5)); + // The callback overruns the deadline: move the clock past it + // instead of sleeping until it. + restoreClock = shiftProductClock( + Math.max(0, deadline - Date.now() + 5) + ); return 0; }, }), @@ -1370,10 +1440,11 @@ boundedTest( ); expect((await readReceipt()).pendingOperation.operation).toBe("stop"); } finally { + restoreClock(); await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -1414,7 +1485,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); for (const linked of [false, true]) { @@ -1498,7 +1569,7 @@ for (const linked of [false, true]) { stage("closed"); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); } @@ -1508,6 +1579,7 @@ for (const stage of ["per-effect freshness", "final observation"] as const) { async () => { await dependencyFixture(); const { store, generation } = await prepared(); + let restoreClock: () => void = () => undefined; try { const binary = await compiler(); await store.publish({ generation, binary }); @@ -1525,6 +1597,11 @@ for (const stage of ["per-effect freshness", "final observation"] as const) { fixture.mode = stage === "per-effect freshness" ? "hang" : "hang-state"; await save(); + // Leave only the lead before the deadline, rather than waiting + // out the whole deadline. Never move the clock back. + restoreClock = shiftProductClock( + Math.max(0, deadline - Date.now() - EXPIRY_LEAD_MS[stage]) + ); if (stage === "per-effect freshness") { await input.assertFresh(); } @@ -1550,10 +1627,11 @@ for (const stage of ["per-effect freshness", "final observation"] as const) { expect(await mutationCommands()).toEqual([]); expect(fixture.volume[0]?.createdAt).toBe(CREATED); } finally { + restoreClock(); await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); } @@ -1578,7 +1656,7 @@ boundedTest( expect(await mutationCommands()).toEqual([]); expect(fixture.volume[0]?.createdAt).toBe(CREATED); }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -1611,7 +1689,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -1672,7 +1750,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -1725,7 +1803,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -1775,7 +1853,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -1811,7 +1889,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -1851,7 +1929,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -1918,7 +1996,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest.skipIf(!process.env.HACK_TEST_OLD_ADOPTION_BINARY)( @@ -1962,7 +2040,7 @@ boundedTest.skipIf(!process.env.HACK_TEST_OLD_ADOPTION_BINARY)( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); /** Current writers keep references at rest; "literal" models a file an earlier @@ -2030,7 +2108,7 @@ async function trackedManagedCheckout() { ]) { const child = Bun.spawn(["/usr/bin/git", "-C", projectRoot, ...args], { env: { - PATH: priorPath ?? "/usr/bin:/bin", + PATH: FILE_ENV.PATH ?? "/usr/bin:/bin", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", }, @@ -2318,7 +2396,7 @@ boundedTest( "active" ); }, - testBudgetMs(90_000) + caseBudgetMs(90_000) ); boundedTest( @@ -2482,7 +2560,7 @@ boundedTest( }) ).toMatchObject({ removed: true, adoptionBound: false }); }, - testBudgetMs(90_000) + caseBudgetMs(90_000) ); test("v3 value-free env override binds the engine config hash computed with the values (HACK-1273)", async () => { @@ -2870,7 +2948,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(240_000) + caseBudgetMs(240_000) ); boundedTest( @@ -2933,7 +3011,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(60_000) + caseBudgetMs(60_000) ); async function artifactPath(name: string) { const receipt = JSON.parse( @@ -2969,7 +3047,7 @@ async function linkedCheckout() { const git = async (args: readonly string[]) => { const child = Bun.spawn(["/usr/bin/git", "-C", projectRoot, ...args], { env: { - PATH: priorPath ?? "/usr/bin:/bin", + PATH: FILE_ENV.PATH ?? "/usr/bin:/bin", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", }, @@ -3109,7 +3187,7 @@ test("v13 linked default pins the physical branch originals and explicit selecti stdout: "ignore", stderr: "ignore", env: { - PATH: priorPath ?? "/usr/bin:/bin", + PATH: FILE_ENV.PATH ?? "/usr/bin:/bin", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", }, @@ -3538,7 +3616,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); test("v10 saved bridge and health owner rejects other contract receipts", async () => { @@ -3626,7 +3704,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); boundedTest( @@ -3661,7 +3739,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); test("rolled-back v10 can reprepare the same owned bridge without health under v6", async () => { @@ -3831,7 +3909,7 @@ boundedTest( await store.close(); } }, - testBudgetMs(30_000) + caseBudgetMs(30_000) ); test("v6 rollback permits a later plain owner without inheriting topology authority", async () => { From b18202cfe8096527e53d875047b2bf554e2614a9 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Sat, 24 Oct 2026 18:03:08 -0400 Subject: [PATCH 10/10] test(native): wait for a slow overrunning adoption case before the next one A loaded run (1-min load 100-120) showed a real case body outlasting its 150 s budget and 85 s drain because several owner operations were still ahead of it. The next 92 cases then refused at once, so the reservation never had time to clear. The next case's beforeEach now waits for such a body (one more case ceiling, outside that case's own timeout) and starts normally once it settles. Only a body that outlasts that wait too makes later cases refuse, without waiting again. Refs HACK-1367 Co-Authored-By: Claude Opus 5.5 --- tests/case-drain.test.ts | 50 +++++++++++++- tests/helpers/case-drain.ts | 66 +++++++++++++------ ...native-compose-adoption-generation.test.ts | 19 ++++-- 3 files changed, 109 insertions(+), 26 deletions(-) diff --git a/tests/case-drain.test.ts b/tests/case-drain.test.ts index 92b5f2648..d704fcb90 100644 --- a/tests/case-drain.test.ts +++ b/tests/case-drain.test.ts @@ -41,7 +41,7 @@ test("a case still running past its drain reserves shared state only until its b await expect( cases.run("hung", () => release.promise, BUDGET_MS) ).rejects.toThrow( - /^E_TEST_CASE_BUDGET: Fixture case exceeded .*stays reserved/ + /^E_TEST_CASE_BUDGET: Fixture case exceeded .*the next case waits/ ); expect(cases.reservedBy()).toBe("hung"); expect(() => cases.requireUnreserved()).toThrow( @@ -102,3 +102,51 @@ test("the Bun timeout covers the budget, the drain and the margin", () => { BUDGET_MS + DRAIN_MS + CASE_DRAIN_BUN_MARGIN_MS ); }); + +test("the next case waits for a reserving body and starts once it settles", async () => { + const cases = createCaseDrain({ label: "Fixture", drainMs: DRAIN_MS }); + const release = Promise.withResolvers(); + await expect( + cases.run("slow", () => release.promise, BUDGET_MS) + ).rejects.toThrow(/^E_TEST_CASE_BUDGET/); + setTimeout(() => release.resolve(), 50); + await cases.awaitUnreserved(DRAIN_MS); + expect(cases.reservedBy()).toBeUndefined(); + let ran = false; + await cases.run( + "next", + () => { + ran = true; + }, + BUDGET_MS + ); + expect(ran).toBe(true); +}); + +test("after one wait runs out, later cases refuse at once until the body settles", async () => { + const cases = createCaseDrain({ label: "Fixture", drainMs: DRAIN_MS }); + const release = Promise.withResolvers(); + await expect( + cases.run("hung", () => release.promise, BUDGET_MS) + ).rejects.toThrow(/^E_TEST_CASE_BUDGET/); + const started = performance.now(); + await expect(cases.awaitUnreserved(BUDGET_MS)).rejects.toThrow( + /^E_TEST_CASE_RESERVED: Fixture case "hung"/ + ); + expect(performance.now() - started).toBeGreaterThanOrEqual(BUDGET_MS - 5); + const again = performance.now(); + await expect(cases.awaitUnreserved(DRAIN_MS)).rejects.toThrow( + /^E_TEST_CASE_RESERVED/ + ); + expect(performance.now() - again).toBeLessThan(BUDGET_MS); + release.resolve(); + await Bun.sleep(0); + await cases.awaitUnreserved(DRAIN_MS); + expect(cases.reservedBy()).toBeUndefined(); +}); + +test("awaiting with no reservation returns at once", async () => { + const cases = createCaseDrain({ label: "Fixture", drainMs: DRAIN_MS }); + await cases.awaitUnreserved(DRAIN_MS); + expect(cases.reservedBy()).toBeUndefined(); +}); diff --git a/tests/helpers/case-drain.ts b/tests/helpers/case-drain.ts index 2cb652d41..e7c8b914a 100644 --- a/tests/helpers/case-drain.ts +++ b/tests/helpers/case-drain.ts @@ -10,17 +10,17 @@ import { test as bunTest } from "bun:test"; * used to latch after the first abandoned case and fail every later case. * * This guard runs each body under its own budget instead, so an overrun fails - * that case by name. The overrunning body then gets `drainMs` (inside the same - * case's Bun timeout) to settle; bodies normally end within it, because each - * owner operation they await has its own caller deadline below `drainMs`. + * that case by name, once. The overrunning body then gets `drainMs`, inside + * the same case's Bun timeout, to settle. * - * - Settled: the case still fails once with its budget refusal, and the next - * case starts normally. - * - Still running: the shared state stays reserved for that body. `reservedBy` - * names it, so the file's `beforeEach` refuses with a closed reason instead - * of reusing state the body may still touch, and `afterEach` keeps its roots. - * The reservation clears by itself when the body settles, and later cases - * run normally again. + * - Settled within the drain: the next case starts normally. + * - Still running: the shared state stays reserved for that body, and + * `afterEach` keeps its roots. The next case's `beforeEach` calls + * `awaitUnreserved`, which waits (outside that case's own timeout) for the + * body to settle; a slow body on a loaded host usually does, and the case + * then starts normally. Only a body that outlasts that wait too makes later + * cases refuse with a closed reason naming it, without waiting again, until + * it settles. */ export type CaseDrain = { /** Register a case: `budgetMs` bounds its body; the Bun timeout adds the drain. */ @@ -39,6 +39,12 @@ export type CaseDrain = { readonly bunTimeoutMs: (budgetMs: number) => number; /** The case whose body still runs past its budget and drain, if any. */ readonly reservedBy: () => string | undefined; + /** + * Wait up to `waitMs` for a reserving body to settle, then refuse with a + * closed reason. After one wait has run out, refuse at once until the body + * settles. + */ + readonly awaitUnreserved: (waitMs: number) => Promise; /** Refuse, with a closed reason, while an earlier case's body still runs. */ readonly requireUnreserved: () => void; }; @@ -66,20 +72,39 @@ async function within(running: Promise, ms: number): Promise { } } +type Reservation = { + readonly name: string; + /** Resolves once the body settled and this reservation was cleared. */ + readonly released: Promise; + waitedOut: boolean; +}; + export function createCaseDrain(opts: { /** Prefix for refusals, e.g. "Adoption fixture". */ readonly label: string; - /** Time an overrunning body gets to settle before its state stays reserved. */ + /** Time an overrunning body gets to settle inside its own case. */ readonly drainMs: number; }): CaseDrain { - let reservation: { readonly name: string } | undefined; + let reservation: Reservation | undefined; const bunTimeoutMs = (budgetMs: number) => budgetMs + opts.drainMs + CASE_DRAIN_BUN_MARGIN_MS; + const refusal = (held: Reservation) => + new Error( + `E_TEST_CASE_RESERVED: ${opts.label} case "${held.name}" still runs past its budget, drain and wait, so this case did not start; cases start again once it settles.` + ); const requireUnreserved = () => { if (reservation !== undefined) { - throw new Error( - `E_TEST_CASE_RESERVED: ${opts.label} case "${reservation.name}" still runs past its budget and drain, so this case did not start; cases start again once it settles.` - ); + throw refusal(reservation); + } + }; + const awaitUnreserved = async (waitMs: number) => { + const held = reservation; + if (held === undefined) { + return; + } + if (held.waitedOut || (await within(held.released, waitMs)) === "overrun") { + held.waitedOut = true; + throw refusal(held); } }; const run = async ( @@ -97,20 +122,20 @@ export function createCaseDrain(opts: { } const drained = await within(running, opts.drainMs); if (drained === "overrun") { - const held = { name }; - reservation = held; - running + const released = running .catch(() => undefined) - .finally(() => { + .then(() => { if (reservation === held) { reservation = undefined; } }); + const held: Reservation = { name, released, waitedOut: false }; + reservation = held; } throw new Error( drained === "settled" ? `E_TEST_CASE_BUDGET: ${opts.label} case exceeded its ${budgetMs} ms budget; its body settled within the ${opts.drainMs} ms drain, so later cases start normally.` - : `E_TEST_CASE_BUDGET: ${opts.label} case exceeded its ${budgetMs} ms budget and its ${opts.drainMs} ms drain; its state stays reserved until its body settles.` + : `E_TEST_CASE_BUDGET: ${opts.label} case exceeded its ${budgetMs} ms budget and its ${opts.drainMs} ms drain; the next case waits for its body to settle.` ); }; return { @@ -120,6 +145,7 @@ export function createCaseDrain(opts: { run, bunTimeoutMs, reservedBy: () => reservation?.name, + awaitUnreserved, requireUnreserved, }; } diff --git a/tests/native-compose-adoption-generation.test.ts b/tests/native-compose-adoption-generation.test.ts index 5faeec446..2bf62c659 100644 --- a/tests/native-compose-adoption-generation.test.ts +++ b/tests/native-compose-adoption-generation.test.ts @@ -92,10 +92,19 @@ import { /** Caller deadline for one owner operation in a case that is not about the deadline. */ const OPERATION_DEADLINE_MS = testBudgetMs(4000); /** - * Time a case body that overran its budget gets to settle. Such a body is - * normally inside one owner operation, which ends by its own caller deadline. + * Time a case body that overran its budget gets to settle inside its own case: + * one caller deadline plus slack. A body with several owner operations left + * can need longer on a loaded host; the next case then waits for it. */ const CASE_DRAIN_MS = OPERATION_DEADLINE_MS + testBudgetMs(250); +/** + * How long the next case's `beforeEach` waits for a body still running after + * its budget and drain, outside that case's own timeout: one more case + * ceiling. The overrunning case has already failed by name by then. + */ +const CASE_RESERVATION_WAIT_MS = CASE_TIMEOUT_CEILING_MS; +/** `beforeEach` timeout: the reservation wait plus the fixture setup. */ +const BEFORE_EACH_TIMEOUT_MS = CASE_RESERVATION_WAIT_MS + testBudgetMs(500); /** * Every case shares this file's roots, stub engine and environment, and Bun's * timeout does not cancel a body, so cases run under the drain guard: an @@ -284,8 +293,8 @@ const FILE_ENV = Object.freeze({ }); beforeEach(async () => { // A body still running past its budget and drain owns the shared roots, - // stub engine and environment. Do not recycle them under it. - cases.requireUnreserved(); + // stub engine and environment. Wait for it rather than recycling them. + await cases.awaitUnreserved(CASE_RESERVATION_WAIT_MS); process.env.HACK_RUNTIME_BACKEND = "compose"; // Public dispatch (`tryLegacyComposeAdoptedCommand`) takes its caller // deadline from the Compose startup budget, 90 s by default. No case here @@ -438,7 +447,7 @@ if (fixture.mode === "inventory-change" && kind === "container" && action === "l process.env.PATH = root; await mkdir(join(projectRoot, ".git")); await save(); -}); +}, BEFORE_EACH_TIMEOUT_MS); afterEach(async () => { if (cases.reservedBy() !== undefined) { // The body still runs and owns this root and environment; keep both.