diff --git a/packages/runtime-core/src/lib.rs b/packages/runtime-core/src/lib.rs index c5fc6b8fb..07c33c9d0 100644 --- a/packages/runtime-core/src/lib.rs +++ b/packages/runtime-core/src/lib.rs @@ -19,8 +19,12 @@ pub const CANDIDATE_VERSION: &str = "5.0.0-dev.6"; mod error; mod error_stop_diagnostics; pub use error::CandidateError; +#[cfg(all(test, feature = "native-config-plan"))] +mod test_budget; #[cfg(test)] mod test_scratch; +#[cfg(all(test, feature = "native-config-plan"))] +pub(crate) use test_budget::TEST_RUN_BUDGET; #[cfg(test)] pub(crate) use test_scratch::next_test_scratch; diff --git a/packages/runtime-core/src/provider/graph/native/persistent_data/engine/tests.rs b/packages/runtime-core/src/provider/graph/native/persistent_data/engine/tests.rs index 208e8285f..009e63b53 100644 --- a/packages/runtime-core/src/provider/graph/native/persistent_data/engine/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/persistent_data/engine/tests.rs @@ -6,7 +6,6 @@ use std::{ os::unix::fs::{DirBuilderExt, MetadataExt}, path::PathBuf, rc::Rc, - time::Duration, }; struct Fixture { @@ -169,7 +168,7 @@ fn enroll(fixture: &Fixture, endpoint: &mut Endpoint<'_>) -> Result( state_root: &fixture.0, binding, intent: "dddddddddddddddddddddddddddddddd", - deadline: Instant::now() + Duration::from_secs(10), + deadline: Instant::now() + crate::TEST_RUN_BUDGET, cancelled, } } @@ -184,7 +184,7 @@ fn read(fixture: &Fixture, binding: &Binding, fake: &mut Fake) -> Result( state_root: &f.0, binding: b, intent: "f0000000000000000000000000000000", - deadline: Instant::now() + Duration::from_secs(5), + deadline: Instant::now() + crate::TEST_RUN_BUDGET, cancelled, }, witness: w, @@ -203,7 +203,7 @@ fn read_options<'a>(f: &'a Fixture, b: &'a Binding, c: &'a AtomicBool) -> ReadOp ReadOptions { state_root: &f.0, binding: b, - deadline: Instant::now() + Duration::from_secs(5), + deadline: Instant::now() + crate::TEST_RUN_BUDGET, cancelled: c, } } diff --git a/packages/runtime-core/src/provider/graph/native/runtime/exec_tests.rs b/packages/runtime-core/src/provider/graph/native/runtime/exec_tests.rs index fa0c98913..9d5e90115 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime/exec_tests.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime/exec_tests.rs @@ -76,7 +76,7 @@ fn selected() -> ExecSelection { } } fn ready(session: &mut Session<'_, Fake>, graph: &execution::Graph) { - execution::run(graph, session, Duration::from_secs(2)).unwrap(); + execution::run(graph, session, RUN_BUDGET).unwrap(); session .backend .state @@ -105,7 +105,7 @@ fn finite_exec_uses_exact_live_member_and_one_original_source_data_deadline() { data_refusal: false, }; let original = fs::read(session.root.join("state.json")).unwrap(); - let deadline = Instant::now() + Duration::from_secs(2); + let deadline = Instant::now() + RUN_BUDGET; let result = exec::execute_with(&backend, session.receipt.clone(), &selected(), deadline).unwrap(); assert_eq!(result.exit_code, 17); @@ -163,7 +163,7 @@ fn finite_exec_refuses_changed_member_or_data_before_command_and_withholds_resta + if mode == "deadline" { Duration::ZERO } else { - Duration::from_secs(2) + RUN_BUDGET }; assert!( exec::execute_with(&backend, session.receipt.clone(), &request, deadline).is_err(), diff --git a/packages/runtime-core/src/provider/graph/native/runtime/run_tests.rs b/packages/runtime-core/src/provider/graph/native/runtime/run_tests.rs index 8617fe9d2..d633a6a2b 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime/run_tests.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime/run_tests.rs @@ -152,7 +152,7 @@ fn selected() -> run::RunSelection { } } fn ready(session: &mut Session<'_, Fake>, graph: &execution::Graph) { - execution::run(graph, session, Duration::from_secs(2)).unwrap(); + execution::run(graph, session, RUN_BUDGET).unwrap(); let mut state = session.backend.state.borrow_mut(); let member = state.containers.values_mut().next().unwrap(); member["State"]["StartedAt"] = json!("2026-10-09T00:00:00.000000000Z"); @@ -204,7 +204,7 @@ fn finite_one_off_derives_the_exact_member_and_discards_its_record_after_proven_ one_offs: RefCell::new(BTreeMap::new()), }; let original = fs::read(session.root.join("state.json")).unwrap(); - let deadline = Instant::now() + Duration::from_secs(2); + let deadline = Instant::now() + RUN_BUDGET; let result = run::execute_with( &backend, session.receipt.clone(), @@ -290,7 +290,7 @@ fn finite_one_off_refuses_changed_member_mounts_or_data_before_create_and_retain + if mode == "deadline" { Duration::ZERO } else { - Duration::from_secs(2) + RUN_BUDGET }; assert!( run::execute_with( @@ -337,7 +337,7 @@ fn cleanup_reaps_a_recorded_one_off_before_stopping_members_and_refuses_foreign_ session.receipt.clone(), &session.root, &selected(), - Instant::now() + Duration::from_secs(2) + Instant::now() + RUN_BUDGET ) .is_err() ); diff --git a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs index 6d11b969e..26c4f7468 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs @@ -14,6 +14,9 @@ use std::{ const OWNER: &str = "cccccccccccccccccccccccccccccccc"; const BOOT: &str = "12345678-abcd-abcd-abcd-123456789abc"; const RUN: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +/// Harness budget for each fake-backend graph run, selection, exec, run or log read here +/// (see `crate::TEST_RUN_BUDGET`). +const RUN_BUDGET: Duration = crate::TEST_RUN_BUDGET; struct Fixture { root: PathBuf, project: PathBuf, @@ -59,7 +62,7 @@ impl Fixture { profiles: &[], explicit_overlay: None, metadata, - deadline: Instant::now() + Duration::from_secs(60), + deadline: Instant::now() + RUN_BUDGET, }, ) .unwrap() @@ -140,7 +143,7 @@ fn two_owned_bridges_keep_per_network_members_and_aliases_through_cleanup() { serde_json::to_value(&session.receipt).unwrap()["version"], 5 ); - execution::run(&graph, &mut session, Duration::from_secs(2)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); assert_eq!(session.receipt.phase, Phase::ReadyObserved); assert_eq!(session.backend.state.borrow().networks.len(), 2); snapshot(&session.backend, session.receipt.clone()).unwrap(); @@ -222,7 +225,7 @@ fn two_bridge_cancellation_before_second_create_retains_exact_first_bridge_for_c session.startup_guard = Some(&guard); session.backend.state.borrow_mut().cancel_after_network = Some(canceled.clone()); assert_eq!( - execution::run(&graph, &mut session, Duration::from_secs(2)) + execution::run(&graph, &mut session, RUN_BUDGET) .unwrap_err() .code, "native_graph_canceled" @@ -240,7 +243,7 @@ fn two_bridge_cleanup_retries_only_the_retained_second_bridge_after_first_delete let fixture = Fixture::new(two_bridges()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"db":{},"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(2)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); session .backend .state @@ -648,7 +651,7 @@ fn foreground_cancellation_after_network_creation_prevents_container_effects_and session.startup_guard = Some(&guard); session.backend.state.borrow_mut().cancel_after_network = Some(canceled.clone()); assert_eq!( - execution::run(&graph, &mut session, Duration::from_secs(1)) + execution::run(&graph, &mut session, RUN_BUDGET) .unwrap_err() .code, "native_graph_canceled" @@ -659,7 +662,7 @@ fn foreground_cancellation_after_network_creation_prevents_container_effects_and assert_eq!(retained.resources["network:default"].phase, "created"); assert!(retained.resources["network:default"].id.is_some()); canceled.set(false); - assert!(execution::run(&graph, &mut session, Duration::from_secs(1)).is_err()); + assert!(execution::run(&graph, &mut session, RUN_BUDGET).is_err()); assert_eq!(session.backend.state.borrow().effects, ["create:network"]); cleanup_using(&session.backend, &mut session.receipt, &session.root).unwrap(); assert_eq!(session.receipt.phase, Phase::Removed); @@ -1083,7 +1086,7 @@ fn live_source_controller_preserves_edits_and_cleans_only_original_owned_resourc let fixture = source_fixture(); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); assert_eq!( serde_json::to_value(&session.receipt).unwrap()["version"], 3 @@ -1135,7 +1138,7 @@ fn selected_source_replacement_before_effect_or_after_create_refuses_without_sta fs::rename(&path, fixture.project.join("original")).unwrap(); fs::DirBuilder::new().mode(0o700).create(&path).unwrap(); } - assert!(execution::run(&graph, &mut session, Duration::from_secs(5)).is_err()); + assert!(execution::run(&graph, &mut session, RUN_BUDGET).is_err()); assert!( !session .backend @@ -1155,7 +1158,7 @@ fn selected_source_replacement_before_effect_or_after_create_refuses_without_sta } else { assert!(session.backend.state.borrow().effects.is_empty()); } - assert!(execution::run(&graph, &mut session, Duration::from_secs(5)).is_err()); + assert!(execution::run(&graph, &mut session, RUN_BUDGET).is_err()); } } @@ -1165,7 +1168,7 @@ fn retained_source_bind_or_approved_share_drift_refuses_cleanup_before_mutation( let fixture = source_fixture(); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); if changed_share { session.backend.source.as_mut().unwrap().share.inode += 1; } else { @@ -1184,7 +1187,7 @@ fn source_cleanup_does_not_recapture_a_moved_or_deleted_host_project() { let fixture = source_fixture(); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let moved = fixture.root.join("moved-project"); if deleted { fs::remove_dir_all(&fixture.project).unwrap(); @@ -1215,7 +1218,7 @@ fn real_compiler_job_dependency_and_readiness_drive_durable_native_effects_and_c let fixture = Fixture::new(project); let prepared = fixture.prepared(json!({"web":{},"z.seed":{}}), &BTreeMap::new()); let (graph, mut session) = fixture.session(prepared); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); assert_eq!(session.receipt.phase, Phase::ReadyObserved); assert_eq!( session.backend.state.borrow().effects, @@ -1259,12 +1262,12 @@ fn create_and_start_uncertainty_are_retained_once_without_replay_or_adoption() { fake.fail_create = !fail_start; fake.fail_start = fail_start; } - assert!(execution::run(&graph, &mut session, Duration::from_secs(5)).is_err()); + assert!(execution::run(&graph, &mut session, RUN_BUDGET).is_err()); let (retained, _) = journal::load(&fixture.candidate, RUN, OWNER, BOOT).unwrap(); assert_eq!(retained.resources["container:web"].phase, "uncertain"); assert_eq!(retained.resources["container:web"].id.is_some(), fail_start); let count = session.backend.state.borrow().effects.len(); - assert!(execution::run(&graph, &mut session, Duration::from_secs(5)).is_err()); + assert!(execution::run(&graph, &mut session, RUN_BUDGET).is_err()); assert_eq!(session.backend.state.borrow().effects.len(), count); } } @@ -1277,12 +1280,12 @@ fn project_network_create_uncertainty_is_durable_and_never_replayed() { fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); session.backend.state.borrow_mut().fail_network_create = !applied; session.backend.state.borrow_mut().fail_network_after_apply = applied; - assert!(execution::run(&graph, &mut session, Duration::from_secs(5)).is_err()); + assert!(execution::run(&graph, &mut session, RUN_BUDGET).is_err()); let (retained, _) = journal::load(&fixture.candidate, RUN, OWNER, BOOT).unwrap(); assert_eq!(retained.resources["network:default"].phase, "create-intent"); assert!(retained.resources["network:default"].id.is_none()); assert_eq!(retained.resources["container:web"].phase, "reserved"); - assert!(execution::run(&graph, &mut session, Duration::from_secs(5)).is_err()); + assert!(execution::run(&graph, &mut session, RUN_BUDGET).is_err()); assert_eq!( session.backend.state.borrow().effects, vec!["create:network"] @@ -1303,7 +1306,7 @@ fn foreign_or_changed_project_networks_refuse_cleanup_before_any_stop_or_delete( let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let name = session.receipt.resources["network:default"].name.clone(); { let mut state = session.backend.state.borrow_mut(); @@ -1336,7 +1339,7 @@ fn changed_network_ids_aliases_extra_attachments_and_endpoint_addresses_refuse_o let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let name = session.receipt.resources["container:web"].name.clone(); let network = session.receipt.resources["network:default"].name.clone(); { @@ -1389,13 +1392,13 @@ fn authored_change_and_foreign_container_identity_refuse_before_effects() { let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); fs::write(fixture.project.join(".hack/hack.project.json"), "changed").unwrap(); - assert!(execution::run(&graph, &mut session, Duration::from_secs(5)).is_err()); + assert!(execution::run(&graph, &mut session, RUN_BUDGET).is_err()); assert!(session.backend.state.borrow().effects.is_empty()); let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let name = session.receipt.resources["container:web"].name.clone(); session .backend @@ -1441,7 +1444,7 @@ fn native_reservations_count_against_the_common_budget_and_uncertainty_blocks_ad let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); for requested in [31, 32] { let mut budget = admission::Budget::default(); for _ in 0..requested { @@ -1495,7 +1498,7 @@ fn private_native_input_crosses_typed_stage_intent_without_entering_public_journ &values, ); let (graph, mut session) = fixture.session(prepared); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); assert_eq!(session.backend.state.borrow().staged, vec!["web"]); assert!( !fs::read_to_string(session.root.join("state.json")) @@ -1542,7 +1545,7 @@ fn cleanup_retry_keeps_observed_stop_request_for_the_same_immutable_id() { let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); session.backend.state.borrow_mut().fail_delete = true; assert!(cleanup_using(&session.backend, &mut session.receipt, &session.root).is_err()); let (mut retained, _) = journal::load(&fixture.candidate, RUN, OWNER, BOOT).unwrap(); @@ -1568,7 +1571,7 @@ fn committed_removal_retry_does_not_repeat_stop_or_regress_the_receipt_phase() { let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); session.backend.state.borrow_mut().fail_delete = true; assert!(cleanup_using(&session.backend, &mut session.receipt, &session.root).is_err()); let (mut retained, _) = journal::load(&fixture.candidate, RUN, OWNER, BOOT).unwrap(); @@ -1597,7 +1600,7 @@ fn stopped_cleanup_resumes_only_terminal_instances_without_another_stop_batch() let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); session.backend.state.borrow_mut().fail_delete = true; assert!(cleanup_using(&session.backend, &mut session.receipt, &session.root).is_err()); // Select the durable Stopped boundary with actual matching terminal observations. @@ -1623,7 +1626,7 @@ fn cleanup_of_removed_inventory_is_read_only_and_keeps_the_committed_bytes() { let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); cleanup_using(&session.backend, &mut session.receipt, &session.root).unwrap(); let before = fs::read(session.root.join("state.json")).unwrap(); let effects = session.backend.state.borrow().effects.clone(); @@ -1648,7 +1651,7 @@ fn stopped_or_removing_inventory_refuses_a_restarted_instance_before_effects() { let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); session.backend.state.borrow_mut().fail_delete = true; assert!(cleanup_using(&session.backend, &mut session.receipt, &session.root).is_err()); session.receipt.phase = phase; @@ -1677,7 +1680,7 @@ fn removed_inventory_refuses_even_matching_resource_reappearance_without_effects let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let containers = session.backend.state.borrow().containers.clone(); let networks = session.backend.state.borrow().networks.clone(); cleanup_using(&session.backend, &mut session.receipt, &session.root).unwrap(); @@ -1712,7 +1715,7 @@ fn cleanup_owner_guard_refuses_before_provider_connection_or_receipt_mutation() assert!(!fixture.candidate.state_root.join("run/smolvm").exists()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let before = session.backend.state.borrow().effects.clone(); assert!( cleanup_using_guarded( @@ -1750,7 +1753,7 @@ fn cleanup_guard_loss_during_first_observation_prevents_next_request_and_stop_in let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let changed = Cell::new(false); let backend = Changed { backend: &session.backend, @@ -1830,7 +1833,7 @@ fn native_stop_failure_keeps_reservation_and_maps_only_owned_ids_to_workload_nam let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); session.backend.state.borrow_mut().fail_stop = true; let error = cleanup_using(&session.backend, &mut session.receipt, &session.root).unwrap_err(); let detail = error.stop_failures.as_ref().unwrap(); @@ -1865,7 +1868,7 @@ fn cleanup_owner_guard_fences_delete_after_stop_and_network_delete_after_contain let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let changed = Rc::new(Cell::new(false)); { let mut state = session.backend.state.borrow_mut(); @@ -1948,7 +1951,7 @@ fn owner_change_during_final_terminal_inspection_refuses_container_delete() { let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let changed = Rc::new(Cell::new(false)); session .backend @@ -2004,7 +2007,7 @@ fn old_boot_removed_history_releases_capacity_only_after_strict_owner_and_remova let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); cleanup_using(&session.backend, &mut session.receipt, &session.root).unwrap(); let new_boot = "87654321-abcd-abcd-abcd-123456789abc"; let check = || { @@ -2845,7 +2848,7 @@ fn finite_native_logs_bind_exact_member_tail_and_restart_generation_without_effe let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(2)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let name = session.receipt.resources["container:web"].name.clone(); let id = session.receipt.resources["container:web"] .id @@ -2859,7 +2862,7 @@ fn finite_native_logs_bind_exact_member_tail_and_restart_generation_without_effe .get_mut(&name) .unwrap()["State"]["StartedAt"] = json!("2026-10-09T00:00:00.000000000Z"); let effects = session.backend.state.borrow().effects.clone(); - let deadline = Instant::now() + Duration::from_secs(2); + let deadline = Instant::now() + RUN_BUDGET; let logs = logs_with( &session.backend, session.receipt.clone(), @@ -2932,7 +2935,7 @@ fn finite_native_logs_share_one_deadline_with_all_source_and_data_checks() { let fixture = Fixture::new(basic()); let (graph, mut session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); - execution::run(&graph, &mut session, Duration::from_secs(2)).unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); let name = session.receipt.resources["container:web"].name.clone(); session .backend @@ -2947,7 +2950,7 @@ fn finite_native_logs_share_one_deadline_with_all_source_and_data_checks() { .borrow_mut() .log_data_deadlines .clear(); - let deadline = Instant::now() + Duration::from_secs(2); + let deadline = Instant::now() + RUN_BUDGET; logs_with( &session.backend, session.receipt.clone(), @@ -2968,6 +2971,8 @@ fn finite_native_logs_share_one_deadline_with_all_source_and_data_checks() { state.log_data_deadlines.clear(); state.expire_source_check = 4; } + // The expiry under test: the fake sleeps past this deadline at the fourth source check, + // so it stays short instead of using `RUN_BUDGET`. let deadline = Instant::now() + Duration::from_secs(1); assert!( logs_with( diff --git a/packages/runtime-core/src/test_budget.rs b/packages/runtime-core/src/test_budget.rs new file mode 100644 index 000000000..20dffb751 --- /dev/null +++ b/packages/runtime-core/src/test_budget.rs @@ -0,0 +1,14 @@ +//! Harness budgets for tests that drive fake backends. + +use std::time::Duration; + +/// Harness deadline for one fake-backend operation in a test: a graph run, selection, +/// exec, log read, or a persistent-data enrollment or retained read. +/// +/// The fakes answer at once, so a passing call returns without waiting on this budget; +/// it only bounds a call that would otherwise hang. It is a test budget, not a product +/// bound: `execution::run` still refuses budgets above its own 600 s limit, and cases +/// that exercise expiry pass their own short or zero deadline. Fixed 1-5 s deadlines +/// failed with `graph_timeout` under host load (journal writes and syncs on a loaded +/// host), so these calls take their deadline from this one constant. +pub(crate) const TEST_RUN_BUDGET: Duration = Duration::from_secs(120); diff --git a/tests/case-drain.test.ts b/tests/case-drain.test.ts new file mode 100644 index 000000000..d704fcb90 --- /dev/null +++ b/tests/case-drain.test.ts @@ -0,0 +1,152 @@ +import { expect, test } from "bun:test"; +import { + CASE_DRAIN_BUN_MARGIN_MS, + createCaseDrain, +} from "./helpers/case-drain.ts"; + +const BUDGET_MS = 40; +const DRAIN_MS = 200; + +test("a case that overruns its budget but settles within the drain fails once and leaves later cases free", async () => { + const cases = createCaseDrain({ label: "Fixture", drainMs: DRAIN_MS }); + let finished = false; + await expect( + cases.run( + "slow", + async () => { + await Bun.sleep(BUDGET_MS + 60); + finished = true; + }, + BUDGET_MS + ) + ).rejects.toThrow(/^E_TEST_CASE_BUDGET: Fixture case exceeded .*settled/); + // The run returns only after the overrunning body settled. + expect(finished).toBe(true); + expect(cases.reservedBy()).toBeUndefined(); + expect(() => cases.requireUnreserved()).not.toThrow(); + let ran = false; + await cases.run( + "next", + () => { + ran = true; + }, + BUDGET_MS + ); + expect(ran).toBe(true); +}); + +test("a case still running past its drain reserves shared state only until its body settles", async () => { + const cases = createCaseDrain({ label: "Fixture", drainMs: DRAIN_MS }); + const release = Promise.withResolvers(); + await expect( + cases.run("hung", () => release.promise, BUDGET_MS) + ).rejects.toThrow( + /^E_TEST_CASE_BUDGET: Fixture case exceeded .*the next case waits/ + ); + expect(cases.reservedBy()).toBe("hung"); + expect(() => cases.requireUnreserved()).toThrow( + /^E_TEST_CASE_RESERVED: Fixture case "hung" still runs/ + ); + let ran = false; + await expect( + cases.run( + "while-hung", + () => { + ran = true; + }, + BUDGET_MS + ) + ).rejects.toThrow(/^E_TEST_CASE_RESERVED/); + expect(ran).toBe(false); + release.resolve(); + await Bun.sleep(0); + expect(cases.reservedBy()).toBeUndefined(); + await cases.run( + "after", + () => { + ran = true; + }, + BUDGET_MS + ); + expect(ran).toBe(true); +}); + +test("a case that fails within its budget reports its own error and reserves nothing", async () => { + const cases = createCaseDrain({ label: "Fixture", drainMs: DRAIN_MS }); + await expect( + cases.run( + "fails", + () => { + throw new Error("own failure"); + }, + BUDGET_MS + ) + ).rejects.toThrow("own failure"); + expect(cases.reservedBy()).toBeUndefined(); +}); + +test("a hung body that later rejects still clears its reservation", async () => { + const cases = createCaseDrain({ label: "Fixture", drainMs: DRAIN_MS }); + const release = Promise.withResolvers(); + await expect( + cases.run("hung", () => release.promise, BUDGET_MS) + ).rejects.toThrow(/^E_TEST_CASE_BUDGET/); + release.reject(new Error("late failure")); + await Bun.sleep(0); + expect(cases.reservedBy()).toBeUndefined(); +}); + +test("the Bun timeout covers the budget, the drain and the margin", () => { + const cases = createCaseDrain({ label: "Fixture", drainMs: DRAIN_MS }); + expect(cases.bunTimeoutMs(BUDGET_MS)).toBe( + BUDGET_MS + DRAIN_MS + CASE_DRAIN_BUN_MARGIN_MS + ); +}); + +test("the next case waits for a reserving body and starts once it settles", async () => { + const cases = createCaseDrain({ label: "Fixture", drainMs: DRAIN_MS }); + const release = Promise.withResolvers(); + await expect( + cases.run("slow", () => release.promise, BUDGET_MS) + ).rejects.toThrow(/^E_TEST_CASE_BUDGET/); + setTimeout(() => release.resolve(), 50); + await cases.awaitUnreserved(DRAIN_MS); + expect(cases.reservedBy()).toBeUndefined(); + let ran = false; + await cases.run( + "next", + () => { + ran = true; + }, + BUDGET_MS + ); + expect(ran).toBe(true); +}); + +test("after one wait runs out, later cases refuse at once until the body settles", async () => { + const cases = createCaseDrain({ label: "Fixture", drainMs: DRAIN_MS }); + const release = Promise.withResolvers(); + await expect( + cases.run("hung", () => release.promise, BUDGET_MS) + ).rejects.toThrow(/^E_TEST_CASE_BUDGET/); + const started = performance.now(); + await expect(cases.awaitUnreserved(BUDGET_MS)).rejects.toThrow( + /^E_TEST_CASE_RESERVED: Fixture case "hung"/ + ); + expect(performance.now() - started).toBeGreaterThanOrEqual(BUDGET_MS - 5); + const again = performance.now(); + await expect(cases.awaitUnreserved(DRAIN_MS)).rejects.toThrow( + /^E_TEST_CASE_RESERVED/ + ); + expect(performance.now() - again).toBeLessThan(BUDGET_MS); + release.resolve(); + await Bun.sleep(0); + await cases.awaitUnreserved(DRAIN_MS); + expect(cases.reservedBy()).toBeUndefined(); +}); + +test("awaiting with no reservation returns at once", async () => { + const cases = createCaseDrain({ label: "Fixture", drainMs: DRAIN_MS }); + await cases.awaitUnreserved(DRAIN_MS); + expect(cases.reservedBy()).toBeUndefined(); +}); diff --git a/tests/e2e/native-process-policy-initial-replay.ts b/tests/e2e/native-process-policy-initial-replay.ts index 18daa785c..87f2a7ce7 100644 --- a/tests/e2e/native-process-policy-initial-replay.ts +++ b/tests/e2e/native-process-policy-initial-replay.ts @@ -10,6 +10,7 @@ import type { } from "../../src/lib/native-compose-ownership.ts"; import { isNativeComposeTopologyPredicate } from "../../src/lib/native-compose-ownership.ts"; import { exec } from "../../src/lib/shell.ts"; +import { testBudgetMs } from "../helpers/test-budget.ts"; import { type ProcessPolicyInitialTraceQuery, readProcessPolicyInitialTrace, @@ -20,6 +21,27 @@ import { } from "./native-process-policy-startup-diagnostic.ts"; const REFUSAL = "Initial process-policy replay unavailable; values omitted"; +/** + * Aggregate probe bound the replayed ownership policy runs with, unless the + * recorded selection names one: the largest the policy accepts. One query + * spawns a stub engine per recorded reply under a single clock, 15 s by + * default. The replay classifies recorded replies and never tests that clock; + * at the default, a 36-worker churn (load 100-120) expired it + * (`E_NATIVE_COMPOSE_PROBE_TIMEOUT`) and changed the replayed outcome. + */ +const REPLAY_PROBE_BUDGET_MS = 60_000; +/** + * Harness budget for one replay child: a fresh Bun process that transpiles the + * ownership policy and spawns one stub engine per recorded reply. It outlasts + * the policy's own probe bound inside it. A fixed 5 s refused replays on + * loaded hosts. + */ +const REPLAY_CHILD_BUDGET_MS = Math.max( + testBudgetMs(5000), + REPLAY_PROBE_BUDGET_MS + 10_000 +); +/** Harness budget for every replay child of one trace summary together. */ +const REPLAY_SUMMARY_BUDGET_MS = testBudgetMs(30_000); const SERVICES = ["forced", "graceful", "reaper", "retry"] as const; const CODES = [ "E_NATIVE_COMPOSE_OWNERSHIP", @@ -160,7 +182,8 @@ await Bun.write(Bun.stdout,row.stdout);process.exit(row.exitCode); await chmod(docker, 0o700); const program = ` import {assertNativeComposeOwned,observeNativeComposeStartupOwned,NativeComposeOwnershipError,nativeComposeOwnershipRefusal,nativeComposeTopologyPredicate} from ${JSON.stringify(join(import.meta.dir, "../../src/lib/native-compose-ownership.ts"))}; -const {selection}=await Bun.file(${JSON.stringify(inputs)}).json(); +const {selection:recorded}=await Bun.file(${JSON.stringify(inputs)}).json(); +const selection={timeoutMs:${REPLAY_PROBE_BUDGET_MS},...recorded}; let outcome="refused",code=null,reason=null,topologyPredicate=null; try {const value=${REPLAY_CALLS[opts.mode]};outcome=value===null?"unready":"owned";} catch(error){code=error instanceof NativeComposeOwnershipError?error.code:null;reason=nativeComposeOwnershipRefusal(error)??null;topologyPredicate=nativeComposeTopologyPredicate(error)??null;} @@ -173,7 +196,7 @@ process.stdout.write(JSON.stringify({outcome,code,reason,topologyPredicate,consu cwd: opts.directory, env: { PATH: opts.directory }, stdin: "ignore", - timeoutMs: opts.timeoutMs ?? 5000, + timeoutMs: opts.timeoutMs ?? REPLAY_CHILD_BUDGET_MS, } ); if (result.exitCode !== 0 || result.stdout.length > 1024) { @@ -222,13 +245,13 @@ export async function summarizeProcessPolicyInitialTrace(opts: { readonly projectRoot: string; readonly replayRoot: string; }) { - const deadline = Date.now() + 30_000; + const deadline = Date.now() + REPLAY_SUMMARY_BUDGET_MS; const remaining = (): number => { const budget = Math.floor(deadline - Date.now()); if (budget <= 0) { throw new Error(REFUSAL); } - return Math.min(budget, 5000); + return Math.min(budget, REPLAY_CHILD_BUDGET_MS); }; const trace = await readProcessPolicyInitialTrace(opts.directory); const store = await openNativeComposeGenerationStore({ diff --git a/tests/helpers/case-drain.ts b/tests/helpers/case-drain.ts new file mode 100644 index 000000000..e7c8b914a --- /dev/null +++ b/tests/helpers/case-drain.ts @@ -0,0 +1,151 @@ +import { test as bunTest } from "bun:test"; + +/** + * Case budgets for a test file whose cases share module-level fixture state + * (temp roots, a stub engine on `PATH`, process environment). + * + * Bun's own timeout does not cancel a case body. An abandoned body keeps + * running, and while the next case runs it can still read the shared globals, + * spawn through the shared `PATH` and restore environment it saved. Such files + * used to latch after the first abandoned case and fail every later case. + * + * This guard runs each body under its own budget instead, so an overrun fails + * that case by name, once. The overrunning body then gets `drainMs`, inside + * the same case's Bun timeout, to settle. + * + * - Settled within the drain: the next case starts normally. + * - Still running: the shared state stays reserved for that body, and + * `afterEach` keeps its roots. The next case's `beforeEach` calls + * `awaitUnreserved`, which waits (outside that case's own timeout) for the + * body to settle; a slow body on a loaded host usually does, and the case + * then starts normally. Only a body that outlasts that wait too makes later + * cases refuse with a closed reason naming it, without waiting again, until + * it settles. + */ +export type CaseDrain = { + /** Register a case: `budgetMs` bounds its body; the Bun timeout adds the drain. */ + readonly test: ( + name: string, + body: () => void | Promise, + budgetMs: number + ) => void; + /** Run one case body under `budgetMs`, then the drain. Rejects with the case's own outcome. */ + readonly run: ( + name: string, + body: () => void | Promise, + budgetMs: number + ) => Promise; + /** The Bun timeout for a body budget: budget, drain and margin. */ + readonly bunTimeoutMs: (budgetMs: number) => number; + /** The case whose body still runs past its budget and drain, if any. */ + readonly reservedBy: () => string | undefined; + /** + * Wait up to `waitMs` for a reserving body to settle, then refuse with a + * closed reason. After one wait has run out, refuse at once until the body + * settles. + */ + readonly awaitUnreserved: (waitMs: number) => Promise; + /** Refuse, with a closed reason, while an earlier case's body still runs. */ + readonly requireUnreserved: () => void; +}; + +/** Slack between a case's budget plus drain and Bun's own timeout. */ +export const CASE_DRAIN_BUN_MARGIN_MS = 5000; + +type Outcome = "settled" | "overrun"; + +async function within(running: Promise, ms: number): Promise { + let timer: ReturnType | undefined; + const overrun = new Promise((resolve) => { + timer = setTimeout(() => resolve("overrun"), ms); + }); + try { + return await Promise.race([ + running.then( + (): Outcome => "settled", + (): Outcome => "settled" + ), + overrun, + ]); + } finally { + clearTimeout(timer); + } +} + +type Reservation = { + readonly name: string; + /** Resolves once the body settled and this reservation was cleared. */ + readonly released: Promise; + waitedOut: boolean; +}; + +export function createCaseDrain(opts: { + /** Prefix for refusals, e.g. "Adoption fixture". */ + readonly label: string; + /** Time an overrunning body gets to settle inside its own case. */ + readonly drainMs: number; +}): CaseDrain { + let reservation: Reservation | undefined; + const bunTimeoutMs = (budgetMs: number) => + budgetMs + opts.drainMs + CASE_DRAIN_BUN_MARGIN_MS; + const refusal = (held: Reservation) => + new Error( + `E_TEST_CASE_RESERVED: ${opts.label} case "${held.name}" still runs past its budget, drain and wait, so this case did not start; cases start again once it settles.` + ); + const requireUnreserved = () => { + if (reservation !== undefined) { + throw refusal(reservation); + } + }; + const awaitUnreserved = async (waitMs: number) => { + const held = reservation; + if (held === undefined) { + return; + } + if (held.waitedOut || (await within(held.released, waitMs)) === "overrun") { + held.waitedOut = true; + throw refusal(held); + } + }; + const run = async ( + name: string, + body: () => void | Promise, + budgetMs: number + ): Promise => { + requireUnreserved(); + const running = (async () => { + await body(); + })(); + if ((await within(running, budgetMs)) === "settled") { + await running; + return; + } + const drained = await within(running, opts.drainMs); + if (drained === "overrun") { + const released = running + .catch(() => undefined) + .then(() => { + if (reservation === held) { + reservation = undefined; + } + }); + const held: Reservation = { name, released, waitedOut: false }; + reservation = held; + } + throw new Error( + drained === "settled" + ? `E_TEST_CASE_BUDGET: ${opts.label} case exceeded its ${budgetMs} ms budget; its body settled within the ${opts.drainMs} ms drain, so later cases start normally.` + : `E_TEST_CASE_BUDGET: ${opts.label} case exceeded its ${budgetMs} ms budget and its ${opts.drainMs} ms drain; the next case waits for its body to settle.` + ); + }; + return { + test: (name, body, budgetMs) => { + bunTest(name, () => run(name, body, budgetMs), bunTimeoutMs(budgetMs)); + }, + run, + bunTimeoutMs, + reservedBy: () => reservation?.name, + awaitUnreserved, + requireUnreserved, + }; +} diff --git a/tests/helpers/native-fixture-cases.ts b/tests/helpers/native-fixture-cases.ts index 75c715a29..76ad6ebaa 100644 --- a/tests/helpers/native-fixture-cases.ts +++ b/tests/helpers/native-fixture-cases.ts @@ -53,7 +53,7 @@ export type NativeFixtureCases = { }; /** Slack between the case's own deadline plus settlement and Bun's timeout. */ -const BUN_TIMEOUT_MARGIN_MS = 5000; +export const NATIVE_FIXTURE_BUN_MARGIN_MS = 5000; function message(error: unknown): string { return error instanceof Error ? error.message : String(error); @@ -190,7 +190,7 @@ export function createNativeFixtureCases(opts: { bunTest( name, () => runCase(body, timeoutMs), - timeoutMs + opts.settlementBudgetMs + BUN_TIMEOUT_MARGIN_MS + timeoutMs + opts.settlementBudgetMs + NATIVE_FIXTURE_BUN_MARGIN_MS ); }, }; diff --git a/tests/helpers/test-budget.ts b/tests/helpers/test-budget.ts new file mode 100644 index 000000000..760ecc236 --- /dev/null +++ b/tests/helpers/test-budget.ts @@ -0,0 +1,89 @@ +import { DEFAULT_FILE_TIMEOUT_SECONDS } from "../../scripts/test-shard.ts"; + +/** + * Harness budgets for load-sensitive tests. + * + * A harness budget bounds how long a test waits for its own fixture work: + * spawning stub engines and compilers, transpiling a child program, or a + * stub-backed owner operation. It is not a product bound. A passing case + * finishes as soon as that work does, so a larger budget only lengthens how + * long a hung case takes to fail. Product bounds under test (an expiring + * caller deadline, a cancellation latency) keep their own literal values. + * + * Each call site names the time its fixture work needs on an unloaded host; + * `TEST_BUDGET_SCALE` is the headroom for a loaded one. These fixtures are + * spawn-bound (a retained adoption dispatch runs about 750 stub-engine + * processes; an authored start runs the compiler several times), so their wall + * time grows faster than the load average. On the M5 at 1-min load 15-40, and + * on the M3 while another suite ran, they took three to four times their + * unloaded time. With a second suite plus churn workers, the M5 at load 28-90 + * took more than ten times: one public retained `up` dispatch outlasted a + * 200 s deadline, and a v7 dispatch case took 146 s even at load 22-50. + * A deadline a case does not test must not expire first, so the scale keeps + * twice that headroom. + * + * The scale stops at `CASE_TIMEOUT_CEILING_MS`. Hosted CI runs each file under + * the shard's per-file limit; a case whose budget outlives that limit is + * reported as a file timeout instead of by name. So the headroom is real only + * up to the ceiling, and a host loaded past it fails cases by name. + */ +export const TEST_BUDGET_SCALE = 20; + +/** Per-file limit of the hosted test shard (`scripts/test-shard.ts`). */ +export const SHARD_FILE_TIMEOUT_MS = DEFAULT_FILE_TIMEOUT_SECONDS * 1000; + +/** + * Largest Bun timeout one case may have, its settlement or drain included: a + * third of the shard's per-file limit. The slowest hosted file (adoption + * generation, 4m55s-6m57s) plus one case hung for this long still ends inside + * the 720 s limit, so Bun prints that case's own timeout before the shard + * stops the file. + */ +export const CASE_TIMEOUT_CEILING_MS = Math.floor(SHARD_FILE_TIMEOUT_MS / 3); + +/** + * Scale the unloaded time a fixture needs to the budget a loaded host needs, + * never above `ceilingMs` (the case ceiling by default). A file whose Bun + * timeout adds settlement or drain time to a body budget passes the body's + * share of the ceiling. + */ +export function testBudgetMs( + unloadedMs: number, + opts: { readonly ceilingMs?: number } = {} +): number { + const ceilingMs = opts.ceilingMs ?? CASE_TIMEOUT_CEILING_MS; + if ( + !( + Number.isSafeInteger(unloadedMs) && + unloadedMs > 0 && + Number.isSafeInteger(ceilingMs) && + ceilingMs > 0 && + ceilingMs <= CASE_TIMEOUT_CEILING_MS + ) + ) { + throw new Error( + "E_TEST_BUDGET_INVALID: a test budget must be a positive whole number of milliseconds within the case ceiling." + ); + } + return Math.min(unloadedMs * TEST_BUDGET_SCALE, ceilingMs); +} + +/** + * Check a composed Bun timeout (a body budget plus settlement, drain or + * margin) against the case ceiling when the file loads, so a file cannot + * register a case that would outlive the shard's per-file limit. + */ +export function withinCaseCeilingMs(timeoutMs: number): number { + if ( + !( + Number.isSafeInteger(timeoutMs) && + timeoutMs > 0 && + timeoutMs <= CASE_TIMEOUT_CEILING_MS + ) + ) { + throw new Error( + "E_TEST_BUDGET_CEILING: a case timeout must be a positive whole number of milliseconds within the case ceiling." + ); + } + return timeoutMs; +} diff --git a/tests/native-authored-project-start.test.ts b/tests/native-authored-project-start.test.ts index 28c68b2a2..664a69d0e 100644 --- a/tests/native-authored-project-start.test.ts +++ b/tests/native-authored-project-start.test.ts @@ -41,36 +41,62 @@ import { evaluateComposeServiceEnvFreshness, } from "../src/lib/compose-env-fingerprint.ts"; import { writeLifecycleMuxFixture } from "./helpers/lifecycle-mux-fixture.ts"; +import { awaitPidFile } from "./helpers/native-authored-settlement.ts"; import { createNativeFixtureCases, + NATIVE_FIXTURE_BUN_MARGIN_MS, type NativeFixtureCase, } from "./helpers/native-fixture-cases.ts"; +import { + CASE_TIMEOUT_CEILING_MS, + testBudgetMs, + withinCaseCeilingMs, +} from "./helpers/test-budget.ts"; const run = "a".repeat(32); const CANARY = "synthetic-private-native-start-canary"; -/** Allowance for a case's owners, groups and stop endpoints to be absent afterwards. */ -const SETTLEMENT_BUDGET_MS = 15_000; +/** + * Allowance for a case's owners, groups and stop endpoints to be absent + * afterwards. Harness budget: a passing case settles as soon as they exit. + */ +const SETTLEMENT_BUDGET_MS = testBudgetMs(1500); +/** Largest caller startup deadline `serveNativeAuthoredProject` accepts. */ +const MAX_STARTUP_TIMEOUT_MS = 300_000; +/** + * Body budget for every case, the two-cycle case included: what the case + * ceiling leaves after the settlement budget and the guard's margin. A case + * that overruns it fails by name, inside the hosted shard's per-file limit. + */ +const CASE_TIMEOUT_MS = + CASE_TIMEOUT_CEILING_MS - SETTLEMENT_BUDGET_MS - NATIVE_FIXTURE_BUN_MARGIN_MS; +withinCaseCeilingMs( + CASE_TIMEOUT_MS + SETTLEMENT_BUDGET_MS + NATIVE_FIXTURE_BUN_MARGIN_MS +); /** * Caller startup deadline for cases that are not about the deadline itself. * Each start spawns several Bun children (compiler, planner, owner, hooks); on * a loaded host that takes well over the 10-20 s these cases used to allow. + * The scaled budget is capped at the product's own ceiling and at half the + * case budget, so a start and its stop fit in one case. Cases that test the + * deadline pass their own literal product deadline. */ -const STARTUP_TIMEOUT_MS = 60_000; +const STARTUP_TIMEOUT_MS = Math.min( + testBudgetMs(15_000), + MAX_STARTUP_TIMEOUT_MS, + Math.floor(CASE_TIMEOUT_MS / 2) +); /** Caller stop deadline for ordinary `down` in the hook and live-stop cases. */ -const STOP_TIMEOUT_MS = 60_000; -/** - * Body budget for every case: a full start plus a stop, with slack. The guard - * adds the settlement budget on top, so cases never fall back to Bun's 5 s. - */ -const CASE_TIMEOUT_MS = 2 * STARTUP_TIMEOUT_MS; +const STOP_TIMEOUT_MS = STARTUP_TIMEOUT_MS; /** A caller's 50 ms startup deadline must return the frontend well inside this wall bound. */ const DEADLINE_WALL_BOUND_MS = 1000; /** - * Upper bound for a stand-in owner that never receives its release or SIGTERM. - * This bounds the orphan a broken case can leave rather than preventing it: the - * cap outlasts the case's Bun timeout. Passing cases release it explicitly. + * Upper bound for a stand-in owner that never receives its release or SIGTERM, + * and for a stand-in status request that hangs. This bounds the orphan a broken + * case can leave rather than preventing it: it outlasts every case's Bun + * timeout, which the case ceiling bounds, and ends well inside the hosted + * shard's per-file limit. Passing cases release or cancel them explicitly. */ -const OWNER_RELEASE_CAP_MS = 2 * CASE_TIMEOUT_MS; +const STAND_IN_CAP_MS = CASE_TIMEOUT_CEILING_MS; const cases = createNativeFixtureCases({ label: "Startup fixture", defaultTimeoutMs: CASE_TIMEOUT_MS, @@ -464,6 +490,9 @@ console.log(JSON.stringify(result)); group: true, }); // A hanging status request must be killed by cancellation, not outlive the case. + // The stand-in publishes its PID by rename, so the file is either absent or + // complete: a cancel that lands mid-write must not leave an empty record that + // settlement reads as a write still in progress. const statusPid = join(root, "status.pid"); owner.settlement.record({ kind: "pid-file", @@ -475,11 +504,11 @@ console.log(JSON.stringify(result)); await writeFile( binary, `#!${process.execPath} -import {appendFile,readFile,readdir,unlink,writeFile} from 'node:fs/promises'; +import {appendFile,readFile,readdir,rename,unlink,writeFile} from 'node:fs/promises'; import {createHash} from 'node:crypto';import {join} from 'node:path'; const args=process.argv.slice(2);const action=args[4];await appendFile(${JSON.stringify(calls)},JSON.stringify(args)+'\\n'); if(${options.runtimeStopped === true}&&args[2]==='runtime'&&args[3]==='status'){console.log(JSON.stringify({phase:'stopped',process_alive:false}));process.exit(0)} -if(${options.runtimeStatusHangs === true}&&args[2]==='runtime'&&args[3]==='status'){await appendFile(${JSON.stringify(statusPid)},process.pid+'\\n');await new Promise((resolve)=>setTimeout(resolve,${2 * CASE_TIMEOUT_MS}));process.exit(0)} +if(${options.runtimeStatusHangs === true}&&args[2]==='runtime'&&args[3]==='status'){const staged=${JSON.stringify(statusPid)}+'.'+process.pid;await writeFile(staged,process.pid+'\\n');await rename(staged,${JSON.stringify(statusPid)});await new Promise((resolve)=>setTimeout(resolve,${STAND_IN_CAP_MS}));process.exit(0)} if(${options.runtimeStatusWrappedStop === true}&&args[2]==='runtime'&&args[3]==='status'){console.error(JSON.stringify({code:'native_graph_foreground',cause_code:'runtime_not_running',message:${JSON.stringify(CANARY)}}));process.exit(2)} const journal=${JSON.stringify(journal)}; const root=${JSON.stringify(join(projectDir, ".internal", "native-authored-runs"))}; @@ -547,7 +576,7 @@ if(['plan','serve','frontend-plan','frontend-serve'].includes(action)){ // Handshake, not a wall-clock window: retire only once the frontend has // finished ready publication and the test released this run (or on SIGTERM). const release=${JSON.stringify(ownerRelease(root, ""))}+input.run; - const cap=performance.now()+${OWNER_RELEASE_CAP_MS}; + const cap=performance.now()+${STAND_IN_CAP_MS}; while(!(await Bun.file(release).exists())&&performance.now()undefined); await finish(); @@ -578,6 +607,7 @@ if(['plan','serve','frontend-plan','frontend-serve'].includes(action)){ calls, delivery, sources, + statusPid, }; } @@ -887,22 +917,12 @@ macTest( signal: signal.signal, startupTimeoutMs: STARTUP_TIMEOUT_MS, }); - // Cancel once the status request is in flight. The check treats the - // aborted request as inconclusive; the start must still not reserve. - // The stand-in request hangs past the case budget, so this waits on the - // request itself rather than racing a shorter fixed window. - const deadline = performance.now() + CASE_TIMEOUT_MS; - while ( - !( - (await Bun.file(selected.calls).exists()) && - (await Bun.file(selected.calls).text()).includes('"status"') - ) - ) { - if (performance.now() > deadline) { - throw new Error("runtime status was never requested"); - } - await Bun.sleep(20); - } + // Cancel once the status request is in flight and has published its PID. + // The check treats the aborted request as inconclusive; the start must + // still not reserve. The stand-in request hangs past the case budget, so + // this waits on the request's own identity rather than a fixed window, and + // the cancel cannot land while the stand-in is still recording it. + await awaitPidFile(selected.statusPid, { budgetMs: CASE_TIMEOUT_MS }); signal.abort(); const error = await failure(started, "not-started"); expect(error).toHaveProperty("canceled", true); @@ -1069,7 +1089,7 @@ macTest( serveNativeAuthoredProject({ ...selected, run, - startupTimeoutMs: 10_000, + startupTimeoutMs: STARTUP_TIMEOUT_MS, }), "not-started" ); @@ -1776,6 +1796,8 @@ macTest( 2 ); }, + // Two full start and stop cycles. The case ceiling leaves no room for a + // second case budget; a passing cycle takes a small part of one. CASE_TIMEOUT_MS ); diff --git a/tests/native-compose-adoption-generation.test.ts b/tests/native-compose-adoption-generation.test.ts index bdad69630..2bf62c659 100644 --- a/tests/native-compose-adoption-generation.test.ts +++ b/tests/native-compose-adoption-generation.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, - test as boundedTest, + test as bunTest, expect, spyOn, } from "bun:test"; @@ -39,6 +39,7 @@ import { renderComposeEnvReferenceOverride, renderManagedComposeEnvOverride, } from "../src/lib/compose-managed-env.ts"; +import { COMPOSE_STARTUP_TIMEOUT_ENV } from "../src/lib/compose-startup-budget.ts"; import { acquireLegacyComposeAdoptionPreparationBinding } from "../src/lib/native-compose-adoption-binding.ts"; import { tryLegacyComposeAdoptedCommand } from "../src/lib/native-compose-adoption-command.ts"; import { @@ -47,7 +48,7 @@ import { } from "../src/lib/native-compose-adoption-execution.ts"; import { LegacyComposeAdoptedGenerationError, - openLegacyComposeAdoptedGenerationStore, + openLegacyComposeAdoptedGenerationStore as openAdoptedGenerationStore, } from "../src/lib/native-compose-adoption-generation.ts"; import { inspectLegacyComposeAdoptionSelection } from "../src/lib/native-compose-adoption-marker.ts"; import { @@ -66,14 +67,145 @@ import { import { buildRuntimeHostMetadataOverride } from "../src/lib/runtime-host-metadata.ts"; import { captureAdoptionDependencyFirstPrepare } from "./e2e/scenarios/native-compose-adoption-dependency-staged-read.ts"; import { completedJobFixtureSources } from "./e2e/scenarios/native-compose-adoption-job-inputs.ts"; -import { createCompletedJobFixtureSettlement } from "./e2e/scenarios/native-compose-adoption-job-worktrees.ts"; +import { + CASE_DRAIN_BUN_MARGIN_MS, + createCaseDrain, +} from "./helpers/case-drain.ts"; import { restoreEnv } from "./helpers/env.ts"; import { managedEnvCompilerFixture } from "./helpers/managed-env-compiler.ts"; import { privateCliEnv } from "./helpers/private-cli-env.ts"; +import { + CASE_TIMEOUT_CEILING_MS, + testBudgetMs, + withinCaseCeilingMs, +} from "./helpers/test-budget.ts"; -// Each owner workflow performs multiple bounded child probes; allow their cumulative work on shared CI hosts. -const test = (name: string, run: () => Promise) => - boundedTest(name, run, 20_000); +/* + * Harness budgets. Every engine probe below spawns the Bun stub engine, and one + * retained dispatch issues several hundred probes, so an owner operation's wall + * time follows host load. Case budgets and the caller deadlines of cases that + * are not about the deadline derive from `testBudgetMs`, within the case + * ceiling that the hosted shard's per-file limit allows. The expiry cases' + * promptness assertion (1.5 s past the deadline) is a product bound under test + * and stays literal. + */ +/** Caller deadline for one owner operation in a case that is not about the deadline. */ +const OPERATION_DEADLINE_MS = testBudgetMs(4000); +/** + * Time a case body that overran its budget gets to settle inside its own case: + * one caller deadline plus slack. A body with several owner operations left + * can need longer on a loaded host; the next case then waits for it. + */ +const CASE_DRAIN_MS = OPERATION_DEADLINE_MS + testBudgetMs(250); +/** + * How long the next case's `beforeEach` waits for a body still running after + * its budget and drain, outside that case's own timeout: one more case + * ceiling. The overrunning case has already failed by name by then. + */ +const CASE_RESERVATION_WAIT_MS = CASE_TIMEOUT_CEILING_MS; +/** `beforeEach` timeout: the reservation wait plus the fixture setup. */ +const BEFORE_EACH_TIMEOUT_MS = CASE_RESERVATION_WAIT_MS + testBudgetMs(500); +/** + * Every case shares this file's roots, stub engine and environment, and Bun's + * timeout does not cancel a body, so cases run under the drain guard: an + * overrun fails only that case, by name (see `tests/helpers/case-drain.ts`). + */ +const cases = createCaseDrain({ + label: "Adoption fixture", + drainMs: CASE_DRAIN_MS, +}); +/** Largest body budget whose Bun timeout (budget, drain and margin) fits the case ceiling. */ +const CASE_BODY_CEILING_MS = + CASE_TIMEOUT_CEILING_MS - CASE_DRAIN_MS - CASE_DRAIN_BUN_MARGIN_MS; +withinCaseCeilingMs(cases.bunTimeoutMs(CASE_BODY_CEILING_MS)); +/** Body budget for a case that needs `unloadedMs` on an unloaded host. */ +function caseBudgetMs(unloadedMs: number): number { + return testBudgetMs(unloadedMs, { ceilingMs: CASE_BODY_CEILING_MS }); +} +type CaseBody = () => void | Promise; +/** Bun's `test`, `test.each` and `test.skipIf`, each under the drain guard. */ +const boundedTest = Object.assign( + (name: string, body: CaseBody, budgetMs: number) => + cases.test(name, body, budgetMs), + { + each: + (values: readonly T[]) => + ( + name: string, + body: (value: T) => void | Promise, + budgetMs: number + ) => + bunTest.each(values as T[])( + name, + (value: T) => cases.run(name, () => body(value), budgetMs), + cases.bunTimeoutMs(budgetMs) + ), + skipIf: + (condition: boolean) => + (name: string, body: CaseBody, budgetMs: number) => + bunTest.skipIf(condition)( + name, + () => cases.run(name, body, budgetMs), + cases.bunTimeoutMs(budgetMs) + ), + } +); +const test = (name: string, run: CaseBody) => + boundedTest(name, run, caseBudgetMs(20_000)); +/** + * Probe-phase bound for stores these cases open. One store probe carries a + * single aggregate clock across a whole inspection (dozens of stub-engine + * processes), 15 s by default. That default is a product bound no case here + * tests, so stores open with the largest phase bound the product accepts. + * Public dispatch opens its own store and keeps the default (HACK-1458). + */ +const PROBE_PHASE_BUDGET_MS = 60_000; +/** + * Caller deadline in the cases that test its expiry after the callback. The + * product behavior under test (expiry ends a held probe or an overlong callback + * with `E_LEGACY_ADOPTION_DEADLINE`, keeps the operation pending, and returns + * within 1.5 s of the deadline) does not depend on its length, but the work + * before the callback must finish first: a 5 s deadline expired before the + * callback at load 30-45. It stays under the probe-phase bound, so the caller + * deadline, not a probe's own phase clock, ends a held probe. The cases do not + * wait it out: once the callback is reached they move the product's clock to + * just before the deadline (`shiftProductClock`). + */ +const EXPIRY_DEADLINE_MS = PROBE_PHASE_BUDGET_MS - 15_000; +/** + * Clock time an expiry case leaves before the deadline when its callback runs. + * It must cover the work before the held probe and the stub engine recording + * its PID, which the case then proves was killed. For per-effect freshness the + * callback's own probe is the held one (0.3 s from the callback at 1-min load + * 45 on the M5); for the final observation the product first runs its other + * final probes (5.7 s at the same load). + */ +const EXPIRY_LEAD_MS = { + "per-effect freshness": testBudgetMs(250), + "final observation": testBudgetMs(1000), +} as const; +/** + * Move the clock the product reads (`Date.now`) forward by `byMs` for the rest + * of the case; it keeps advancing at the real rate. The product takes the + * caller deadline as a `Date.now` timestamp and has no clock seam, so this + * stands in for one. Timers are not moved: a held probe still ends by the + * timeout the product derives from the remaining clock time. Returns the + * restore function; call it in `finally`. + */ +function shiftProductClock(byMs: number): () => void { + const realNow = Date.now.bind(Date); + const clock = spyOn(Date, "now").mockImplementation(() => realNow() + byMs); + return () => clock.mockRestore(); +} +/** The product store opener, with the harness probe-phase bound unless a case names one. */ +function openLegacyComposeAdoptedGenerationStore( + input: Parameters[0] +): ReturnType { + return openAdoptedGenerationStore({ + timeoutMs: PROBE_PHASE_BUDGET_MS, + ...input, + }); +} const CANARY = "synthetic-private-adoption-canary"; const ID = "a".repeat(64); const NETWORK = "b".repeat(64); @@ -146,21 +278,29 @@ type Fixture = { }; let root: string; let projectRoot: string; -let priorPath: string | undefined; -let priorCI: string | undefined; -let priorExecutionMode: string | undefined; -let priorRuntimeBackend: string | undefined; let fixture: Fixture; -let publicStatic7Commands = 0; -const publicStatic7Settlement = createCompletedJobFixtureSettlement(); +/** + * Environment the cases change, as it was when the file loaded. Every case + * restores these values, so a case that settles late cannot leave its own + * values behind as a later case's "prior" ones. + */ +const FILE_ENV = Object.freeze({ + PATH: process.env.PATH, + CI: process.env.CI, + HACK_EXECUTION_MODE: process.env.HACK_EXECUTION_MODE, + HACK_RUNTIME_BACKEND: process.env.HACK_RUNTIME_BACKEND, + [COMPOSE_STARTUP_TIMEOUT_ENV]: process.env[COMPOSE_STARTUP_TIMEOUT_ENV], +}); beforeEach(async () => { - // An outer test timeout does not cancel an awaited owner. Never recycle its globals or roots. - publicStatic7Settlement.assertConfirmed(); - priorPath = process.env.PATH; - priorCI = process.env.CI; - priorExecutionMode = process.env.HACK_EXECUTION_MODE; - priorRuntimeBackend = process.env.HACK_RUNTIME_BACKEND; + // A body still running past its budget and drain owns the shared roots, + // stub engine and environment. Wait for it rather than recycling them. + await cases.awaitUnreserved(CASE_RESERVATION_WAIT_MS); process.env.HACK_RUNTIME_BACKEND = "compose"; + // Public dispatch (`tryLegacyComposeAdoptedCommand`) takes its caller + // deadline from the Compose startup budget, 90 s by default. No case here + // tests that default, so inject the direct operations' harness deadline + // through the product's own setting. + process.env[COMPOSE_STARTUP_TIMEOUT_ENV] = String(OPERATION_DEADLINE_MS); root = await realpath( await mkdtemp(join(tmpdir(), "native-adoption-binding-")) ); @@ -307,21 +447,18 @@ if (fixture.mode === "inventory-change" && kind === "container" && action === "l process.env.PATH = root; await mkdir(join(projectRoot, ".git")); await save(); -}); +}, BEFORE_EACH_TIMEOUT_MS); afterEach(async () => { - if (publicStatic7Commands !== 0) { - publicStatic7Settlement.markUnconfirmed(); - } - try { - publicStatic7Settlement.assertConfirmed(); - } catch { - // Retain the still-owned fixture; a late callback cannot restore teardown authority. + if (cases.reservedBy() !== undefined) { + // The body still runs and owns this root and environment; keep both. + console.error( + "Adoption fixture root retained for a case still running past its budget and drain." + ); return; } - restoreEnv("PATH", priorPath); - restoreEnv("CI", priorCI); - restoreEnv("HACK_EXECUTION_MODE", priorExecutionMode); - restoreEnv("HACK_RUNTIME_BACKEND", priorRuntimeBackend); + for (const [key, value] of Object.entries(FILE_ENV)) { + restoreEnv(key, value); + } await rm(root, { recursive: true, force: true }); }); async function save() { @@ -607,15 +744,6 @@ else { return binary; } -async function publicStatic7Case(run: () => Promise) { - publicStatic7Commands += 1; - try { - await run(); - } finally { - publicStatic7Commands -= 1; - } -} - function publicStatic7Stages() { const started = Date.now(); return ( @@ -629,29 +757,61 @@ function publicStatic7Stages() { } test("retained Compose fixture admits its explicit backend and preserves opposite-backend refusal before probes", async () => { - await publicStatic7Case(async () => { - expect(process.env.HACK_RUNTIME_BACKEND).toBe("compose"); - const binary = await compiler(); + expect(process.env.HACK_RUNTIME_BACKEND).toBe("compose"); + const binary = await compiler(); + const { store, generation } = await prepared(binary); + const priorCompiler = process.env.HACK_CONFIG_COMPILER_BINARY; + try { + process.env.HACK_CONFIG_COMPILER_BINARY = binary; + await store.publish({ generation, binary }); + const commands = await readFile(join(root, "commands"), "utf8"); + process.env.HACK_RUNTIME_BACKEND = "native"; + try { + await expect( + tryLegacyComposeAdoptedCommand({ + cwd: projectRoot, + operation: "up", + detach: true, + }) + ).rejects.toMatchObject({ code: "E_NATIVE_PROJECT_UNSUPPORTED" }); + expect(await readFile(join(root, "commands"), "utf8")).toBe(commands); + expect((await readReceipt()).pendingOperation).toBeNull(); + } finally { + process.env.HACK_RUNTIME_BACKEND = "compose"; + } + expect( + await tryLegacyComposeAdoptedCommand({ + cwd: projectRoot, + operation: "up", + detach: true, + }) + ).toBe(0); + expect(await mutationCommands()).toEqual([["container", "start", ID]]); + } finally { + restoreEnv("HACK_CONFIG_COMPILER_BINARY", priorCompiler); + await store.close(); + } +}); + +boundedTest.each(["no", ""] as const)( + "public retained up dispatches the maintained static7 family with disabled policy %j", + async (jobRestartPolicy) => { + const stages = publicStatic7Stages(); + stages("prepare", "enter"); + const { app, job } = await maintainedCompletedJobFixture(); + const binary = await maintainedJobCompiler(); const { store, generation } = await prepared(binary); + stages("prepare", "settled"); const priorCompiler = process.env.HACK_CONFIG_COMPILER_BINARY; try { process.env.HACK_CONFIG_COMPILER_BINARY = binary; + stages("publish", "enter"); await store.publish({ generation, binary }); - const commands = await readFile(join(root, "commands"), "utf8"); - process.env.HACK_RUNTIME_BACKEND = "native"; - try { - await expect( - tryLegacyComposeAdoptedCommand({ - cwd: projectRoot, - operation: "up", - detach: true, - }) - ).rejects.toMatchObject({ code: "E_NATIVE_PROJECT_UNSUPPORTED" }); - expect(await readFile(join(root, "commands"), "utf8")).toBe(commands); - expect((await readReceipt()).pendingOperation).toBeNull(); - } finally { - process.env.HACK_RUNTIME_BACKEND = "compose"; - } + stages("publish", "settled"); + fixture.jobRestartPolicy = jobRestartPolicy; + await save(); + expect(generation.report.adoption_generation_version).toBe(7); + stages("start", "enter"); expect( await tryLegacyComposeAdoptedCommand({ cwd: projectRoot, @@ -659,69 +819,33 @@ test("retained Compose fixture admits its explicit backend and preserves opposit detach: true, }) ).toBe(0); - expect(await mutationCommands()).toEqual([["container", "start", ID]]); + stages("start", "settled"); + expect(await mutationCommands()).toEqual([ + ["container", "start", ID], + ["container", "start", job], + ["container", "start", app], + ]); + expect((await readReceipt()).pendingOperation).toBeNull(); + stages("stop", "enter"); + expect( + await tryLegacyComposeAdoptedCommand({ + cwd: projectRoot, + operation: "down", + }) + ).toBe(0); + stages("stop", "settled"); + expect((await mutationCommands()).slice(-3)).toEqual([ + ["container", "stop", app], + ["container", "stop", job], + ["container", "stop", ID], + ]); } finally { restoreEnv("HACK_CONFIG_COMPILER_BINARY", priorCompiler); await store.close(); } - }); -}); - -boundedTest.each(["no", ""] as const)( - "public retained up dispatches the maintained static7 family with disabled policy %j", - async (jobRestartPolicy) => { - await publicStatic7Case(async () => { - const stages = publicStatic7Stages(); - stages("prepare", "enter"); - const { app, job } = await maintainedCompletedJobFixture(); - const binary = await maintainedJobCompiler(); - const { store, generation } = await prepared(binary); - stages("prepare", "settled"); - const priorCompiler = process.env.HACK_CONFIG_COMPILER_BINARY; - try { - process.env.HACK_CONFIG_COMPILER_BINARY = binary; - stages("publish", "enter"); - await store.publish({ generation, binary }); - stages("publish", "settled"); - fixture.jobRestartPolicy = jobRestartPolicy; - await save(); - expect(generation.report.adoption_generation_version).toBe(7); - stages("start", "enter"); - expect( - await tryLegacyComposeAdoptedCommand({ - cwd: projectRoot, - operation: "up", - detach: true, - }) - ).toBe(0); - stages("start", "settled"); - expect(await mutationCommands()).toEqual([ - ["container", "start", ID], - ["container", "start", job], - ["container", "start", app], - ]); - expect((await readReceipt()).pendingOperation).toBeNull(); - stages("stop", "enter"); - expect( - await tryLegacyComposeAdoptedCommand({ - cwd: projectRoot, - operation: "down", - }) - ).toBe(0); - stages("stop", "settled"); - expect((await mutationCommands()).slice(-3)).toEqual([ - ["container", "stop", app], - ["container", "stop", job], - ["container", "stop", ID], - ]); - } finally { - restoreEnv("HACK_CONFIG_COMPILER_BINARY", priorCompiler); - await store.close(); - } - }); }, // Cover prepare/publish and separate bounded up/down operations without changing their deadlines. - 60_000 + caseBudgetMs(60_000) ); boundedTest.each([ @@ -733,110 +857,106 @@ boundedTest.each([ ] as const)( "public static7 observer refuses %s with only closed diagnostics and keeps pending", async (reason) => { - await publicStatic7Case(async () => { - await maintainedCompletedJobFixture(); - const binary = await maintainedJobCompiler(); - const { store, generation } = await prepared(binary); - const priorCompiler = process.env.HACK_CONFIG_COMPILER_BINARY; + await maintainedCompletedJobFixture(); + const binary = await maintainedJobCompiler(); + const { store, generation } = await prepared(binary); + const priorCompiler = process.env.HACK_CONFIG_COMPILER_BINARY; + try { + process.env.HACK_CONFIG_COMPILER_BINARY = binary; + await store.publish({ generation, binary }); + fixture.jobObservation = reason; + await save(); + let rejection: unknown; try { - process.env.HACK_CONFIG_COMPILER_BINARY = binary; - await store.publish({ generation, binary }); - fixture.jobObservation = reason; - await save(); - let rejection: unknown; - try { - await tryLegacyComposeAdoptedCommand({ - cwd: projectRoot, - operation: "up", - detach: true, - }); - } catch (error: unknown) { - rejection = error; - } - expect(rejection).toBeInstanceOf(HackCliError); - if (!(rejection instanceof HackCliError)) { - throw new Error("missing public refusal"); - } - expect(rejection.code).toBe("E_CONFIG_INVALID"); - expect(rejection.detail).toEqual({ - legacy_adoption_refusal: { - stage: "ordered-observation", - reason: reason === "probe" ? "probe-child" : reason, - }, + await tryLegacyComposeAdoptedCommand({ + cwd: projectRoot, + operation: "up", + detach: true, }); - expect( - JSON.stringify({ - message: rejection.message, - detail: rejection.detail, - }) - ).not.toContain(CANARY); - expect((await readReceipt()).pendingOperation.operation).toBe("start"); - expect( - (await mutationCommands()).some( - (args) => args[2] === "e".repeat(64) || args[2] === "c".repeat(64) - ) - ).toBe(false); - } finally { - restoreEnv("HACK_CONFIG_COMPILER_BINARY", priorCompiler); - await store.close(); + } catch (error: unknown) { + rejection = error; } - }); + expect(rejection).toBeInstanceOf(HackCliError); + if (!(rejection instanceof HackCliError)) { + throw new Error("missing public refusal"); + } + expect(rejection.code).toBe("E_CONFIG_INVALID"); + expect(rejection.detail).toEqual({ + legacy_adoption_refusal: { + stage: "ordered-observation", + reason: reason === "probe" ? "probe-child" : reason, + }, + }); + expect( + JSON.stringify({ + message: rejection.message, + detail: rejection.detail, + }) + ).not.toContain(CANARY); + expect((await readReceipt()).pendingOperation.operation).toBe("start"); + expect( + (await mutationCommands()).some( + (args) => args[2] === "e".repeat(64) || args[2] === "c".repeat(64) + ) + ).toBe(false); + } finally { + restoreEnv("HACK_CONFIG_COMPILER_BINARY", priorCompiler); + await store.close(); + } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( "public static7 scheduler classifies actual nonzero job exit without starting its dependent", async () => { - await publicStatic7Case(async () => { - const { job, app } = await maintainedCompletedJobFixture(); - const binary = await maintainedJobCompiler(); - const { store, generation } = await prepared(binary); - const priorCompiler = process.env.HACK_CONFIG_COMPILER_BINARY; + const { job, app } = await maintainedCompletedJobFixture(); + const binary = await maintainedJobCompiler(); + const { store, generation } = await prepared(binary); + const priorCompiler = process.env.HACK_CONFIG_COMPILER_BINARY; + try { + process.env.HACK_CONFIG_COMPILER_BINARY = binary; + await store.publish({ generation, binary }); + const jobState = fixture.jobs?.[job]; + if (!jobState) { + throw new Error("missing job state"); + } + jobState.exitCode = 17; + await save(); + let rejection: unknown; try { - process.env.HACK_CONFIG_COMPILER_BINARY = binary; - await store.publish({ generation, binary }); - const jobState = fixture.jobs?.[job]; - if (!jobState) { - throw new Error("missing job state"); - } - jobState.exitCode = 17; - await save(); - let rejection: unknown; - try { - await tryLegacyComposeAdoptedCommand({ - cwd: projectRoot, - operation: "up", - detach: true, - }); - } catch (error: unknown) { - rejection = error; - } - expect(rejection).toBeInstanceOf(HackCliError); - if (!(rejection instanceof HackCliError)) { - throw new Error("missing public refusal"); - } - expect(rejection.detail).toEqual({ - legacy_adoption_refusal: { - stage: "ordered-scheduler", - reason: "job-failed", - }, + await tryLegacyComposeAdoptedCommand({ + cwd: projectRoot, + operation: "up", + detach: true, }); - expect(await mutationCommands()).toEqual([ - ["container", "start", ID], - ["container", "start", job], - ]); - expect((await mutationCommands()).some((args) => args[2] === app)).toBe( - false - ); - expect((await readReceipt()).pendingOperation.operation).toBe("start"); - } finally { - restoreEnv("HACK_CONFIG_COMPILER_BINARY", priorCompiler); - await store.close(); + } catch (error: unknown) { + rejection = error; } - }); + expect(rejection).toBeInstanceOf(HackCliError); + if (!(rejection instanceof HackCliError)) { + throw new Error("missing public refusal"); + } + expect(rejection.detail).toEqual({ + legacy_adoption_refusal: { + stage: "ordered-scheduler", + reason: "job-failed", + }, + }); + expect(await mutationCommands()).toEqual([ + ["container", "start", ID], + ["container", "start", job], + ]); + expect((await mutationCommands()).some((args) => args[2] === app)).toBe( + false + ); + expect((await readReceipt()).pendingOperation.operation).toBe("start"); + } finally { + restoreEnv("HACK_CONFIG_COMPILER_BINARY", priorCompiler); + await store.close(); + } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest.each(["start", "restart", "stop"] as const)( @@ -855,7 +975,7 @@ boundedTest.each(["start", "restart", "stop"] as const)( fixture.states = { [ID]: true, [app]: true, [job]: false }; await save(); } - const deadline = Date.now() + 20_000; + const deadline = Date.now() + OPERATION_DEADLINE_MS; expect( await store.withMutation({ generation, @@ -914,7 +1034,7 @@ boundedTest.each(["start", "restart", "stop"] as const)( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -931,7 +1051,7 @@ boundedTest( binary, operation: "start", services: [], - deadline: Date.now() + 20_000, + deadline: Date.now() + OPERATION_DEADLINE_MS, run: async () => { fixture.states = { [ID]: true, [app]: true, [job]: false }; await save(); @@ -941,7 +1061,7 @@ boundedTest( ).rejects.toThrow("values omitted"); expect((await readReceipt()).pendingOperation.operation).toBe("start"); expect(await mutationCommands()).toEqual([]); - const deadline = Date.now() + 20_000; + const deadline = Date.now() + OPERATION_DEADLINE_MS; expect( await store.withMutation({ generation, @@ -969,7 +1089,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -985,7 +1105,7 @@ boundedTest( try { const binary = await compiler(); await store.publish({ generation, binary }); - const deadline = Date.now() + 20_000; + const deadline = Date.now() + OPERATION_DEADLINE_MS; await expect( store.withMutation({ generation, @@ -1012,7 +1132,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -1024,7 +1144,7 @@ boundedTest( const binary = await compiler(); await store.publish({ generation, binary }); let previous: LegacyComposeRetainedOutcome | undefined; - const deadline = Date.now() + 20_000; + const deadline = Date.now() + OPERATION_DEADLINE_MS; expect( await store.withMutation({ generation, @@ -1051,7 +1171,7 @@ boundedTest( binary, operation: "start", services: [], - deadline: Date.now() + 20_000, + deadline: Date.now() + OPERATION_DEADLINE_MS, run: async () => { if (!previous) { throw new Error("missing completion"); @@ -1066,7 +1186,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -1077,7 +1197,7 @@ boundedTest( try { const binary = await compiler(); await store.publish({ generation, binary }); - const deadline = Date.now() + 20_000; + const deadline = Date.now() + OPERATION_DEADLINE_MS; await expect( store.withMutation({ generation, @@ -1107,7 +1227,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -1122,7 +1242,7 @@ boundedTest( join(projectRoot, ".hack/hack.project.json"), "utf8" ); - const deadline = Date.now() + 20_000; + const deadline = Date.now() + OPERATION_DEADLINE_MS; await refusal( store.withMutation({ generation, @@ -1156,7 +1276,7 @@ boundedTest( fixture.jobFinalRace = false; await save(); await writeFile(join(projectRoot, ".hack/hack.project.json"), original); - const recoveryDeadline = Date.now() + 20_000; + const recoveryDeadline = Date.now() + OPERATION_DEADLINE_MS; expect( await store.withMutation({ generation, @@ -1178,7 +1298,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -1189,7 +1309,7 @@ boundedTest( try { const binary = await compiler(); await store.publish({ generation, binary }); - const deadline = Date.now() + 20_000; + const deadline = Date.now() + OPERATION_DEADLINE_MS; await expect( store.withMutation({ generation, @@ -1225,7 +1345,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -1252,7 +1372,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -1265,7 +1385,7 @@ boundedTest( const controller = new AbortController(); try { const binary = await compiler(), - deadline = Date.now() + 20_000; + deadline = Date.now() + OPERATION_DEADLINE_MS; expect( await store.withPreparationStop({ generation, @@ -1295,7 +1415,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -1303,10 +1423,11 @@ boundedTest( async () => { await dependencyFixture(); const { store, generation } = await prepared(); + let restoreClock: () => void = () => undefined; try { const binary = await compiler(); await store.publish({ generation, binary }); - const deadline = Date.now() + 5000; + const deadline = Date.now() + EXPIRY_DEADLINE_MS; await refusal( store.withMutation({ generation, @@ -1315,7 +1436,11 @@ boundedTest( services: [], deadline, run: async () => { - await Bun.sleep(Math.max(1, deadline - Date.now() + 5)); + // The callback overruns the deadline: move the clock past it + // instead of sleeping until it. + restoreClock = shiftProductClock( + Math.max(0, deadline - Date.now() + 5) + ); return 0; }, }), @@ -1324,10 +1449,11 @@ boundedTest( ); expect((await readReceipt()).pendingOperation.operation).toBe("stop"); } finally { + restoreClock(); await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -1368,7 +1494,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); for (const linked of [false, true]) { @@ -1452,7 +1578,7 @@ for (const linked of [false, true]) { stage("closed"); } }, - 30_000 + caseBudgetMs(30_000) ); } @@ -1462,10 +1588,11 @@ for (const stage of ["per-effect freshness", "final observation"] as const) { async () => { await dependencyFixture(); const { store, generation } = await prepared(); + let restoreClock: () => void = () => undefined; try { const binary = await compiler(); await store.publish({ generation, binary }); - const deadline = Date.now() + 5000; + const deadline = Date.now() + EXPIRY_DEADLINE_MS; let reachedCallback = false; await refusal( store.withMutation({ @@ -1479,6 +1606,11 @@ for (const stage of ["per-effect freshness", "final observation"] as const) { fixture.mode = stage === "per-effect freshness" ? "hang" : "hang-state"; await save(); + // Leave only the lead before the deadline, rather than waiting + // out the whole deadline. Never move the clock back. + restoreClock = shiftProductClock( + Math.max(0, deadline - Date.now() - EXPIRY_LEAD_MS[stage]) + ); if (stage === "per-effect freshness") { await input.assertFresh(); } @@ -1504,10 +1636,11 @@ for (const stage of ["per-effect freshness", "final observation"] as const) { expect(await mutationCommands()).toEqual([]); expect(fixture.volume[0]?.createdAt).toBe(CREATED); } finally { + restoreClock(); await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); } @@ -1532,7 +1665,7 @@ boundedTest( expect(await mutationCommands()).toEqual([]); expect(fixture.volume[0]?.createdAt).toBe(CREATED); }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -1565,7 +1698,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -1626,7 +1759,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -1640,7 +1773,7 @@ boundedTest( let calls = 0; await refusal( store.withMutation({ - deadline: Date.now() + 20_000, + deadline: Date.now() + OPERATION_DEADLINE_MS, generation, binary, operation: "start", @@ -1656,7 +1789,7 @@ boundedTest( expect((await readReceipt()).pendingOperation).toBeNull(); let retained: (() => Promise) | undefined; await store.withMutation({ - deadline: Date.now() + 20_000, + deadline: Date.now() + OPERATION_DEADLINE_MS, generation, binary, operation: "stop", @@ -1679,7 +1812,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -1729,7 +1862,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -1742,7 +1875,7 @@ boundedTest( await store.publish({ generation, binary }); await refusal( store.withMutation({ - deadline: Date.now() + 20_000, + deadline: Date.now() + OPERATION_DEADLINE_MS, generation, binary, operation: "start", @@ -1765,7 +1898,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -1784,7 +1917,7 @@ boundedTest( let effects = 0; await refusal( store.withMutation({ - deadline: Date.now() + 20_000, + deadline: Date.now() + OPERATION_DEADLINE_MS, generation, binary, operation: "start", @@ -1805,7 +1938,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -1823,7 +1956,7 @@ boundedTest( await store.publish({ generation, binary }); fixture.hangMutation = true; await save(); - const deadline = Date.now() + 20_000; + const deadline = Date.now() + OPERATION_DEADLINE_MS; let effectStartedAt = 0; await refusal( store.withMutation({ @@ -1839,7 +1972,7 @@ boundedTest( deadline, signal: controller.signal, }); - const readyUntil = Date.now() + 4000; + const readyUntil = Date.now() + testBudgetMs(4000); while ( !(await Bun.file(join(root, "effect-started")).exists()) && Date.now() < readyUntil @@ -1872,7 +2005,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest.skipIf(!process.env.HACK_TEST_OLD_ADOPTION_BINARY)( @@ -1900,7 +2033,7 @@ boundedTest.skipIf(!process.env.HACK_TEST_OLD_ADOPTION_BINARY)( stdin: "ignore", stdout: "pipe", stderr: "pipe", - timeout: 5000, + timeout: testBudgetMs(5000), } ); const [stdout, stderr, code] = await Promise.all([ @@ -1916,7 +2049,7 @@ boundedTest.skipIf(!process.env.HACK_TEST_OLD_ADOPTION_BINARY)( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); /** Current writers keep references at rest; "literal" models a file an earlier @@ -1984,7 +2117,7 @@ async function trackedManagedCheckout() { ]) { const child = Bun.spawn(["/usr/bin/git", "-C", projectRoot, ...args], { env: { - PATH: priorPath ?? "/usr/bin:/bin", + PATH: FILE_ENV.PATH ?? "/usr/bin:/bin", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", }, @@ -2272,7 +2405,7 @@ boundedTest( "active" ); }, - 90_000 + caseBudgetMs(90_000) ); boundedTest( @@ -2436,7 +2569,7 @@ boundedTest( }) ).toMatchObject({ removed: true, adoptionBound: false }); }, - 90_000 + caseBudgetMs(90_000) ); test("v3 value-free env override binds the engine config hash computed with the values (HACK-1273)", async () => { @@ -2644,7 +2777,7 @@ test("v4 typed locals preserve generated bytes, exact original IDs and key-free stdin: "ignore", stdout: "pipe", stderr: "pipe", - timeout: 5000, + timeout: testBudgetMs(5000), } ); const [stdout, stderr, code] = await Promise.all([ @@ -2824,7 +2957,7 @@ boundedTest( await store.close(); } }, - 240_000 + caseBudgetMs(240_000) ); boundedTest( @@ -2887,7 +3020,7 @@ boundedTest( await store.close(); } }, - 60_000 + caseBudgetMs(60_000) ); async function artifactPath(name: string) { const receipt = JSON.parse( @@ -2923,7 +3056,7 @@ async function linkedCheckout() { const git = async (args: readonly string[]) => { const child = Bun.spawn(["/usr/bin/git", "-C", projectRoot, ...args], { env: { - PATH: priorPath ?? "/usr/bin:/bin", + PATH: FILE_ENV.PATH ?? "/usr/bin:/bin", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", }, @@ -3063,7 +3196,7 @@ test("v13 linked default pins the physical branch originals and explicit selecti stdout: "ignore", stderr: "ignore", env: { - PATH: priorPath ?? "/usr/bin:/bin", + PATH: FILE_ENV.PATH ?? "/usr/bin:/bin", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", }, @@ -3492,7 +3625,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); test("v10 saved bridge and health owner rejects other contract receipts", async () => { @@ -3580,7 +3713,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); boundedTest( @@ -3615,7 +3748,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); test("rolled-back v10 can reprepare the same owned bridge without health under v6", async () => { @@ -3785,7 +3918,7 @@ boundedTest( await store.close(); } }, - 30_000 + caseBudgetMs(30_000) ); test("v6 rollback permits a later plain owner without inheriting topology authority", async () => { diff --git a/tests/native-process-policy-initial-trace.test.ts b/tests/native-process-policy-initial-trace.test.ts index 9439adfc3..124f30a6d 100644 --- a/tests/native-process-policy-initial-trace.test.ts +++ b/tests/native-process-policy-initial-trace.test.ts @@ -1,4 +1,4 @@ -import { expect, test } from "bun:test"; +import { test as bunTest, expect } from "bun:test"; import { chmod, mkdtemp, @@ -33,6 +33,25 @@ import { classifyProcessPolicyMemberKeys, } from "./e2e/native-process-policy-startup-diagnostic.ts"; import { restoreEnv } from "./helpers/env.ts"; +import { testBudgetMs } from "./helpers/test-budget.ts"; + +/** Bun's own default case timeout, used as the unloaded time of a case that names none. */ +const BUN_DEFAULT_CASE_TIMEOUT_MS = 5000; + +/** + * A case whose last argument is the time it needs on an unloaded host. The Bun + * timeout is that time scaled by `testBudgetMs`, since these cases cold-start + * Bun children: trace forwarders, stub engines and replay children. A case + * that names no time gets Bun's default, scaled the same way, so no case here + * keeps an unscaled 5 s budget. + */ +function test( + name: string, + run: () => void | Promise, + unloadedTimeoutMs = BUN_DEFAULT_CASE_TIMEOUT_MS +): void { + bunTest(name, run, testBudgetMs(unloadedTimeoutMs)); +} const CANARY = "synthetic-unknown-docker-stdout-stderr-canary"; const EMPTY_MEMBER_KEYS = { hex64: 0, epPrefixed: 0, other: 0 } as const; @@ -114,7 +133,7 @@ const FORWARDED_DOCKER_EXEC_TIMEOUT_MS = 30_000; /** Four forwarded commands at their exec bound, plus trace reads and rewrites. */ const INITIAL_FORWARDING_TEST_TIMEOUT_MS = 150_000; -test( +bunTest( "initial forwarding preserves bytes/exits but records no unknown output, args or stderr", async () => { const root = await mkdtemp(join(tmpdir(), "process-initial-forwarding-")); @@ -252,7 +271,12 @@ test("initial forwarding preserves a multicall Docker entry and refuses its repl await symlink(target, entry); process.env.PATH = root; expect( - (await exec([target], { stdin: "ignore", timeoutMs: 3000 })).exitCode + ( + await exec([target], { + stdin: "ignore", + timeoutMs: FORWARDED_DOCKER_EXEC_TIMEOUT_MS, + }) + ).exitCode ).toBe(79); const prepared = await prepareProcessPolicyInitialTrace({ directory: traceRoot, @@ -261,7 +285,7 @@ test("initial forwarding preserves a multicall Docker entry and refuses its repl await exec([join(traceRoot, "docker"), "version"], { env: { PATH: prepared.path }, stdin: "ignore", - timeoutMs: 3000, + timeoutMs: FORWARDED_DOCKER_EXEC_TIMEOUT_MS, }); expect(await run()).toEqual({ exitCode: 0, @@ -312,7 +336,10 @@ test("malformed UTF-8 keeps the caller's exact bytes/exit and makes replay unava expect(await new Response(child.stderr).text()).toBe(""); process.env.PATH = prepared.path; await expect( - assertNativeComposeOwned({ ...selection, timeoutMs: 3000 }) + assertNativeComposeOwned({ + ...selection, + timeoutMs: FORWARDED_DOCKER_EXEC_TIMEOUT_MS, + }) ).rejects.toMatchObject({ code: "E_NATIVE_COMPOSE_PROBE" }); await expect( readProcessPolicyInitialTrace(prepared.directory) diff --git a/tests/test-budget.test.ts b/tests/test-budget.test.ts new file mode 100644 index 000000000..c0026608a --- /dev/null +++ b/tests/test-budget.test.ts @@ -0,0 +1,44 @@ +import { expect, test } from "bun:test"; +import { DEFAULT_FILE_TIMEOUT_SECONDS } from "../scripts/test-shard.ts"; +import { + CASE_TIMEOUT_CEILING_MS, + SHARD_FILE_TIMEOUT_MS, + TEST_BUDGET_SCALE, + testBudgetMs, + withinCaseCeilingMs, +} from "./helpers/test-budget.ts"; + +test("the case ceiling follows the hosted shard's per-file limit", () => { + expect(SHARD_FILE_TIMEOUT_MS).toBe(DEFAULT_FILE_TIMEOUT_SECONDS * 1000); + // The slowest hosted file took up to 417 s; it plus one hung case must end + // before the shard stops the file, so Bun names that case. + expect(417_000 + CASE_TIMEOUT_CEILING_MS).toBeLessThan(SHARD_FILE_TIMEOUT_MS); +}); + +test("test budgets scale the unloaded time and stop at the ceiling", () => { + expect(testBudgetMs(250)).toBe(250 * TEST_BUDGET_SCALE); + expect(testBudgetMs(1_000_000)).toBe(CASE_TIMEOUT_CEILING_MS); + expect(testBudgetMs(1_000_000, { ceilingMs: 10_000 })).toBe(10_000); + expect(testBudgetMs(100, { ceilingMs: 10_000 })).toBe( + Math.min(100 * TEST_BUDGET_SCALE, 10_000) + ); +}); + +test("test budgets refuse invalid times and ceilings above the case ceiling", () => { + for (const unloadedMs of [0, -1, 1.5, Number.NaN]) { + expect(() => testBudgetMs(unloadedMs)).toThrow(/^E_TEST_BUDGET_INVALID/); + } + expect(() => + testBudgetMs(1000, { ceilingMs: CASE_TIMEOUT_CEILING_MS + 1 }) + ).toThrow(/^E_TEST_BUDGET_INVALID/); +}); + +test("a composed case timeout must fit the case ceiling", () => { + expect(withinCaseCeilingMs(CASE_TIMEOUT_CEILING_MS)).toBe( + CASE_TIMEOUT_CEILING_MS + ); + expect(() => withinCaseCeilingMs(CASE_TIMEOUT_CEILING_MS + 1)).toThrow( + /^E_TEST_BUDGET_CEILING/ + ); + expect(() => withinCaseCeilingMs(0)).toThrow(/^E_TEST_BUDGET_CEILING/); +});