From ed8d8f13d6a896a091b505268e9a82aa53814c4e Mon Sep 17 00:00:00 2001 From: Andre Sutherland Date: Thu, 27 Aug 2026 11:05:01 +0100 Subject: [PATCH 1/6] docs(runtime-libs): make the documented examples compile and hold Seven doc examples across three crates did not compile, so nothing had ever checked them and they had drifted from what they document. A workspace test run stops at the first crate that fails, which is why only two of them were ever visible. The signature examples named values that were never bound and a type that was never imported. One of them then asserted that a signature header starts with the key id, when it starts with the key id under a name, so it failed once it could run at all. The helpers examples called a method without the trait that provides it in scope, and referred to the function and type they document without importing either. The jsonpath one also asserted that injecting a value returned false while asserting the value had been injected, which contradicts itself and the sentence above it. The workflow examples named a crate that does not exist, called asynchronous methods without awaiting them and used a service nothing had bound. Its payload had drifted furthest, missing a field and giving three others types they have never had, since nothing was compiling it as the struct changed underneath. Every example is now a compiled one, with the setup a reader does not need hidden and the assertions saying what the code actually does. Co-Authored-By: Claude Opus 5 (1M context) --- libs/runtime/helpers/src/jsonpath.rs | 7 ++- libs/runtime/helpers/src/time.rs | 4 +- libs/runtime/signature/src/sigv1.rs | 23 ++++++--- .../workflow/src/workflow_executions.rs | 48 ++++++++++++++----- 4 files changed, 58 insertions(+), 24 deletions(-) diff --git a/libs/runtime/helpers/src/jsonpath.rs b/libs/runtime/helpers/src/jsonpath.rs index 3648c209..04f0b1f6 100644 --- a/libs/runtime/helpers/src/jsonpath.rs +++ b/libs/runtime/helpers/src/jsonpath.rs @@ -12,8 +12,10 @@ use serde_json::Value; /// /// An example of how you might use this function: /// ``` +/// # use celerity_helpers::jsonpath::jsonpath_inject_root; +/// # use jsonpath_rust::JsonPath; /// # use serde_json::json; -/// +/// # use std::str::FromStr; /// let mut json = json!({ /// "name": "John Doe", /// }); @@ -26,7 +28,8 @@ use serde_json::Value; /// "planPrice": 99.99, /// }); /// let result = jsonpath_inject_root(&path, &mut json, value); -/// assert_eq!(result, false); +/// // True, since the path names a field of the root object. +/// assert!(result); /// assert_eq!(json, json!({ /// "name": "John Doe", /// "planInfo": { diff --git a/libs/runtime/helpers/src/time.rs b/libs/runtime/helpers/src/time.rs index 2f98f861..84fba6e9 100644 --- a/libs/runtime/helpers/src/time.rs +++ b/libs/runtime/helpers/src/time.rs @@ -23,9 +23,7 @@ impl Debug for dyn Clock + Send + Sync { /// # Examples /// /// ``` -/// # use celerity_helpers::time::DefaultClock; -/// # use std::time::SystemTime; -/// +/// # use celerity_helpers::time::{Clock, DefaultClock}; /// let clock = DefaultClock::new(); /// let now = clock.now(); /// ``` diff --git a/libs/runtime/signature/src/sigv1.rs b/libs/runtime/signature/src/sigv1.rs index 579303db..b98d8b46 100644 --- a/libs/runtime/signature/src/sigv1.rs +++ b/libs/runtime/signature/src/sigv1.rs @@ -32,8 +32,15 @@ pub const DEFAULT_CLOCK_SKEW: u64 = 300; /// /// ``` /// # use bluelink_signature::sigv1::*; +/// # use bluelink_signature::types::KeyPair; /// # use celerity_helpers::time::DefaultClock; -/// +/// # use http::HeaderMap; +/// # use std::collections::HashMap; +/// # let key_pairs = HashMap::from([( +/// # "key-id".to_string(), +/// # KeyPair { key_id: "key-id".to_string(), secret_key: "secret-key".to_string() }, +/// # )]); +/// # let headers = HeaderMap::new(); /// let clock = DefaultClock::new(); /// // Uses the default clock skew of 5 minutes. /// match verify_signature(&key_pairs, &headers, &clock, None) { @@ -99,21 +106,25 @@ pub fn verify_signature( /// /// ``` /// # use bluelink_signature::sigv1::*; +/// # use bluelink_signature::types::KeyPair; /// # use celerity_helpers::time::DefaultClock; /// # use http::HeaderMap; -/// /// let key_pair = KeyPair { -/// key_id: "key-id".to_string(), -/// secret_key: "secret-key".to_string() +/// key_id: "key-id".to_string(), +/// secret_key: "secret-key".to_string(), /// }; -/// let headers = HeaderMap::new(); +/// let mut headers = HeaderMap::new(); /// headers.insert("X-Custom-Header", "custom-value".parse().unwrap()); /// let custom_header_names = vec!["X-Custom-Header".to_string()]; /// let clock = DefaultClock::new(); /// let signature_header = create_signature_header(&key_pair, &mut headers, custom_header_names, &clock) /// .expect("signature header to be created without any issues"); /// -/// assert!(signature_header.starts_with("key-id")); +/// // keyId="key-id", headers="bluelink-date x-custom-header", signature="..." +/// assert!(signature_header.starts_with("keyId=\"key-id\"")); +/// assert!(signature_header.contains("x-custom-header")); +/// // A date header is added where the caller did not set one, since the +/// // signature covers it. /// assert!(headers.get("Bluelink-Date").is_some()); /// ``` pub fn create_signature_header( diff --git a/libs/runtime/workflow/src/workflow_executions.rs b/libs/runtime/workflow/src/workflow_executions.rs index 5f4ba43c..27ed8c8c 100644 --- a/libs/runtime/workflow/src/workflow_executions.rs +++ b/libs/runtime/workflow/src/workflow_executions.rs @@ -18,20 +18,30 @@ pub trait WorkflowExecutionService { /// /// The following is an example of how you might call this method: /// ``` - /// # use workflow::workflow_executions::{SaveWorkflowExecutionPayload, WorkflowExecutionService}; - /// + /// # use celerity_runtime_workflow::workflow_executions::{ + /// # SaveWorkflowExecutionPayload, WorkflowExecutionService, + /// # WorkflowExecutionServiceError, WorkflowExecutionState, WorkflowExecutionStatus, + /// # }; + /// # use serde_json::json; + /// # async fn example( + /// # service: &dyn WorkflowExecutionService, + /// # updated_states: Vec, + /// # ) -> Result<(), WorkflowExecutionServiceError> { /// let saved_workflow_execution = service.save_workflow_execution( - /// "7e4f50be-6ecf-4ab3-8974-f65964615c44".to_string(), + /// "7e4f50be-6ecf-4ab3-8974-f65964615c44".to_string(), /// SaveWorkflowExecutionPayload { + /// input: json!({"documentId": "doc-1"}), /// started: 1728232655000, /// completed: Some(1728232657000), - /// duration: Some(2), - /// status: "completed".to_string(), + /// duration: Some(2.0), + /// status: WorkflowExecutionStatus::Succeeded, /// status_detail: "Workflow execution completed successfully".to_string(), - /// current_state: "uploadProcessedDocument".to_string(), + /// current_state: Some("uploadProcessedDocument".to_string()), /// states: updated_states, /// }, - /// )?; + /// ).await?; + /// # Ok(()) + /// # } /// ``` async fn save_workflow_execution( &self, @@ -45,11 +55,17 @@ pub trait WorkflowExecutionService { /// /// The following is an example of how you might call this method: /// ``` - /// # use workflow::workflow_executions::WorkflowExecutionService; - /// + /// # use celerity_runtime_workflow::workflow_executions::{ + /// # WorkflowExecutionService, WorkflowExecutionServiceError, + /// # }; + /// # async fn example( + /// # service: &dyn WorkflowExecutionService, + /// # ) -> Result<(), WorkflowExecutionServiceError> { /// let workflow_execution = service.get_workflow_execution( /// "7e4f50be-6ecf-4ab3-8974-f65964615c44".to_string(), - /// )?; + /// ).await?; + /// # Ok(()) + /// # } /// ``` async fn get_workflow_execution( &self, @@ -64,9 +80,15 @@ pub trait WorkflowExecutionService { /// /// The following is an example of how you might call this method: /// ``` - /// # use workflow::workflow_executions::WorkflowExecutionService; - /// - /// let workflow_executions = service.get_latest_workflow_executions(10)?; + /// # use celerity_runtime_workflow::workflow_executions::{ + /// # WorkflowExecutionService, WorkflowExecutionServiceError, + /// # }; + /// # async fn example( + /// # service: &dyn WorkflowExecutionService, + /// # ) -> Result<(), WorkflowExecutionServiceError> { + /// let workflow_executions = service.get_latest_workflow_executions(10).await?; + /// # Ok(()) + /// # } /// ``` async fn get_latest_workflow_executions( &self, From 0b8e13285e6a1310415417b711f7e555dff48897 Mon Sep 17 00:00:00 2001 From: Andre Sutherland Date: Thu, 27 Aug 2026 11:05:16 +0100 Subject: [PATCH 2/6] test(lib-rt-consumer-sqs): bound the wait for the messages a test expects The test collected messages until it had counted three hundred iterations, with nothing bounding how long that took. A queue nothing had created, an endpoint nothing was listening on, or a consumer that had stopped all presented the same way, as a run that never returned and never said why. Counting iterations rather than messages made the other outcome no better. A closed channel spun through the remaining count and asserted on an empty collection, saying only that zero is not three hundred. It now counts what it collected, stops on a deadline or on the channel closing, and reports how many of the three hundred arrived along with the two things worth checking first. The deadline is generous, since the sender spaces its messages out and a loaded machine is not a failure. Co-Authored-By: Claude Opus 5 (1M context) --- .../consumer-sqs/tests/integration_test.rs | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/libs/runtime/consumers/consumer-sqs/tests/integration_test.rs b/libs/runtime/consumers/consumer-sqs/tests/integration_test.rs index f12a5250..b51138e0 100644 --- a/libs/runtime/consumers/consumer-sqs/tests/integration_test.rs +++ b/libs/runtime/consumers/consumer-sqs/tests/integration_test.rs @@ -70,20 +70,33 @@ async fn test_receive_messages_and_fire_message_handler() { let mut collected_messages = Vec::>::new(); - let mut i = 0; // 200 messages in batches (20 * 10) + 100 individual messages. - while i < 300 { + const EXPECTED_MESSAGES: usize = 300; + // Bounded so that a queue nothing created, an endpoint nothing is listening + // on, or a consumer that stopped, each fail with a count rather than + // waiting for messages that are not coming. Generous, since the sender + // spaces the individual messages out and a loaded machine is not a failure. + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + + while collected_messages.len() < EXPECTED_MESSAGES { tokio::select! { - msg = rx.recv() => { - if let Some(unwrapped) = msg { - collected_messages.push(unwrapped); - } + msg = rx.recv() => match msg { + Some(message) => collected_messages.push(message), + // The consumer stopped, so nothing further is coming and + // waiting out the deadline would only delay the report. + None => break, }, + _ = tokio::time::sleep_until(deadline) => break, } - i += 1 } - assert_eq!(collected_messages.len(), 300); + assert_eq!( + collected_messages.len(), + EXPECTED_MESSAGES, + "collected {} of {EXPECTED_MESSAGES} messages before the deadline, \ + check the consumer is running and the queue it polls exists", + collected_messages.len(), + ); for (i, message) in collected_messages.into_iter().enumerate() { // The first 200 messages were sent in batches, // the last 100 were sent individually. From 30d7b00c84cf41620d7358ec387b4d395d2afb72 Mon Sep 17 00:00:00 2001 From: Andre Sutherland Date: Thu, 27 Aug 2026 13:16:50 +0100 Subject: [PATCH 3/6] refactor(runtime-libs): drop the event result field nothing carried Every event result set a context to none and nothing ever read one, so the field was a line on each of the eight places that built a result and an optional value in the shape every SDK binding compiles against. Removed rather than filled in, since what it was for is not recorded anywhere and the two call modes both answer callers without it. Co-Authored-By: Claude Opus 5 (1M context) --- libs/runtime/core/src/consumer_handler.rs | 1 - libs/runtime/core/src/ipc_frames.rs | 1 - libs/runtime/core/src/types.rs | 2 -- libs/runtime/core/tests/integration_consumer_test.rs | 2 -- libs/runtime/sdk/node/src/consumer.rs | 2 -- libs/runtime/sdk/python/src/consumer.rs | 2 -- 6 files changed, 10 deletions(-) diff --git a/libs/runtime/core/src/consumer_handler.rs b/libs/runtime/core/src/consumer_handler.rs index 81f6751f..2542e6d2 100644 --- a/libs/runtime/core/src/consumer_handler.rs +++ b/libs/runtime/core/src/consumer_handler.rs @@ -936,7 +936,6 @@ mod tests { EventResult { event_id: event_id.to_string(), data, - context: None, } } diff --git a/libs/runtime/core/src/ipc_frames.rs b/libs/runtime/core/src/ipc_frames.rs index 53616481..38f5ecee 100644 --- a/libs/runtime/core/src/ipc_frames.rs +++ b/libs/runtime/core/src/ipc_frames.rs @@ -218,7 +218,6 @@ pub fn event_result_from_frame(result: proto::Result, waiting_for: &EventType) - EventResult { event_id: result.id, data, - context: None, } } diff --git a/libs/runtime/core/src/types.rs b/libs/runtime/core/src/types.rs index f9ca3d85..31f6ae1b 100644 --- a/libs/runtime/core/src/types.rs +++ b/libs/runtime/core/src/types.rs @@ -229,8 +229,6 @@ pub struct EventResult { #[serde(rename = "eventId")] pub event_id: String, pub data: EventResultData, - #[serde(skip_serializing_if = "Option::is_none")] - pub context: Option, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] diff --git a/libs/runtime/core/tests/integration_consumer_test.rs b/libs/runtime/core/tests/integration_consumer_test.rs index 1e379d4b..49c7493c 100644 --- a/libs/runtime/core/tests/integration_consumer_test.rs +++ b/libs/runtime/core/tests/integration_consumer_test.rs @@ -69,7 +69,6 @@ impl ConsumerEventHandler for MockConsumerEventHandler { Ok(EventResult { event_id: String::new(), data: success_result_data(), - context: None, }) } @@ -81,7 +80,6 @@ impl ConsumerEventHandler for MockConsumerEventHandler { Ok(EventResult { event_id: String::new(), data: success_schedule_result_data(), - context: None, }) } } diff --git a/libs/runtime/sdk/node/src/consumer.rs b/libs/runtime/sdk/node/src/consumer.rs index f9f16eaf..405f943b 100644 --- a/libs/runtime/sdk/node/src/consumer.rs +++ b/libs/runtime/sdk/node/src/consumer.rs @@ -200,7 +200,6 @@ fn js_result_to_consumer_event_result(result: JsEventResult, event_id: &str) -> .collect() }), }), - context: None, } } @@ -211,7 +210,6 @@ fn js_result_to_schedule_event_result(result: JsEventResult, event_id: &str) -> success: result.success, error_message: result.error_message, }), - context: None, } } diff --git a/libs/runtime/sdk/python/src/consumer.rs b/libs/runtime/sdk/python/src/consumer.rs index 399a07cb..f585ca10 100644 --- a/libs/runtime/sdk/python/src/consumer.rs +++ b/libs/runtime/sdk/python/src/consumer.rs @@ -226,7 +226,6 @@ fn py_obj_to_consumer_event_result(py_obj: Py, event_id: &str) -> PyResul success: result.success, failures, }), - context: None, }) }) } @@ -241,7 +240,6 @@ fn py_obj_to_schedule_event_result(py_obj: Py, event_id: &str) -> PyResul success: result.success, error_message: result.error_message.clone(), }), - context: None, }) }) } From bcd1cf3348b50808ada6baa9e119492edf85cbec Mon Sep 17 00:00:00 2001 From: Andre Sutherland Date: Thu, 27 Aug 2026 13:17:07 +0100 Subject: [PATCH 4/6] fix(lib-rt-consumer-redis): advance the offset once the outcome is recorded The offset moved before the result was examined and before any failure reached the dead letter queue. A process that stopped in between left the batch behind the offset with nothing anywhere recording it, and since this consumer reads by offset rather than through a consumer group, behind the offset is gone. It now moves last. Straight away where every message was processed, after the failures are written where a failure was reported, and not at all where that write did not succeed, so the batch is read again rather than passed over. Reading it again redelivers what did succeed, which is the cost of one watermark covering a whole batch and cheaper than losing the rest. There is no test for this. What it guards against is a process stopping between two calls to Redis, which the suite has no way to stage without injecting a fault into the connection, so the existing tests show the paths either side are intact and nothing more. Co-Authored-By: Claude Opus 5 (1M context) --- .../consumer-redis/src/message_consumer.rs | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/libs/runtime/consumers/consumer-redis/src/message_consumer.rs b/libs/runtime/consumers/consumer-redis/src/message_consumer.rs index 45c423a9..3244fb36 100644 --- a/libs/runtime/consumers/consumer-redis/src/message_consumer.rs +++ b/libs/runtime/consumers/consumer-redis/src/message_consumer.rs @@ -444,12 +444,6 @@ impl RedisMessageConsumer { } } - let update_last_message_id_result = - self.update_last_message_id(&last_message_id, conn).await; - if let Err(err) = update_last_message_id_result { - error!("failed to update last message ID: {err}"); - } - let failures = match result { Ok(_) => vec![], Err(error) => match error { @@ -488,13 +482,37 @@ impl RedisMessageConsumer { }, }; - if !failures.is_empty() { - if let Err(err) = self + // Moved before the offset advances, since this consumer reads by offset + // rather than through a consumer group. Advancing first and then dying, + // or failing to write to the dead letter queue, would put the batch + // behind the offset with no record of it anywhere. + let recorded = if failures.is_empty() { + true + } else { + match self .move_failed_messages_to_dlq(&failures, &messages, conn) .await { - error!("failed to move failed messages to DLQ: {err}"); + Ok(()) => true, + Err(err) => { + error!("failed to move failed messages to DLQ: {err}"); + false + } } + }; + + // Left where it is otherwise, so the batch is read again rather than + // passed over. That redelivers what did succeed, which is the cost of + // one watermark for the whole batch and cheaper than losing the rest. + if recorded { + if let Err(err) = self.update_last_message_id(&last_message_id, conn).await { + error!("failed to update last message ID: {err}"); + } + } else { + warn!( + "leaving the last message ID where it is, so the batch is read again rather \ + than passed over with nothing recording what became of it" + ); } } From fb7c507c2a4987f26b411b5a9a36c175321ac208 Mon Sep 17 00:00:00 2001 From: Andre Sutherland Date: Thu, 27 Aug 2026 13:38:22 +0100 Subject: [PATCH 5/6] fix(runtime-libs): make the test harness able to run one package Naming a package built a command cargo refuses, since the exclude it carries alongside is only allowed with a workspace selection. Getting past that, the features are the workspace's own and a single package does not have them, so the flag has never run anything. A named package now narrows which tests run rather than which packages are built, which resolves exactly the features a full run does and leaves the two passes comparable. The cluster pass is skipped where the named package has no Redis-backed tests, rather than running the same seven suites whatever was asked for. Co-Authored-By: Claude Opus 5 (1M context) --- libs/runtime/scripts/package-test-tools.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/libs/runtime/scripts/package-test-tools.py b/libs/runtime/scripts/package-test-tools.py index 1e929e04..897c1dc9 100644 --- a/libs/runtime/scripts/package-test-tools.py +++ b/libs/runtime/scripts/package-test-tools.py @@ -65,9 +65,13 @@ def _run_integration_tests(args: argparse.Namespace) -> None: env_copy = os.environ.copy() in_ci_env = os.environ.get('GITHUB_ACTIONS') test_env_file = '.env.test-ci' if in_ci_env else '.env.test' - selection_flags = f"--package {args.package}" if args.package else "--workspace" exclude_flags = "--exclude celerity-python-runtime-sdk" feature_flags = "--features celerity_local_consumers,celerity_runtime_core/ws_clustering" + # A named package narrows which tests run rather than which packages are + # built, so a scoped run resolves exactly the features a full one does. + # Selecting the package instead would take the workspace features with it, + # and most packages do not have them. + filter_flags = f"-E 'package({args.package})'" if args.package else "" test_env = { # Integration tests can't share the same env vars (.env) as the API running # in docker as the host endpoints for the AWS service mocks are different. @@ -83,14 +87,14 @@ def _run_integration_tests(args: argparse.Namespace) -> None: # Use cargo-nextest for cleaner per-test output with a summary at the end. # Debug output is only shown for failing tests. completed_process = subprocess.run( - f'cargo llvm-cov --no-report nextest {selection_flags} {exclude_flags} ' - f'{feature_flags} --no-fail-fast --color always', + f'cargo llvm-cov --no-report nextest --workspace {exclude_flags} ' + f'{feature_flags} {filter_flags} --no-fail-fast --color always', env=test_env, shell=True, check=False ) - cluster_process = _run_cluster_mode_tests(test_env) + cluster_process = _run_cluster_mode_tests(test_env, args.package) report_flags = "--lcov --output-path coverage.lcov" if in_ci_env else "--html" subprocess.run(f'cargo llvm-cov report {report_flags}', shell=True, check=False) @@ -111,7 +115,10 @@ def _run_integration_tests(args: argparse.Namespace) -> None: CLUSTER_MODE_PACKAGES = '--package celerity_ws_redis --package celerity_runtime_core' -def _run_cluster_mode_tests(test_env: Dict[str, str]) -> subprocess.CompletedProcess: +def _run_cluster_mode_tests( + test_env: Dict[str, str], + package: str | None, +) -> subprocess.CompletedProcess: """ Runs the Redis-backed tests a second time against a real cluster. @@ -124,6 +131,10 @@ def _run_cluster_mode_tests(test_env: Dict[str, str]) -> subprocess.CompletedPro first pass made, so this is a second or two rather than a pass over the whole workspace. """ + if package is not None and package not in CLUSTER_MODE_PACKAGES: + print(f'{package} has no Redis-backed tests, skipping the cluster pass ...') + return subprocess.CompletedProcess(args=[], returncode=0) + print('Running the Redis-backed tests again against a cluster ...') return subprocess.run( f'cargo llvm-cov --no-report nextest {CLUSTER_MODE_PACKAGES} ' From eb994b8b038894279872ee4ad3453571b77e735a Mon Sep 17 00:00:00 2001 From: Andre Sutherland Date: Thu, 27 Aug 2026 13:38:43 +0100 Subject: [PATCH 6/6] fix(lib-rt-core): send a batch of websocket messages per connection The runtime took a batch and sent it in one line, awaiting each message before starting the next. For a message asking its client to acknowledge it, that answer is a round trip to the client rather than a write to a socket, so every message behind it waited out the one before, and a client that never answered held up all of them until the runtime gave up on its message. That made a batch worth no more than the sends it replaced, which is most of the reason for carrying several messages in one frame. Messages are now grouped by the connection they are for and the groups are sent alongside each other, so one client being slow to answer costs only the messages meant for it. Within a connection they are still sent in the order the handler listed them, which is the one ordering a handler can reasonably expect and costs nothing to keep. Failures are put back into the order the handler gave before they are reported. The connections are served in no order at all, and the position a message had in the batch is the only thing the handler has to match a failure against, since the runtime generates the id where one was not supplied. The registry the tests stand up can hold a connection open, so one test shows a later message reaching a different client while the first is still waiting and another shows two messages for one client keeping their order. Co-Authored-By: Claude Opus 5 (1M context) --- libs/runtime/core/src/ipc_stream.rs | 146 ++++++++++++++++++++++++++-- 1 file changed, 139 insertions(+), 7 deletions(-) diff --git a/libs/runtime/core/src/ipc_stream.rs b/libs/runtime/core/src/ipc_stream.rs index d77474a7..685130c6 100644 --- a/libs/runtime/core/src/ipc_stream.rs +++ b/libs/runtime/core/src/ipc_stream.rs @@ -640,16 +640,58 @@ async fn send_frame( /// exactly those. Reporting only a single outcome for the batch would leave a /// handler no choice but to resend all of it, delivering the successful ones /// twice. +/// Delivers every message in a batch and reports what became of each. +/// +/// Messages for one connection are sent in the order the handler listed them, +/// and different connections do not wait on each other. That matters most for +/// a message asking its client to acknowledge it, where sending the batch in +/// one line would have every later message wait out the round trip of the one +/// before, and a client that never answers hold up all of them. async fn send_to_websockets( stream_id: StreamId, context: &StreamContext, send: proto::WsSend, ) -> proto::WsSendAck { let correlation_id = send.correlation_id; - let mut failures = Vec::new(); + // Keyed by connection, holding the position each message had in the batch, + // which is what a failure names and what the handler knows it by. + let mut by_connection: HashMap> = HashMap::new(); for (index, outbound) in send.messages.into_iter().enumerate() { - let index = index as u32; + by_connection + .entry(outbound.connection_id.clone()) + .or_default() + .push((index as u32, outbound)); + } + + let mut failures = futures::future::join_all( + by_connection + .into_values() + .map(|messages| send_to_one_connection(stream_id, context, messages)), + ) + .await + .concat(); + + // Reported in the order the handler sent them, since it has nothing else to + // match them against and the connections were served in no order at all. + failures.sort_by_key(|failure| failure.index); + + proto::WsSendAck { + correlation_id, + success: failures.is_empty(), + failures, + } +} + +/// Sends one connection's share of a batch, in the order it was given. +async fn send_to_one_connection( + stream_id: StreamId, + context: &StreamContext, + messages: Vec<(u32, proto::WsOutbound)>, +) -> Vec { + let mut failures = Vec::new(); + + for (index, outbound) in messages { let connection_id = outbound.connection_id.clone(); let (message_type, message) = match encode_outbound(&outbound) { Ok(encoded) => encoded, @@ -696,11 +738,7 @@ async fn send_to_websockets( } } - proto::WsSendAck { - correlation_id, - success: failures.is_empty(), - failures, - } + failures } /// Renders an outbound message the way the connection registry expects it. @@ -858,6 +896,10 @@ mod tests { /// acknowledgement the handler requested from one it did not. contexts: std::sync::Mutex>>, unreachable: HashSet, + /// A connection whose sends wait to be released, standing in for a + /// client that is slow to acknowledge. + held: Option, + release: Arc, } impl std::fmt::Display for RecordingWsRegistry { @@ -876,6 +918,9 @@ mod tests { message: String, ctx: Option, ) -> Result<(), celerity_ws_registry::errors::WebSocketConnError> { + if self.held.as_deref() == Some(connection_id.as_str()) { + self.release.notified().await; + } self.contexts.lock().unwrap().push(ctx); if self.unreachable.contains(&connection_id) { return Err( @@ -1327,6 +1372,93 @@ mod tests { assert!(ctx.wait_for_ack); } + /// A message asking its client to acknowledge it is not answered until that + /// client does, so a batch sent in one line would have every later message + /// wait out the round trip of the one before it. + #[tokio::test] + async fn one_connection_holding_up_a_batch_does_not_hold_up_the_others() { + let release = Arc::new(tokio::sync::Notify::new()); + let mut harness = start_with_registry( + &["schedule::a"], + RecordingWsRegistry { + held: Some("connection-slow".to_string()), + release: release.clone(), + ..RecordingWsRegistry::default() + }, + ); + let _registration = ready_stream(&mut harness).await; + + harness + .inbound_tx + .send(Ok(ws_send( + "correlation-1", + vec![ + outbound("connection-slow", b"{}", false), + outbound("connection-fast", b"{}", false), + ], + ))) + .await + .unwrap(); + + // The one behind it reaches its connection while the first is still + // waiting, which is the whole point of not sending them in sequence. + let sent_to_fast = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if harness + .ws_registry + .sent + .lock() + .unwrap() + .iter() + .any(|(connection_id, ..)| connection_id == "connection-fast") + { + return; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await; + assert!( + sent_to_fast.is_ok(), + "the second message waited for the first connection to answer" + ); + + release.notify_waiters(); + + let Some(proto::runtime_message::Frame::WsAck(ack)) = + next_frame(&mut harness.outbound_rx).await + else { + panic!("expected an acknowledgement for the batch"); + }; + assert!(ack.success, "failures = {:?}", ack.failures); + } + + /// Two messages for one connection arrive in the order the handler listed + /// them, which is the one ordering a handler can reasonably expect. + #[tokio::test] + async fn messages_for_one_connection_keep_the_order_they_were_given() { + let mut harness = start(&["schedule::a"]); + let _registration = ready_stream(&mut harness).await; + + harness + .inbound_tx + .send(Ok(ws_send( + "correlation-1", + vec![ + outbound("connection-1", b"{\"n\":1}", false), + outbound("connection-1", b"{\"n\":2}", false), + ], + ))) + .await + .unwrap(); + + let _ack = next_frame(&mut harness.outbound_rx).await; + + let sent = harness.ws_registry.sent.lock().unwrap(); + let bodies: Vec<&str> = sent.iter().map(|(_, _, _, body)| body.as_str()).collect(); + assert_eq!(bodies, vec!["{\"n\":1}", "{\"n\":2}"]); + } + /// Asking for neither an acknowledgement nor anyone informed is the common /// case, and it settles on the write with nothing to carry. #[tokio::test]