diff --git a/libs/runtime/consumers/consumer-redis/src/message_consumer.rs b/libs/runtime/consumers/consumer-redis/src/message_consumer.rs index 45c423a9..3244fb36 100644 --- a/libs/runtime/consumers/consumer-redis/src/message_consumer.rs +++ b/libs/runtime/consumers/consumer-redis/src/message_consumer.rs @@ -444,12 +444,6 @@ impl RedisMessageConsumer { } } - let update_last_message_id_result = - self.update_last_message_id(&last_message_id, conn).await; - if let Err(err) = update_last_message_id_result { - error!("failed to update last message ID: {err}"); - } - let failures = match result { Ok(_) => vec![], Err(error) => match error { @@ -488,13 +482,37 @@ impl RedisMessageConsumer { }, }; - if !failures.is_empty() { - if let Err(err) = self + // Moved before the offset advances, since this consumer reads by offset + // rather than through a consumer group. Advancing first and then dying, + // or failing to write to the dead letter queue, would put the batch + // behind the offset with no record of it anywhere. + let recorded = if failures.is_empty() { + true + } else { + match self .move_failed_messages_to_dlq(&failures, &messages, conn) .await { - error!("failed to move failed messages to DLQ: {err}"); + Ok(()) => true, + Err(err) => { + error!("failed to move failed messages to DLQ: {err}"); + false + } } + }; + + // Left where it is otherwise, so the batch is read again rather than + // passed over. That redelivers what did succeed, which is the cost of + // one watermark for the whole batch and cheaper than losing the rest. + if recorded { + if let Err(err) = self.update_last_message_id(&last_message_id, conn).await { + error!("failed to update last message ID: {err}"); + } + } else { + warn!( + "leaving the last message ID where it is, so the batch is read again rather \ + than passed over with nothing recording what became of it" + ); } } diff --git a/libs/runtime/consumers/consumer-sqs/tests/integration_test.rs b/libs/runtime/consumers/consumer-sqs/tests/integration_test.rs index f12a5250..b51138e0 100644 --- a/libs/runtime/consumers/consumer-sqs/tests/integration_test.rs +++ b/libs/runtime/consumers/consumer-sqs/tests/integration_test.rs @@ -70,20 +70,33 @@ async fn test_receive_messages_and_fire_message_handler() { let mut collected_messages = Vec::>::new(); - let mut i = 0; // 200 messages in batches (20 * 10) + 100 individual messages. - while i < 300 { + const EXPECTED_MESSAGES: usize = 300; + // Bounded so that a queue nothing created, an endpoint nothing is listening + // on, or a consumer that stopped, each fail with a count rather than + // waiting for messages that are not coming. Generous, since the sender + // spaces the individual messages out and a loaded machine is not a failure. + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + + while collected_messages.len() < EXPECTED_MESSAGES { tokio::select! { - msg = rx.recv() => { - if let Some(unwrapped) = msg { - collected_messages.push(unwrapped); - } + msg = rx.recv() => match msg { + Some(message) => collected_messages.push(message), + // The consumer stopped, so nothing further is coming and + // waiting out the deadline would only delay the report. + None => break, }, + _ = tokio::time::sleep_until(deadline) => break, } - i += 1 } - assert_eq!(collected_messages.len(), 300); + assert_eq!( + collected_messages.len(), + EXPECTED_MESSAGES, + "collected {} of {EXPECTED_MESSAGES} messages before the deadline, \ + check the consumer is running and the queue it polls exists", + collected_messages.len(), + ); for (i, message) in collected_messages.into_iter().enumerate() { // The first 200 messages were sent in batches, // the last 100 were sent individually. diff --git a/libs/runtime/core/src/consumer_handler.rs b/libs/runtime/core/src/consumer_handler.rs index 81f6751f..2542e6d2 100644 --- a/libs/runtime/core/src/consumer_handler.rs +++ b/libs/runtime/core/src/consumer_handler.rs @@ -936,7 +936,6 @@ mod tests { EventResult { event_id: event_id.to_string(), data, - context: None, } } diff --git a/libs/runtime/core/src/ipc_frames.rs b/libs/runtime/core/src/ipc_frames.rs index 53616481..38f5ecee 100644 --- a/libs/runtime/core/src/ipc_frames.rs +++ b/libs/runtime/core/src/ipc_frames.rs @@ -218,7 +218,6 @@ pub fn event_result_from_frame(result: proto::Result, waiting_for: &EventType) - EventResult { event_id: result.id, data, - context: None, } } diff --git a/libs/runtime/core/src/ipc_stream.rs b/libs/runtime/core/src/ipc_stream.rs index d77474a7..685130c6 100644 --- a/libs/runtime/core/src/ipc_stream.rs +++ b/libs/runtime/core/src/ipc_stream.rs @@ -640,16 +640,58 @@ async fn send_frame( /// exactly those. Reporting only a single outcome for the batch would leave a /// handler no choice but to resend all of it, delivering the successful ones /// twice. +/// Delivers every message in a batch and reports what became of each. +/// +/// Messages for one connection are sent in the order the handler listed them, +/// and different connections do not wait on each other. That matters most for +/// a message asking its client to acknowledge it, where sending the batch in +/// one line would have every later message wait out the round trip of the one +/// before, and a client that never answers hold up all of them. async fn send_to_websockets( stream_id: StreamId, context: &StreamContext, send: proto::WsSend, ) -> proto::WsSendAck { let correlation_id = send.correlation_id; - let mut failures = Vec::new(); + // Keyed by connection, holding the position each message had in the batch, + // which is what a failure names and what the handler knows it by. + let mut by_connection: HashMap> = HashMap::new(); for (index, outbound) in send.messages.into_iter().enumerate() { - let index = index as u32; + by_connection + .entry(outbound.connection_id.clone()) + .or_default() + .push((index as u32, outbound)); + } + + let mut failures = futures::future::join_all( + by_connection + .into_values() + .map(|messages| send_to_one_connection(stream_id, context, messages)), + ) + .await + .concat(); + + // Reported in the order the handler sent them, since it has nothing else to + // match them against and the connections were served in no order at all. + failures.sort_by_key(|failure| failure.index); + + proto::WsSendAck { + correlation_id, + success: failures.is_empty(), + failures, + } +} + +/// Sends one connection's share of a batch, in the order it was given. +async fn send_to_one_connection( + stream_id: StreamId, + context: &StreamContext, + messages: Vec<(u32, proto::WsOutbound)>, +) -> Vec { + let mut failures = Vec::new(); + + for (index, outbound) in messages { let connection_id = outbound.connection_id.clone(); let (message_type, message) = match encode_outbound(&outbound) { Ok(encoded) => encoded, @@ -696,11 +738,7 @@ async fn send_to_websockets( } } - proto::WsSendAck { - correlation_id, - success: failures.is_empty(), - failures, - } + failures } /// Renders an outbound message the way the connection registry expects it. @@ -858,6 +896,10 @@ mod tests { /// acknowledgement the handler requested from one it did not. contexts: std::sync::Mutex>>, unreachable: HashSet, + /// A connection whose sends wait to be released, standing in for a + /// client that is slow to acknowledge. + held: Option, + release: Arc, } impl std::fmt::Display for RecordingWsRegistry { @@ -876,6 +918,9 @@ mod tests { message: String, ctx: Option, ) -> Result<(), celerity_ws_registry::errors::WebSocketConnError> { + if self.held.as_deref() == Some(connection_id.as_str()) { + self.release.notified().await; + } self.contexts.lock().unwrap().push(ctx); if self.unreachable.contains(&connection_id) { return Err( @@ -1327,6 +1372,93 @@ mod tests { assert!(ctx.wait_for_ack); } + /// A message asking its client to acknowledge it is not answered until that + /// client does, so a batch sent in one line would have every later message + /// wait out the round trip of the one before it. + #[tokio::test] + async fn one_connection_holding_up_a_batch_does_not_hold_up_the_others() { + let release = Arc::new(tokio::sync::Notify::new()); + let mut harness = start_with_registry( + &["schedule::a"], + RecordingWsRegistry { + held: Some("connection-slow".to_string()), + release: release.clone(), + ..RecordingWsRegistry::default() + }, + ); + let _registration = ready_stream(&mut harness).await; + + harness + .inbound_tx + .send(Ok(ws_send( + "correlation-1", + vec![ + outbound("connection-slow", b"{}", false), + outbound("connection-fast", b"{}", false), + ], + ))) + .await + .unwrap(); + + // The one behind it reaches its connection while the first is still + // waiting, which is the whole point of not sending them in sequence. + let sent_to_fast = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if harness + .ws_registry + .sent + .lock() + .unwrap() + .iter() + .any(|(connection_id, ..)| connection_id == "connection-fast") + { + return; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await; + assert!( + sent_to_fast.is_ok(), + "the second message waited for the first connection to answer" + ); + + release.notify_waiters(); + + let Some(proto::runtime_message::Frame::WsAck(ack)) = + next_frame(&mut harness.outbound_rx).await + else { + panic!("expected an acknowledgement for the batch"); + }; + assert!(ack.success, "failures = {:?}", ack.failures); + } + + /// Two messages for one connection arrive in the order the handler listed + /// them, which is the one ordering a handler can reasonably expect. + #[tokio::test] + async fn messages_for_one_connection_keep_the_order_they_were_given() { + let mut harness = start(&["schedule::a"]); + let _registration = ready_stream(&mut harness).await; + + harness + .inbound_tx + .send(Ok(ws_send( + "correlation-1", + vec![ + outbound("connection-1", b"{\"n\":1}", false), + outbound("connection-1", b"{\"n\":2}", false), + ], + ))) + .await + .unwrap(); + + let _ack = next_frame(&mut harness.outbound_rx).await; + + let sent = harness.ws_registry.sent.lock().unwrap(); + let bodies: Vec<&str> = sent.iter().map(|(_, _, _, body)| body.as_str()).collect(); + assert_eq!(bodies, vec!["{\"n\":1}", "{\"n\":2}"]); + } + /// Asking for neither an acknowledgement nor anyone informed is the common /// case, and it settles on the write with nothing to carry. #[tokio::test] diff --git a/libs/runtime/core/src/types.rs b/libs/runtime/core/src/types.rs index f9ca3d85..31f6ae1b 100644 --- a/libs/runtime/core/src/types.rs +++ b/libs/runtime/core/src/types.rs @@ -229,8 +229,6 @@ pub struct EventResult { #[serde(rename = "eventId")] pub event_id: String, pub data: EventResultData, - #[serde(skip_serializing_if = "Option::is_none")] - pub context: Option, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] diff --git a/libs/runtime/core/tests/integration_consumer_test.rs b/libs/runtime/core/tests/integration_consumer_test.rs index 1e379d4b..49c7493c 100644 --- a/libs/runtime/core/tests/integration_consumer_test.rs +++ b/libs/runtime/core/tests/integration_consumer_test.rs @@ -69,7 +69,6 @@ impl ConsumerEventHandler for MockConsumerEventHandler { Ok(EventResult { event_id: String::new(), data: success_result_data(), - context: None, }) } @@ -81,7 +80,6 @@ impl ConsumerEventHandler for MockConsumerEventHandler { Ok(EventResult { event_id: String::new(), data: success_schedule_result_data(), - context: None, }) } } diff --git a/libs/runtime/helpers/src/jsonpath.rs b/libs/runtime/helpers/src/jsonpath.rs index 3648c209..04f0b1f6 100644 --- a/libs/runtime/helpers/src/jsonpath.rs +++ b/libs/runtime/helpers/src/jsonpath.rs @@ -12,8 +12,10 @@ use serde_json::Value; /// /// An example of how you might use this function: /// ``` +/// # use celerity_helpers::jsonpath::jsonpath_inject_root; +/// # use jsonpath_rust::JsonPath; /// # use serde_json::json; -/// +/// # use std::str::FromStr; /// let mut json = json!({ /// "name": "John Doe", /// }); @@ -26,7 +28,8 @@ use serde_json::Value; /// "planPrice": 99.99, /// }); /// let result = jsonpath_inject_root(&path, &mut json, value); -/// assert_eq!(result, false); +/// // True, since the path names a field of the root object. +/// assert!(result); /// assert_eq!(json, json!({ /// "name": "John Doe", /// "planInfo": { diff --git a/libs/runtime/helpers/src/time.rs b/libs/runtime/helpers/src/time.rs index 2f98f861..84fba6e9 100644 --- a/libs/runtime/helpers/src/time.rs +++ b/libs/runtime/helpers/src/time.rs @@ -23,9 +23,7 @@ impl Debug for dyn Clock + Send + Sync { /// # Examples /// /// ``` -/// # use celerity_helpers::time::DefaultClock; -/// # use std::time::SystemTime; -/// +/// # use celerity_helpers::time::{Clock, DefaultClock}; /// let clock = DefaultClock::new(); /// let now = clock.now(); /// ``` diff --git a/libs/runtime/scripts/package-test-tools.py b/libs/runtime/scripts/package-test-tools.py index 1e929e04..897c1dc9 100644 --- a/libs/runtime/scripts/package-test-tools.py +++ b/libs/runtime/scripts/package-test-tools.py @@ -65,9 +65,13 @@ def _run_integration_tests(args: argparse.Namespace) -> None: env_copy = os.environ.copy() in_ci_env = os.environ.get('GITHUB_ACTIONS') test_env_file = '.env.test-ci' if in_ci_env else '.env.test' - selection_flags = f"--package {args.package}" if args.package else "--workspace" exclude_flags = "--exclude celerity-python-runtime-sdk" feature_flags = "--features celerity_local_consumers,celerity_runtime_core/ws_clustering" + # A named package narrows which tests run rather than which packages are + # built, so a scoped run resolves exactly the features a full one does. + # Selecting the package instead would take the workspace features with it, + # and most packages do not have them. + filter_flags = f"-E 'package({args.package})'" if args.package else "" test_env = { # Integration tests can't share the same env vars (.env) as the API running # in docker as the host endpoints for the AWS service mocks are different. @@ -83,14 +87,14 @@ def _run_integration_tests(args: argparse.Namespace) -> None: # Use cargo-nextest for cleaner per-test output with a summary at the end. # Debug output is only shown for failing tests. completed_process = subprocess.run( - f'cargo llvm-cov --no-report nextest {selection_flags} {exclude_flags} ' - f'{feature_flags} --no-fail-fast --color always', + f'cargo llvm-cov --no-report nextest --workspace {exclude_flags} ' + f'{feature_flags} {filter_flags} --no-fail-fast --color always', env=test_env, shell=True, check=False ) - cluster_process = _run_cluster_mode_tests(test_env) + cluster_process = _run_cluster_mode_tests(test_env, args.package) report_flags = "--lcov --output-path coverage.lcov" if in_ci_env else "--html" subprocess.run(f'cargo llvm-cov report {report_flags}', shell=True, check=False) @@ -111,7 +115,10 @@ def _run_integration_tests(args: argparse.Namespace) -> None: CLUSTER_MODE_PACKAGES = '--package celerity_ws_redis --package celerity_runtime_core' -def _run_cluster_mode_tests(test_env: Dict[str, str]) -> subprocess.CompletedProcess: +def _run_cluster_mode_tests( + test_env: Dict[str, str], + package: str | None, +) -> subprocess.CompletedProcess: """ Runs the Redis-backed tests a second time against a real cluster. @@ -124,6 +131,10 @@ def _run_cluster_mode_tests(test_env: Dict[str, str]) -> subprocess.CompletedPro first pass made, so this is a second or two rather than a pass over the whole workspace. """ + if package is not None and package not in CLUSTER_MODE_PACKAGES: + print(f'{package} has no Redis-backed tests, skipping the cluster pass ...') + return subprocess.CompletedProcess(args=[], returncode=0) + print('Running the Redis-backed tests again against a cluster ...') return subprocess.run( f'cargo llvm-cov --no-report nextest {CLUSTER_MODE_PACKAGES} ' diff --git a/libs/runtime/sdk/node/src/consumer.rs b/libs/runtime/sdk/node/src/consumer.rs index f9f16eaf..405f943b 100644 --- a/libs/runtime/sdk/node/src/consumer.rs +++ b/libs/runtime/sdk/node/src/consumer.rs @@ -200,7 +200,6 @@ fn js_result_to_consumer_event_result(result: JsEventResult, event_id: &str) -> .collect() }), }), - context: None, } } @@ -211,7 +210,6 @@ fn js_result_to_schedule_event_result(result: JsEventResult, event_id: &str) -> success: result.success, error_message: result.error_message, }), - context: None, } } diff --git a/libs/runtime/sdk/python/src/consumer.rs b/libs/runtime/sdk/python/src/consumer.rs index 399a07cb..f585ca10 100644 --- a/libs/runtime/sdk/python/src/consumer.rs +++ b/libs/runtime/sdk/python/src/consumer.rs @@ -226,7 +226,6 @@ fn py_obj_to_consumer_event_result(py_obj: Py, event_id: &str) -> PyResul success: result.success, failures, }), - context: None, }) }) } @@ -241,7 +240,6 @@ fn py_obj_to_schedule_event_result(py_obj: Py, event_id: &str) -> PyResul success: result.success, error_message: result.error_message.clone(), }), - context: None, }) }) } diff --git a/libs/runtime/signature/src/sigv1.rs b/libs/runtime/signature/src/sigv1.rs index 579303db..b98d8b46 100644 --- a/libs/runtime/signature/src/sigv1.rs +++ b/libs/runtime/signature/src/sigv1.rs @@ -32,8 +32,15 @@ pub const DEFAULT_CLOCK_SKEW: u64 = 300; /// /// ``` /// # use bluelink_signature::sigv1::*; +/// # use bluelink_signature::types::KeyPair; /// # use celerity_helpers::time::DefaultClock; -/// +/// # use http::HeaderMap; +/// # use std::collections::HashMap; +/// # let key_pairs = HashMap::from([( +/// # "key-id".to_string(), +/// # KeyPair { key_id: "key-id".to_string(), secret_key: "secret-key".to_string() }, +/// # )]); +/// # let headers = HeaderMap::new(); /// let clock = DefaultClock::new(); /// // Uses the default clock skew of 5 minutes. /// match verify_signature(&key_pairs, &headers, &clock, None) { @@ -99,21 +106,25 @@ pub fn verify_signature( /// /// ``` /// # use bluelink_signature::sigv1::*; +/// # use bluelink_signature::types::KeyPair; /// # use celerity_helpers::time::DefaultClock; /// # use http::HeaderMap; -/// /// let key_pair = KeyPair { -/// key_id: "key-id".to_string(), -/// secret_key: "secret-key".to_string() +/// key_id: "key-id".to_string(), +/// secret_key: "secret-key".to_string(), /// }; -/// let headers = HeaderMap::new(); +/// let mut headers = HeaderMap::new(); /// headers.insert("X-Custom-Header", "custom-value".parse().unwrap()); /// let custom_header_names = vec!["X-Custom-Header".to_string()]; /// let clock = DefaultClock::new(); /// let signature_header = create_signature_header(&key_pair, &mut headers, custom_header_names, &clock) /// .expect("signature header to be created without any issues"); /// -/// assert!(signature_header.starts_with("key-id")); +/// // keyId="key-id", headers="bluelink-date x-custom-header", signature="..." +/// assert!(signature_header.starts_with("keyId=\"key-id\"")); +/// assert!(signature_header.contains("x-custom-header")); +/// // A date header is added where the caller did not set one, since the +/// // signature covers it. /// assert!(headers.get("Bluelink-Date").is_some()); /// ``` pub fn create_signature_header( diff --git a/libs/runtime/workflow/src/workflow_executions.rs b/libs/runtime/workflow/src/workflow_executions.rs index 5f4ba43c..27ed8c8c 100644 --- a/libs/runtime/workflow/src/workflow_executions.rs +++ b/libs/runtime/workflow/src/workflow_executions.rs @@ -18,20 +18,30 @@ pub trait WorkflowExecutionService { /// /// The following is an example of how you might call this method: /// ``` - /// # use workflow::workflow_executions::{SaveWorkflowExecutionPayload, WorkflowExecutionService}; - /// + /// # use celerity_runtime_workflow::workflow_executions::{ + /// # SaveWorkflowExecutionPayload, WorkflowExecutionService, + /// # WorkflowExecutionServiceError, WorkflowExecutionState, WorkflowExecutionStatus, + /// # }; + /// # use serde_json::json; + /// # async fn example( + /// # service: &dyn WorkflowExecutionService, + /// # updated_states: Vec, + /// # ) -> Result<(), WorkflowExecutionServiceError> { /// let saved_workflow_execution = service.save_workflow_execution( - /// "7e4f50be-6ecf-4ab3-8974-f65964615c44".to_string(), + /// "7e4f50be-6ecf-4ab3-8974-f65964615c44".to_string(), /// SaveWorkflowExecutionPayload { + /// input: json!({"documentId": "doc-1"}), /// started: 1728232655000, /// completed: Some(1728232657000), - /// duration: Some(2), - /// status: "completed".to_string(), + /// duration: Some(2.0), + /// status: WorkflowExecutionStatus::Succeeded, /// status_detail: "Workflow execution completed successfully".to_string(), - /// current_state: "uploadProcessedDocument".to_string(), + /// current_state: Some("uploadProcessedDocument".to_string()), /// states: updated_states, /// }, - /// )?; + /// ).await?; + /// # Ok(()) + /// # } /// ``` async fn save_workflow_execution( &self, @@ -45,11 +55,17 @@ pub trait WorkflowExecutionService { /// /// The following is an example of how you might call this method: /// ``` - /// # use workflow::workflow_executions::WorkflowExecutionService; - /// + /// # use celerity_runtime_workflow::workflow_executions::{ + /// # WorkflowExecutionService, WorkflowExecutionServiceError, + /// # }; + /// # async fn example( + /// # service: &dyn WorkflowExecutionService, + /// # ) -> Result<(), WorkflowExecutionServiceError> { /// let workflow_execution = service.get_workflow_execution( /// "7e4f50be-6ecf-4ab3-8974-f65964615c44".to_string(), - /// )?; + /// ).await?; + /// # Ok(()) + /// # } /// ``` async fn get_workflow_execution( &self, @@ -64,9 +80,15 @@ pub trait WorkflowExecutionService { /// /// The following is an example of how you might call this method: /// ``` - /// # use workflow::workflow_executions::WorkflowExecutionService; - /// - /// let workflow_executions = service.get_latest_workflow_executions(10)?; + /// # use celerity_runtime_workflow::workflow_executions::{ + /// # WorkflowExecutionService, WorkflowExecutionServiceError, + /// # }; + /// # async fn example( + /// # service: &dyn WorkflowExecutionService, + /// # ) -> Result<(), WorkflowExecutionServiceError> { + /// let workflow_executions = service.get_latest_workflow_executions(10).await?; + /// # Ok(()) + /// # } /// ``` async fn get_latest_workflow_executions( &self,