From 768b9a85ca443e48d4716fb24afb6ab146455c28 Mon Sep 17 00:00:00 2001 From: Andre Sutherland Date: Thu, 27 Aug 2026 09:58:33 +0100 Subject: [PATCH 1/3] docs(lib-rt-ws-registry): correct what waiting for an acknowledgement means The comment said this was only used when broadcasting to other nodes in a cluster, which the work that made a send mean the same thing wherever the connection is had already made untrue. A message to a client on this node is waited for the same way, which was the point of it. It now says that, and says what a caller can otherwise get wrong, that a message which does not ask its client to acknowledge it is settled by the write, since nothing was asked and no answer is coming. Co-Authored-By: Claude Opus 5 (1M context) --- libs/runtime/ws/ws-registry/src/registry.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/libs/runtime/ws/ws-registry/src/registry.rs b/libs/runtime/ws/ws-registry/src/registry.rs index 906c9e10..ea2334a4 100644 --- a/libs/runtime/ws/ws-registry/src/registry.rs +++ b/libs/runtime/ws/ws-registry/src/registry.rs @@ -93,11 +93,14 @@ pub struct SendContext { // If a message is considered lost, the caller will be included in the message sent // to the clients in the inform_clients list. pub caller: Option, - // Whether to wait for an acknowledgement from the node that has the connection - // that the message was sent for, a WebSocketConnError::MessageLost error will be returned - // for the caller to handle the case where the message was lost - // when the wait_for_ack flag is set to true. - // This is only used when broadcasting messages to other nodes in a cluster. + // Whether to wait for the client to acknowledge the message, returning + // WebSocketConnError::MessageLost where it never does. + // + // Waited for wherever the connection is, on this node or another one, so a + // send means the same thing in a single node deployment as in a cluster. + // + // A message that does not ask its client to acknowledge is settled by + // the write, since nothing was asked and no answer is coming. pub wait_for_ack: bool, // The connection IDs of clients that should be informed if a message is lost // (an acknowledgement was not received after the maximum number of retries). From 3ed067e6d8f12da58f118a772a0e52f4f3ed1f56 Mon Sep 17 00:00:00 2001 From: Andre Sutherland Date: Thu, 27 Aug 2026 09:59:06 +0100 Subject: [PATCH 2/3] feat(runtime-libs): let a handler ask for a client acknowledgement over ipc A handler running as a separate process could not ask for one. The contract carried the clients to inform if a message was lost and the caller to name in that event, and nothing for the acknowledgement those exist alongside, so the runtime built the context with the request hard coded off. Everything the two branches before this built, waiting for the client, sending again while attempts remain and declaring a message lost, was reachable only by a caller holding the bindings directly. The message carries the request now, and the runtime builds a context where a handler asked for either that or clients informed, rather than only the latter. The acknowledgement a handler receives for the send then reports what its client made of the message rather than what the socket made of it. What a handler can get wrong is recorded on the field. The message has to ask its client for an acknowledgement in the payload it composes, which the runtime does not write, so asking for one on a message that asks nothing waits until the runtime gives up on it. The stand-in registry in the tests keeps what each send asked for, so the test covers the path from the frame to the registry rather than the decision alone, which is where this was wrong. Co-Authored-By: Claude Opus 5 (1M context) --- .../core/src/generated/celerity.runtime.v1.rs | 13 +++ libs/runtime/core/src/ipc_stream.rs | 86 +++++++++++++++++-- .../proto/celerity/runtime/v1/runtime.proto | 12 +++ libs/runtime/ws/ws-registry/src/registry.rs | 2 +- 4 files changed, 107 insertions(+), 6 deletions(-) diff --git a/libs/runtime/core/src/generated/celerity.runtime.v1.rs b/libs/runtime/core/src/generated/celerity.runtime.v1.rs index dbcd735b..c64984b7 100644 --- a/libs/runtime/core/src/generated/celerity.runtime.v1.rs +++ b/libs/runtime/core/src/generated/celerity.runtime.v1.rs @@ -577,6 +577,19 @@ pub struct WsOutbound { /// meaningful alongside inform_clients_on_loss. #[prost(string, tag = "6")] pub caller: ::prost::alloc::string::String, + /// Whether the WsSendAck for this message waits for the client to acknowledge + /// it, rather than reporting the write to the socket. + /// + /// The message has to ask the client for an acknowledgement in the payload it + /// carries, which the runtime does not compose. Setting this for one that + /// asks nothing leaves the send waiting for an answer that is never coming, + /// until the runtime declares the message lost. + /// + /// Waited for wherever the connection is, so a send means the same thing in a + /// single node deployment as in a cluster. Sending it again while attempts + /// remain, and declaring it lost when they run out, is the runtime's to do. + #[prost(bool, tag = "7")] + pub wait_for_ack: bool, } /// Reports what happened to each message in a WsSend. /// diff --git a/libs/runtime/core/src/ipc_stream.rs b/libs/runtime/core/src/ipc_stream.rs index e481fe21..d77474a7 100644 --- a/libs/runtime/core/src/ipc_stream.rs +++ b/libs/runtime/core/src/ipc_stream.rs @@ -679,7 +679,11 @@ async fn send_to_websockets( message_id, message_type, message, - send_context(outbound.caller, outbound.inform_clients_on_loss), + send_context( + outbound.caller, + outbound.inform_clients_on_loss, + outbound.wait_for_ack, + ), ) .await { @@ -722,14 +726,23 @@ fn encode_outbound(outbound: &proto::WsOutbound) -> Result<(MessageType, String) /// context is produced without them. Acknowledgements are not waited for, since /// the handler has already been told the send was accepted and holding the /// stream for a cluster round trip would delay every message behind it. -fn send_context(caller: String, inform_clients: Vec) -> Option { - if inform_clients.is_empty() { +/// What the registry needs beyond the message itself, where the handler asked +/// for anything at all. +/// +/// Absent where it asked for neither, which is the common case and the one +/// that settles on the write. +fn send_context( + caller: String, + inform_clients: Vec, + wait_for_ack: bool, +) -> Option { + if !wait_for_ack && inform_clients.is_empty() { return None; } Some(SendContext { caller: (!caller.is_empty()).then_some(caller), inform_clients, - wait_for_ack: false, + wait_for_ack, }) } @@ -841,6 +854,9 @@ mod tests { #[derive(Debug, Default)] struct RecordingWsRegistry { sent: std::sync::Mutex>, + /// What each send asked for beyond the message, so a test can tell an + /// acknowledgement the handler requested from one it did not. + contexts: std::sync::Mutex>>, unreachable: HashSet, } @@ -858,8 +874,9 @@ mod tests { message_id: String, message_type: MessageType, message: String, - _: Option, + ctx: Option, ) -> Result<(), celerity_ws_registry::errors::WebSocketConnError> { + self.contexts.lock().unwrap().push(ctx); if self.unreachable.contains(&connection_id) { return Err( celerity_ws_registry::errors::WebSocketConnError::MessageLost(message_id), @@ -1265,6 +1282,13 @@ mod tests { } } + fn outbound_asking_for_ack(connection_id: &str, message: &[u8]) -> proto::WsOutbound { + proto::WsOutbound { + wait_for_ack: true, + ..outbound(connection_id, message, false) + } + } + fn outbound(connection_id: &str, message: &[u8], is_binary: bool) -> proto::WsOutbound { proto::WsOutbound { connection_id: connection_id.to_string(), @@ -1273,9 +1297,61 @@ mod tests { inform_clients_on_loss: vec![], message_id: String::new(), caller: String::new(), + wait_for_ack: false, } } + /// The runtime waits for the client, sends the message again while attempts + /// remain and declares it lost when they run out, none of which happens + /// unless the request reaches the registry. + #[tokio::test] + async fn carries_a_requested_client_acknowledgement_to_the_registry() { + let mut harness = start(&["schedule::a"]); + let _registration = ready_stream(&mut harness).await; + + harness + .inbound_tx + .send(Ok(ws_send( + "correlation-1", + vec![outbound_asking_for_ack("connection-1", b"{}")], + ))) + .await + .unwrap(); + + let _ack = next_frame(&mut harness.outbound_rx).await; + + let contexts = harness.ws_registry.contexts.lock().unwrap(); + let Some(Some(ctx)) = contexts.first() else { + panic!("the send asked for an acknowledgement and reached the registry without one"); + }; + assert!(ctx.wait_for_ack); + } + + /// Asking for neither an acknowledgement nor anyone informed is the common + /// case, and it settles on the write with nothing to carry. + #[tokio::test] + async fn carries_nothing_for_a_message_that_asked_for_nothing() { + let mut harness = start(&["schedule::a"]); + let _registration = ready_stream(&mut harness).await; + + harness + .inbound_tx + .send(Ok(ws_send( + "correlation-1", + vec![outbound("connection-1", b"{}", false)], + ))) + .await + .unwrap(); + + let _ack = next_frame(&mut harness.outbound_rx).await; + + let contexts = harness.ws_registry.contexts.lock().unwrap(); + assert!( + matches!(contexts.first(), Some(None)), + "contexts = {contexts:?}" + ); + } + #[tokio::test] async fn delivers_a_websocket_message_a_handler_sent() { let mut harness = start(&["schedule::a"]); diff --git a/libs/runtime/proto/celerity/runtime/v1/runtime.proto b/libs/runtime/proto/celerity/runtime/v1/runtime.proto index 72991602..63e6d9dc 100644 --- a/libs/runtime/proto/celerity/runtime/v1/runtime.proto +++ b/libs/runtime/proto/celerity/runtime/v1/runtime.proto @@ -394,6 +394,18 @@ message WsOutbound { // loss event so an informed client can tell who was being replied to. Only // meaningful alongside inform_clients_on_loss. string caller = 6; + // Whether the WsSendAck for this message waits for the client to acknowledge + // it, rather than reporting the write to the socket. + // + // The message has to ask the client for an acknowledgement in the payload it + // carries, which the runtime does not compose. Setting this for one that + // asks nothing leaves the send waiting for an answer that is never coming, + // until the runtime declares the message lost. + // + // Waited for wherever the connection is, so a send means the same thing in a + // single node deployment as in a cluster. Sending it again while attempts + // remain, and declaring it lost when they run out, is the runtime's to do. + bool wait_for_ack = 7; } // Reports what happened to each message in a WsSend. diff --git a/libs/runtime/ws/ws-registry/src/registry.rs b/libs/runtime/ws/ws-registry/src/registry.rs index ea2334a4..0419ce86 100644 --- a/libs/runtime/ws/ws-registry/src/registry.rs +++ b/libs/runtime/ws/ws-registry/src/registry.rs @@ -86,7 +86,7 @@ pub trait NodeLivenessStore: Send + Sync + Debug { } // Additional context for sending messages to a connection in a WebSocket registry. -#[derive(Default)] +#[derive(Debug, Default)] pub struct SendContext { // The caller that is sending the message. // This is useful for providing context about the purpose or origin of the message. From 2fadb638bb216f184b839f20660fd7b7d5cf00d6 Mon Sep 17 00:00:00 2001 From: Andre Sutherland Date: Thu, 27 Aug 2026 10:03:05 +0100 Subject: [PATCH 3/3] docs(runtime-libs): say what a consumer result decides in the sdk bindings Both bindings described the shape a handler returns and not what returning it causes. For a source that acknowledges, the result decides whether the message is delivered again, which is the one thing a handler author needs to know before answering, and neither binding said it. They now record that a success acknowledges the message and a failure leaves it on its source, that reporting success for work a handler did not do means never seeing that work again, that naming failures answers for each message in a batch separately while naming none answers for the whole of it, and that a non-empty list is taken as the answer even alongside a success. The id a failure names has to be the one the message arrived with. One matching nothing in the batch would leave nothing behind, so the runtime refuses the whole answer rather than acknowledging a batch it cannot apply. Co-Authored-By: Claude Opus 5 (1M context) --- libs/runtime/sdk/node/index.d.ts | 37 ++++++++++++++++--- libs/runtime/sdk/node/src/consumer.rs | 25 ++++++++++++- .../_celerity_runtime_sdk.pyi | 19 +++++++++- libs/runtime/sdk/python/src/consumer.rs | 18 +++++++++ 4 files changed, 90 insertions(+), 9 deletions(-) diff --git a/libs/runtime/sdk/node/index.d.ts b/libs/runtime/sdk/node/index.d.ts index 6289e25c..df1c64cc 100644 --- a/libs/runtime/sdk/node/index.d.ts +++ b/libs/runtime/sdk/node/index.d.ts @@ -650,13 +650,34 @@ export interface JsConsumerMessage { vendor: any } -/** The result returned from a consumer or schedule event handler. */ +/** + * The result returned from a consumer or schedule event handler. + * + * For a source that acknowledges, a queue, a topic or a schedule, this decides + * what happens to the message rather than only reporting what happened. A + * success acknowledges it and it is not delivered again; a failure leaves it + * on its source, which redelivers it according to its own rules. + */ export interface JsEventResult { - /** Whether the event was processed successfully. */ + /** + * Whether the event was processed successfully. + * + * Reporting success for work the handler did not do means never seeing that + * work again. + */ success: boolean /** - * Optional list of individual message processing failures - * (for partial failure reporting in consumer handlers). + * The messages in a batch that could not be processed. + * + * Naming messages is how a handler answers for each one separately, those + * named are left on the source to be delivered again and the rest are + * acknowledged, so a handler that processed most of a batch does not have + * all of it sent back. Failing without naming any answers for the whole + * batch, and none of it is acknowledged. + * + * A non-empty list is taken as the answer even alongside `success: true`. + * Ignored for schedule handlers, which receive one message and have nothing + * to name. */ failures?: Array /** Optional error message (used for schedule handler failures). */ @@ -665,7 +686,13 @@ export interface JsEventResult { /** Represents a failure to process an individual message in a consumer batch. */ export interface JsMessageProcessingFailure { - /** The ID of the message that failed to process. */ + /** + * The ID of the message that failed to process. + * + * Must be the `messageId` the message arrived with. A message ID matching + * nothing in the batch would leave nothing behind, so the runtime refuses + * the whole answer rather than acknowledging a batch it cannot apply. + */ messageId: string /** Optional description of the error. */ errorMessage?: string diff --git a/libs/runtime/sdk/node/src/consumer.rs b/libs/runtime/sdk/node/src/consumer.rs index 8992cc1e..f9f16eaf 100644 --- a/libs/runtime/sdk/node/src/consumer.rs +++ b/libs/runtime/sdk/node/src/consumer.rs @@ -99,12 +99,29 @@ pub struct JsScheduleEventInput { } /// The result returned from a consumer or schedule event handler. +/// +/// For a source that acknowledges, a queue, a topic or a schedule, this decides +/// what happens to the message rather than only reporting what happened. A +/// success acknowledges it and it is not delivered again; a failure leaves it +/// on its source, which redelivers it according to its own rules. #[napi(object)] pub struct JsEventResult { /// Whether the event was processed successfully. + /// + /// Reporting success for work the handler did not do means never seeing that + /// work again. pub success: bool, - /// Optional list of individual message processing failures - /// (for partial failure reporting in consumer handlers). + /// The messages in a batch that could not be processed. + /// + /// Naming messages is how a handler answers for each one separately, those + /// named are left on the source to be delivered again and the rest are + /// acknowledged, so a handler that processed most of a batch does not have + /// all of it sent back. Failing without naming any answers for the whole + /// batch, and none of it is acknowledged. + /// + /// A non-empty list is taken as the answer even alongside `success: true`. + /// Ignored for schedule handlers, which receive one message and have nothing + /// to name. pub failures: Option>, /// Optional error message (used for schedule handler failures). pub error_message: Option, @@ -114,6 +131,10 @@ pub struct JsEventResult { #[napi(object)] pub struct JsMessageProcessingFailure { /// The ID of the message that failed to process. + /// + /// Must be the `messageId` the message arrived with. A message ID matching + /// nothing in the batch would leave nothing behind, so the runtime refuses + /// the whole answer rather than acknowledging a batch it cannot apply. pub message_id: String, /// Optional description of the error. pub error_message: Option, diff --git a/libs/runtime/sdk/python/python/celerity_runtime_sdk/_celerity_runtime_sdk.pyi b/libs/runtime/sdk/python/python/celerity_runtime_sdk/_celerity_runtime_sdk.pyi index 7cc7b550..163c05b0 100644 --- a/libs/runtime/sdk/python/python/celerity_runtime_sdk/_celerity_runtime_sdk.pyi +++ b/libs/runtime/sdk/python/python/celerity_runtime_sdk/_celerity_runtime_sdk.pyi @@ -1433,10 +1433,21 @@ class EventResult: """ The result returned from a consumer or schedule event handler. + For a source that acknowledges, a queue, a topic or a schedule, this decides + what happens to the message rather than only reporting what happened. A + success acknowledges it and it is not delivered again; a failure leaves it + on its source, which redelivers it according to its own rules. Reporting + success for work the handler did not do means never seeing that work again. + + Naming failures is how a handler answers for each message in a batch + separately, those named are left on the source and the rest are + acknowledged. Failing without naming any answers for the whole batch. A + non-empty list is taken as the answer even alongside a success, and is + ignored for schedule handlers, which receive one message. + Attributes: success: Whether the event was processed successfully. - failures: Optional list of individual message processing failures - (for partial failure reporting in consumer handlers). + failures: The messages in a batch that could not be processed. error_message: Optional error message (used for schedule handler failures). """ success: bool @@ -1463,6 +1474,10 @@ class MessageProcessingFailure: """ Represents a failure to process an individual message in a consumer batch. + The message id must be the one the message arrived with. An id matching + nothing in the batch would leave nothing behind, so the runtime refuses the + whole answer rather than acknowledging a batch it cannot apply. + Attributes: message_id: The ID of the message that failed to process. error_message: Optional description of the error. diff --git a/libs/runtime/sdk/python/src/consumer.rs b/libs/runtime/sdk/python/src/consumer.rs index d89712de..399a07cb 100644 --- a/libs/runtime/sdk/python/src/consumer.rs +++ b/libs/runtime/sdk/python/src/consumer.rs @@ -72,6 +72,19 @@ pub struct PyScheduleEventInput { pub trace_context: Option>, } +/// The result returned from a consumer or schedule event handler. +/// +/// For a source that acknowledges, a queue, a topic or a schedule, this decides +/// what happens to the message rather than only reporting what happened. A +/// success acknowledges it and it is not delivered again; a failure leaves it +/// on its source, which redelivers it according to its own rules. Reporting +/// success for work the handler did not do means never seeing that work again. +/// +/// Naming failures is how a handler answers for each message in a batch +/// separately, those named are left on the source and the rest are +/// acknowledged. Failing without naming any answers for the whole batch. A +/// non-empty list is taken as the answer even alongside a success, and is +/// ignored for schedule handlers, which receive one message. #[pyclass(name = "EventResult")] pub struct PyEventResult { #[pyo3(get, set)] @@ -99,6 +112,11 @@ impl PyEventResult { } } +/// One message in a batch that could not be processed. +/// +/// The message id must be the one the message arrived with. An id matching +/// nothing in the batch would leave nothing behind, so the runtime refuses the +/// whole answer rather than acknowledging a batch it cannot apply. #[pyclass(name = "MessageProcessingFailure")] pub struct PyMessageProcessingFailure { #[pyo3(get, set)]