diff --git a/libs/runtime/core/src/generated/celerity.runtime.v1.rs b/libs/runtime/core/src/generated/celerity.runtime.v1.rs index dbcd735b..c64984b7 100644 --- a/libs/runtime/core/src/generated/celerity.runtime.v1.rs +++ b/libs/runtime/core/src/generated/celerity.runtime.v1.rs @@ -577,6 +577,19 @@ pub struct WsOutbound { /// meaningful alongside inform_clients_on_loss. #[prost(string, tag = "6")] pub caller: ::prost::alloc::string::String, + /// Whether the WsSendAck for this message waits for the client to acknowledge + /// it, rather than reporting the write to the socket. + /// + /// The message has to ask the client for an acknowledgement in the payload it + /// carries, which the runtime does not compose. Setting this for one that + /// asks nothing leaves the send waiting for an answer that is never coming, + /// until the runtime declares the message lost. + /// + /// Waited for wherever the connection is, so a send means the same thing in a + /// single node deployment as in a cluster. Sending it again while attempts + /// remain, and declaring it lost when they run out, is the runtime's to do. + #[prost(bool, tag = "7")] + pub wait_for_ack: bool, } /// Reports what happened to each message in a WsSend. /// diff --git a/libs/runtime/core/src/ipc_stream.rs b/libs/runtime/core/src/ipc_stream.rs index e481fe21..d77474a7 100644 --- a/libs/runtime/core/src/ipc_stream.rs +++ b/libs/runtime/core/src/ipc_stream.rs @@ -679,7 +679,11 @@ async fn send_to_websockets( message_id, message_type, message, - send_context(outbound.caller, outbound.inform_clients_on_loss), + send_context( + outbound.caller, + outbound.inform_clients_on_loss, + outbound.wait_for_ack, + ), ) .await { @@ -722,14 +726,23 @@ fn encode_outbound(outbound: &proto::WsOutbound) -> Result<(MessageType, String) /// context is produced without them. Acknowledgements are not waited for, since /// the handler has already been told the send was accepted and holding the /// stream for a cluster round trip would delay every message behind it. -fn send_context(caller: String, inform_clients: Vec) -> Option { - if inform_clients.is_empty() { +/// What the registry needs beyond the message itself, where the handler asked +/// for anything at all. +/// +/// Absent where it asked for neither, which is the common case and the one +/// that settles on the write. +fn send_context( + caller: String, + inform_clients: Vec, + wait_for_ack: bool, +) -> Option { + if !wait_for_ack && inform_clients.is_empty() { return None; } Some(SendContext { caller: (!caller.is_empty()).then_some(caller), inform_clients, - wait_for_ack: false, + wait_for_ack, }) } @@ -841,6 +854,9 @@ mod tests { #[derive(Debug, Default)] struct RecordingWsRegistry { sent: std::sync::Mutex>, + /// What each send asked for beyond the message, so a test can tell an + /// acknowledgement the handler requested from one it did not. + contexts: std::sync::Mutex>>, unreachable: HashSet, } @@ -858,8 +874,9 @@ mod tests { message_id: String, message_type: MessageType, message: String, - _: Option, + ctx: Option, ) -> Result<(), celerity_ws_registry::errors::WebSocketConnError> { + self.contexts.lock().unwrap().push(ctx); if self.unreachable.contains(&connection_id) { return Err( celerity_ws_registry::errors::WebSocketConnError::MessageLost(message_id), @@ -1265,6 +1282,13 @@ mod tests { } } + fn outbound_asking_for_ack(connection_id: &str, message: &[u8]) -> proto::WsOutbound { + proto::WsOutbound { + wait_for_ack: true, + ..outbound(connection_id, message, false) + } + } + fn outbound(connection_id: &str, message: &[u8], is_binary: bool) -> proto::WsOutbound { proto::WsOutbound { connection_id: connection_id.to_string(), @@ -1273,9 +1297,61 @@ mod tests { inform_clients_on_loss: vec![], message_id: String::new(), caller: String::new(), + wait_for_ack: false, } } + /// The runtime waits for the client, sends the message again while attempts + /// remain and declares it lost when they run out, none of which happens + /// unless the request reaches the registry. + #[tokio::test] + async fn carries_a_requested_client_acknowledgement_to_the_registry() { + let mut harness = start(&["schedule::a"]); + let _registration = ready_stream(&mut harness).await; + + harness + .inbound_tx + .send(Ok(ws_send( + "correlation-1", + vec![outbound_asking_for_ack("connection-1", b"{}")], + ))) + .await + .unwrap(); + + let _ack = next_frame(&mut harness.outbound_rx).await; + + let contexts = harness.ws_registry.contexts.lock().unwrap(); + let Some(Some(ctx)) = contexts.first() else { + panic!("the send asked for an acknowledgement and reached the registry without one"); + }; + assert!(ctx.wait_for_ack); + } + + /// Asking for neither an acknowledgement nor anyone informed is the common + /// case, and it settles on the write with nothing to carry. + #[tokio::test] + async fn carries_nothing_for_a_message_that_asked_for_nothing() { + let mut harness = start(&["schedule::a"]); + let _registration = ready_stream(&mut harness).await; + + harness + .inbound_tx + .send(Ok(ws_send( + "correlation-1", + vec![outbound("connection-1", b"{}", false)], + ))) + .await + .unwrap(); + + let _ack = next_frame(&mut harness.outbound_rx).await; + + let contexts = harness.ws_registry.contexts.lock().unwrap(); + assert!( + matches!(contexts.first(), Some(None)), + "contexts = {contexts:?}" + ); + } + #[tokio::test] async fn delivers_a_websocket_message_a_handler_sent() { let mut harness = start(&["schedule::a"]); diff --git a/libs/runtime/proto/celerity/runtime/v1/runtime.proto b/libs/runtime/proto/celerity/runtime/v1/runtime.proto index 72991602..63e6d9dc 100644 --- a/libs/runtime/proto/celerity/runtime/v1/runtime.proto +++ b/libs/runtime/proto/celerity/runtime/v1/runtime.proto @@ -394,6 +394,18 @@ message WsOutbound { // loss event so an informed client can tell who was being replied to. Only // meaningful alongside inform_clients_on_loss. string caller = 6; + // Whether the WsSendAck for this message waits for the client to acknowledge + // it, rather than reporting the write to the socket. + // + // The message has to ask the client for an acknowledgement in the payload it + // carries, which the runtime does not compose. Setting this for one that + // asks nothing leaves the send waiting for an answer that is never coming, + // until the runtime declares the message lost. + // + // Waited for wherever the connection is, so a send means the same thing in a + // single node deployment as in a cluster. Sending it again while attempts + // remain, and declaring it lost when they run out, is the runtime's to do. + bool wait_for_ack = 7; } // Reports what happened to each message in a WsSend. diff --git a/libs/runtime/sdk/node/index.d.ts b/libs/runtime/sdk/node/index.d.ts index 6289e25c..df1c64cc 100644 --- a/libs/runtime/sdk/node/index.d.ts +++ b/libs/runtime/sdk/node/index.d.ts @@ -650,13 +650,34 @@ export interface JsConsumerMessage { vendor: any } -/** The result returned from a consumer or schedule event handler. */ +/** + * The result returned from a consumer or schedule event handler. + * + * For a source that acknowledges, a queue, a topic or a schedule, this decides + * what happens to the message rather than only reporting what happened. A + * success acknowledges it and it is not delivered again; a failure leaves it + * on its source, which redelivers it according to its own rules. + */ export interface JsEventResult { - /** Whether the event was processed successfully. */ + /** + * Whether the event was processed successfully. + * + * Reporting success for work the handler did not do means never seeing that + * work again. + */ success: boolean /** - * Optional list of individual message processing failures - * (for partial failure reporting in consumer handlers). + * The messages in a batch that could not be processed. + * + * Naming messages is how a handler answers for each one separately, those + * named are left on the source to be delivered again and the rest are + * acknowledged, so a handler that processed most of a batch does not have + * all of it sent back. Failing without naming any answers for the whole + * batch, and none of it is acknowledged. + * + * A non-empty list is taken as the answer even alongside `success: true`. + * Ignored for schedule handlers, which receive one message and have nothing + * to name. */ failures?: Array /** Optional error message (used for schedule handler failures). */ @@ -665,7 +686,13 @@ export interface JsEventResult { /** Represents a failure to process an individual message in a consumer batch. */ export interface JsMessageProcessingFailure { - /** The ID of the message that failed to process. */ + /** + * The ID of the message that failed to process. + * + * Must be the `messageId` the message arrived with. A message ID matching + * nothing in the batch would leave nothing behind, so the runtime refuses + * the whole answer rather than acknowledging a batch it cannot apply. + */ messageId: string /** Optional description of the error. */ errorMessage?: string diff --git a/libs/runtime/sdk/node/src/consumer.rs b/libs/runtime/sdk/node/src/consumer.rs index 8992cc1e..f9f16eaf 100644 --- a/libs/runtime/sdk/node/src/consumer.rs +++ b/libs/runtime/sdk/node/src/consumer.rs @@ -99,12 +99,29 @@ pub struct JsScheduleEventInput { } /// The result returned from a consumer or schedule event handler. +/// +/// For a source that acknowledges, a queue, a topic or a schedule, this decides +/// what happens to the message rather than only reporting what happened. A +/// success acknowledges it and it is not delivered again; a failure leaves it +/// on its source, which redelivers it according to its own rules. #[napi(object)] pub struct JsEventResult { /// Whether the event was processed successfully. + /// + /// Reporting success for work the handler did not do means never seeing that + /// work again. pub success: bool, - /// Optional list of individual message processing failures - /// (for partial failure reporting in consumer handlers). + /// The messages in a batch that could not be processed. + /// + /// Naming messages is how a handler answers for each one separately, those + /// named are left on the source to be delivered again and the rest are + /// acknowledged, so a handler that processed most of a batch does not have + /// all of it sent back. Failing without naming any answers for the whole + /// batch, and none of it is acknowledged. + /// + /// A non-empty list is taken as the answer even alongside `success: true`. + /// Ignored for schedule handlers, which receive one message and have nothing + /// to name. pub failures: Option>, /// Optional error message (used for schedule handler failures). pub error_message: Option, @@ -114,6 +131,10 @@ pub struct JsEventResult { #[napi(object)] pub struct JsMessageProcessingFailure { /// The ID of the message that failed to process. + /// + /// Must be the `messageId` the message arrived with. A message ID matching + /// nothing in the batch would leave nothing behind, so the runtime refuses + /// the whole answer rather than acknowledging a batch it cannot apply. pub message_id: String, /// Optional description of the error. pub error_message: Option, diff --git a/libs/runtime/sdk/python/python/celerity_runtime_sdk/_celerity_runtime_sdk.pyi b/libs/runtime/sdk/python/python/celerity_runtime_sdk/_celerity_runtime_sdk.pyi index 7cc7b550..163c05b0 100644 --- a/libs/runtime/sdk/python/python/celerity_runtime_sdk/_celerity_runtime_sdk.pyi +++ b/libs/runtime/sdk/python/python/celerity_runtime_sdk/_celerity_runtime_sdk.pyi @@ -1433,10 +1433,21 @@ class EventResult: """ The result returned from a consumer or schedule event handler. + For a source that acknowledges, a queue, a topic or a schedule, this decides + what happens to the message rather than only reporting what happened. A + success acknowledges it and it is not delivered again; a failure leaves it + on its source, which redelivers it according to its own rules. Reporting + success for work the handler did not do means never seeing that work again. + + Naming failures is how a handler answers for each message in a batch + separately, those named are left on the source and the rest are + acknowledged. Failing without naming any answers for the whole batch. A + non-empty list is taken as the answer even alongside a success, and is + ignored for schedule handlers, which receive one message. + Attributes: success: Whether the event was processed successfully. - failures: Optional list of individual message processing failures - (for partial failure reporting in consumer handlers). + failures: The messages in a batch that could not be processed. error_message: Optional error message (used for schedule handler failures). """ success: bool @@ -1463,6 +1474,10 @@ class MessageProcessingFailure: """ Represents a failure to process an individual message in a consumer batch. + The message id must be the one the message arrived with. An id matching + nothing in the batch would leave nothing behind, so the runtime refuses the + whole answer rather than acknowledging a batch it cannot apply. + Attributes: message_id: The ID of the message that failed to process. error_message: Optional description of the error. diff --git a/libs/runtime/sdk/python/src/consumer.rs b/libs/runtime/sdk/python/src/consumer.rs index d89712de..399a07cb 100644 --- a/libs/runtime/sdk/python/src/consumer.rs +++ b/libs/runtime/sdk/python/src/consumer.rs @@ -72,6 +72,19 @@ pub struct PyScheduleEventInput { pub trace_context: Option>, } +/// The result returned from a consumer or schedule event handler. +/// +/// For a source that acknowledges, a queue, a topic or a schedule, this decides +/// what happens to the message rather than only reporting what happened. A +/// success acknowledges it and it is not delivered again; a failure leaves it +/// on its source, which redelivers it according to its own rules. Reporting +/// success for work the handler did not do means never seeing that work again. +/// +/// Naming failures is how a handler answers for each message in a batch +/// separately, those named are left on the source and the rest are +/// acknowledged. Failing without naming any answers for the whole batch. A +/// non-empty list is taken as the answer even alongside a success, and is +/// ignored for schedule handlers, which receive one message. #[pyclass(name = "EventResult")] pub struct PyEventResult { #[pyo3(get, set)] @@ -99,6 +112,11 @@ impl PyEventResult { } } +/// One message in a batch that could not be processed. +/// +/// The message id must be the one the message arrived with. An id matching +/// nothing in the batch would leave nothing behind, so the runtime refuses the +/// whole answer rather than acknowledging a batch it cannot apply. #[pyclass(name = "MessageProcessingFailure")] pub struct PyMessageProcessingFailure { #[pyo3(get, set)] diff --git a/libs/runtime/ws/ws-registry/src/registry.rs b/libs/runtime/ws/ws-registry/src/registry.rs index 906c9e10..0419ce86 100644 --- a/libs/runtime/ws/ws-registry/src/registry.rs +++ b/libs/runtime/ws/ws-registry/src/registry.rs @@ -86,18 +86,21 @@ pub trait NodeLivenessStore: Send + Sync + Debug { } // Additional context for sending messages to a connection in a WebSocket registry. -#[derive(Default)] +#[derive(Debug, Default)] pub struct SendContext { // The caller that is sending the message. // This is useful for providing context about the purpose or origin of the message. // If a message is considered lost, the caller will be included in the message sent // to the clients in the inform_clients list. pub caller: Option, - // Whether to wait for an acknowledgement from the node that has the connection - // that the message was sent for, a WebSocketConnError::MessageLost error will be returned - // for the caller to handle the case where the message was lost - // when the wait_for_ack flag is set to true. - // This is only used when broadcasting messages to other nodes in a cluster. + // Whether to wait for the client to acknowledge the message, returning + // WebSocketConnError::MessageLost where it never does. + // + // Waited for wherever the connection is, on this node or another one, so a + // send means the same thing in a single node deployment as in a cluster. + // + // A message that does not ask its client to acknowledge is settled by + // the write, since nothing was asked and no answer is coming. pub wait_for_ack: bool, // The connection IDs of clients that should be informed if a message is lost // (an acknowledgement was not received after the maximum number of retries).