Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions libs/runtime/core/src/generated/celerity.runtime.v1.rs
Original file line number Diff line number Diff line change
Expand Up @@ -577,6 +577,19 @@ pub struct WsOutbound {
/// meaningful alongside inform_clients_on_loss.
#[prost(string, tag = "6")]
pub caller: ::prost::alloc::string::String,
/// Whether the WsSendAck for this message waits for the client to acknowledge
/// it, rather than reporting the write to the socket.
///
/// The message has to ask the client for an acknowledgement in the payload it
/// carries, which the runtime does not compose. Setting this for one that
/// asks nothing leaves the send waiting for an answer that is never coming,
/// until the runtime declares the message lost.
///
/// Waited for wherever the connection is, so a send means the same thing in a
/// single node deployment as in a cluster. Sending it again while attempts
/// remain, and declaring it lost when they run out, is the runtime's to do.
#[prost(bool, tag = "7")]
pub wait_for_ack: bool,
}
/// Reports what happened to each message in a WsSend.
///
Expand Down
86 changes: 81 additions & 5 deletions libs/runtime/core/src/ipc_stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -679,7 +679,11 @@ async fn send_to_websockets(
message_id,
message_type,
message,
send_context(outbound.caller, outbound.inform_clients_on_loss),
send_context(
outbound.caller,
outbound.inform_clients_on_loss,
outbound.wait_for_ack,
),
)
.await
{
Expand Down Expand Up @@ -722,14 +726,23 @@ fn encode_outbound(outbound: &proto::WsOutbound) -> Result<(MessageType, String)
/// context is produced without them. Acknowledgements are not waited for, since
/// the handler has already been told the send was accepted and holding the
/// stream for a cluster round trip would delay every message behind it.
fn send_context(caller: String, inform_clients: Vec<String>) -> Option<SendContext> {
if inform_clients.is_empty() {
/// What the registry needs beyond the message itself, where the handler asked
/// for anything at all.
///
/// Absent where it asked for neither, which is the common case and the one
/// that settles on the write.
fn send_context(
caller: String,
inform_clients: Vec<String>,
wait_for_ack: bool,
) -> Option<SendContext> {
if !wait_for_ack && inform_clients.is_empty() {
return None;
}
Some(SendContext {
caller: (!caller.is_empty()).then_some(caller),
inform_clients,
wait_for_ack: false,
wait_for_ack,
})
}

Expand Down Expand Up @@ -841,6 +854,9 @@ mod tests {
#[derive(Debug, Default)]
struct RecordingWsRegistry {
sent: std::sync::Mutex<Vec<(String, String, MessageType, String)>>,
/// What each send asked for beyond the message, so a test can tell an
/// acknowledgement the handler requested from one it did not.
contexts: std::sync::Mutex<Vec<Option<SendContext>>>,
unreachable: HashSet<String>,
}

Expand All @@ -858,8 +874,9 @@ mod tests {
message_id: String,
message_type: MessageType,
message: String,
_: Option<SendContext>,
ctx: Option<SendContext>,
) -> Result<(), celerity_ws_registry::errors::WebSocketConnError> {
self.contexts.lock().unwrap().push(ctx);
if self.unreachable.contains(&connection_id) {
return Err(
celerity_ws_registry::errors::WebSocketConnError::MessageLost(message_id),
Expand Down Expand Up @@ -1265,6 +1282,13 @@ mod tests {
}
}

fn outbound_asking_for_ack(connection_id: &str, message: &[u8]) -> proto::WsOutbound {
proto::WsOutbound {
wait_for_ack: true,
..outbound(connection_id, message, false)
}
}

fn outbound(connection_id: &str, message: &[u8], is_binary: bool) -> proto::WsOutbound {
proto::WsOutbound {
connection_id: connection_id.to_string(),
Expand All @@ -1273,9 +1297,61 @@ mod tests {
inform_clients_on_loss: vec![],
message_id: String::new(),
caller: String::new(),
wait_for_ack: false,
}
}

/// The runtime waits for the client, sends the message again while attempts
/// remain and declares it lost when they run out, none of which happens
/// unless the request reaches the registry.
#[tokio::test]
async fn carries_a_requested_client_acknowledgement_to_the_registry() {
let mut harness = start(&["schedule::a"]);
let _registration = ready_stream(&mut harness).await;

harness
.inbound_tx
.send(Ok(ws_send(
"correlation-1",
vec![outbound_asking_for_ack("connection-1", b"{}")],
)))
.await
.unwrap();

let _ack = next_frame(&mut harness.outbound_rx).await;

let contexts = harness.ws_registry.contexts.lock().unwrap();
let Some(Some(ctx)) = contexts.first() else {
panic!("the send asked for an acknowledgement and reached the registry without one");
};
assert!(ctx.wait_for_ack);
}

/// Asking for neither an acknowledgement nor anyone informed is the common
/// case, and it settles on the write with nothing to carry.
#[tokio::test]
async fn carries_nothing_for_a_message_that_asked_for_nothing() {
let mut harness = start(&["schedule::a"]);
let _registration = ready_stream(&mut harness).await;

harness
.inbound_tx
.send(Ok(ws_send(
"correlation-1",
vec![outbound("connection-1", b"{}", false)],
)))
.await
.unwrap();

let _ack = next_frame(&mut harness.outbound_rx).await;

let contexts = harness.ws_registry.contexts.lock().unwrap();
assert!(
matches!(contexts.first(), Some(None)),
"contexts = {contexts:?}"
);
}

#[tokio::test]
async fn delivers_a_websocket_message_a_handler_sent() {
let mut harness = start(&["schedule::a"]);
Expand Down
12 changes: 12 additions & 0 deletions libs/runtime/proto/celerity/runtime/v1/runtime.proto
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,18 @@ message WsOutbound {
// loss event so an informed client can tell who was being replied to. Only
// meaningful alongside inform_clients_on_loss.
string caller = 6;
// Whether the WsSendAck for this message waits for the client to acknowledge
// it, rather than reporting the write to the socket.
//
// The message has to ask the client for an acknowledgement in the payload it
// carries, which the runtime does not compose. Setting this for one that
// asks nothing leaves the send waiting for an answer that is never coming,
// until the runtime declares the message lost.
//
// Waited for wherever the connection is, so a send means the same thing in a
// single node deployment as in a cluster. Sending it again while attempts
// remain, and declaring it lost when they run out, is the runtime's to do.
bool wait_for_ack = 7;
}

// Reports what happened to each message in a WsSend.
Expand Down
37 changes: 32 additions & 5 deletions libs/runtime/sdk/node/index.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -650,13 +650,34 @@ export interface JsConsumerMessage {
vendor: any
}

/** The result returned from a consumer or schedule event handler. */
/**
* The result returned from a consumer or schedule event handler.
*
* For a source that acknowledges, a queue, a topic or a schedule, this decides
* what happens to the message rather than only reporting what happened. A
* success acknowledges it and it is not delivered again; a failure leaves it
* on its source, which redelivers it according to its own rules.
*/
export interface JsEventResult {
/** Whether the event was processed successfully. */
/**
* Whether the event was processed successfully.
*
* Reporting success for work the handler did not do means never seeing that
* work again.
*/
success: boolean
/**
* Optional list of individual message processing failures
* (for partial failure reporting in consumer handlers).
* The messages in a batch that could not be processed.
*
* Naming messages is how a handler answers for each one separately, those
* named are left on the source to be delivered again and the rest are
* acknowledged, so a handler that processed most of a batch does not have
* all of it sent back. Failing without naming any answers for the whole
* batch, and none of it is acknowledged.
*
* A non-empty list is taken as the answer even alongside `success: true`.
* Ignored for schedule handlers, which receive one message and have nothing
* to name.
*/
failures?: Array<JsMessageProcessingFailure>
/** Optional error message (used for schedule handler failures). */
Expand All @@ -665,7 +686,13 @@ export interface JsEventResult {

/** Represents a failure to process an individual message in a consumer batch. */
export interface JsMessageProcessingFailure {
/** The ID of the message that failed to process. */
/**
* The ID of the message that failed to process.
*
* Must be the `messageId` the message arrived with. A message ID matching
* nothing in the batch would leave nothing behind, so the runtime refuses
* the whole answer rather than acknowledging a batch it cannot apply.
*/
messageId: string
/** Optional description of the error. */
errorMessage?: string
Expand Down
25 changes: 23 additions & 2 deletions libs/runtime/sdk/node/src/consumer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -99,12 +99,29 @@ pub struct JsScheduleEventInput {
}

/// The result returned from a consumer or schedule event handler.
///
/// For a source that acknowledges, a queue, a topic or a schedule, this decides
/// what happens to the message rather than only reporting what happened. A
/// success acknowledges it and it is not delivered again; a failure leaves it
/// on its source, which redelivers it according to its own rules.
#[napi(object)]
pub struct JsEventResult {
/// Whether the event was processed successfully.
///
/// Reporting success for work the handler did not do means never seeing that
/// work again.
pub success: bool,
/// Optional list of individual message processing failures
/// (for partial failure reporting in consumer handlers).
/// The messages in a batch that could not be processed.
///
/// Naming messages is how a handler answers for each one separately, those
/// named are left on the source to be delivered again and the rest are
/// acknowledged, so a handler that processed most of a batch does not have
/// all of it sent back. Failing without naming any answers for the whole
/// batch, and none of it is acknowledged.
///
/// A non-empty list is taken as the answer even alongside `success: true`.
/// Ignored for schedule handlers, which receive one message and have nothing
/// to name.
pub failures: Option<Vec<JsMessageProcessingFailure>>,
/// Optional error message (used for schedule handler failures).
pub error_message: Option<String>,
Expand All @@ -114,6 +131,10 @@ pub struct JsEventResult {
#[napi(object)]
pub struct JsMessageProcessingFailure {
/// The ID of the message that failed to process.
///
/// Must be the `messageId` the message arrived with. A message ID matching
/// nothing in the batch would leave nothing behind, so the runtime refuses
/// the whole answer rather than acknowledging a batch it cannot apply.
pub message_id: String,
/// Optional description of the error.
pub error_message: Option<String>,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1433,10 +1433,21 @@ class EventResult:
"""
The result returned from a consumer or schedule event handler.

For a source that acknowledges, a queue, a topic or a schedule, this decides
what happens to the message rather than only reporting what happened. A
success acknowledges it and it is not delivered again; a failure leaves it
on its source, which redelivers it according to its own rules. Reporting
success for work the handler did not do means never seeing that work again.

Naming failures is how a handler answers for each message in a batch
separately, those named are left on the source and the rest are
acknowledged. Failing without naming any answers for the whole batch. A
non-empty list is taken as the answer even alongside a success, and is
ignored for schedule handlers, which receive one message.

Attributes:
success: Whether the event was processed successfully.
failures: Optional list of individual message processing failures
(for partial failure reporting in consumer handlers).
failures: The messages in a batch that could not be processed.
error_message: Optional error message (used for schedule handler failures).
"""
success: bool
Expand All @@ -1463,6 +1474,10 @@ class MessageProcessingFailure:
"""
Represents a failure to process an individual message in a consumer batch.

The message id must be the one the message arrived with. An id matching
nothing in the batch would leave nothing behind, so the runtime refuses the
whole answer rather than acknowledging a batch it cannot apply.

Attributes:
message_id: The ID of the message that failed to process.
error_message: Optional description of the error.
Expand Down
18 changes: 18 additions & 0 deletions libs/runtime/sdk/python/src/consumer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,19 @@ pub struct PyScheduleEventInput {
pub trace_context: Option<HashMap<String, String>>,
}

/// The result returned from a consumer or schedule event handler.
///
/// For a source that acknowledges, a queue, a topic or a schedule, this decides
/// what happens to the message rather than only reporting what happened. A
/// success acknowledges it and it is not delivered again; a failure leaves it
/// on its source, which redelivers it according to its own rules. Reporting
/// success for work the handler did not do means never seeing that work again.
///
/// Naming failures is how a handler answers for each message in a batch
/// separately, those named are left on the source and the rest are
/// acknowledged. Failing without naming any answers for the whole batch. A
/// non-empty list is taken as the answer even alongside a success, and is
/// ignored for schedule handlers, which receive one message.
#[pyclass(name = "EventResult")]
pub struct PyEventResult {
#[pyo3(get, set)]
Expand Down Expand Up @@ -99,6 +112,11 @@ impl PyEventResult {
}
}

/// One message in a batch that could not be processed.
///
/// The message id must be the one the message arrived with. An id matching
/// nothing in the batch would leave nothing behind, so the runtime refuses the
/// whole answer rather than acknowledging a batch it cannot apply.
#[pyclass(name = "MessageProcessingFailure")]
pub struct PyMessageProcessingFailure {
#[pyo3(get, set)]
Expand Down
15 changes: 9 additions & 6 deletions libs/runtime/ws/ws-registry/src/registry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -86,18 +86,21 @@ pub trait NodeLivenessStore: Send + Sync + Debug {
}

// Additional context for sending messages to a connection in a WebSocket registry.
#[derive(Default)]
#[derive(Debug, Default)]
pub struct SendContext {
// The caller that is sending the message.
// This is useful for providing context about the purpose or origin of the message.
// If a message is considered lost, the caller will be included in the message sent
// to the clients in the inform_clients list.
pub caller: Option<String>,
// Whether to wait for an acknowledgement from the node that has the connection
// that the message was sent for, a WebSocketConnError::MessageLost error will be returned
// for the caller to handle the case where the message was lost
// when the wait_for_ack flag is set to true.
// This is only used when broadcasting messages to other nodes in a cluster.
// Whether to wait for the client to acknowledge the message, returning
// WebSocketConnError::MessageLost where it never does.
//
// Waited for wherever the connection is, on this node or another one, so a
// send means the same thing in a single node deployment as in a cluster.
//
// A message that does not ask its client to acknowledge is settled by
// the write, since nothing was asked and no answer is coming.
pub wait_for_ack: bool,
// The connection IDs of clients that should be informed if a message is lost
// (an acknowledgement was not received after the maximum number of retries).
Expand Down
Loading