Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 27 additions & 9 deletions libs/runtime/consumers/consumer-redis/src/message_consumer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -444,12 +444,6 @@ impl RedisMessageConsumer {
}
}

let update_last_message_id_result =
self.update_last_message_id(&last_message_id, conn).await;
if let Err(err) = update_last_message_id_result {
error!("failed to update last message ID: {err}");
}

let failures = match result {
Ok(_) => vec![],
Err(error) => match error {
Expand Down Expand Up @@ -488,13 +482,37 @@ impl RedisMessageConsumer {
},
};

if !failures.is_empty() {
if let Err(err) = self
// Moved before the offset advances, since this consumer reads by offset
// rather than through a consumer group. Advancing first and then dying,
// or failing to write to the dead letter queue, would put the batch
// behind the offset with no record of it anywhere.
let recorded = if failures.is_empty() {
true
} else {
match self
.move_failed_messages_to_dlq(&failures, &messages, conn)
.await
{
error!("failed to move failed messages to DLQ: {err}");
Ok(()) => true,
Err(err) => {
error!("failed to move failed messages to DLQ: {err}");
false
}
}
};

// Left where it is otherwise, so the batch is read again rather than
// passed over. That redelivers what did succeed, which is the cost of
// one watermark for the whole batch and cheaper than losing the rest.
if recorded {
if let Err(err) = self.update_last_message_id(&last_message_id, conn).await {
error!("failed to update last message ID: {err}");
}
} else {
warn!(
"leaving the last message ID where it is, so the batch is read again rather \
than passed over with nothing recording what became of it"
);
}
}

Expand Down
29 changes: 21 additions & 8 deletions libs/runtime/consumers/consumer-sqs/tests/integration_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -70,20 +70,33 @@ async fn test_receive_messages_and_fire_message_handler() {

let mut collected_messages = Vec::<Message<SQSMessageMetadata>>::new();

let mut i = 0;
// 200 messages in batches (20 * 10) + 100 individual messages.
while i < 300 {
const EXPECTED_MESSAGES: usize = 300;
// Bounded so that a queue nothing created, an endpoint nothing is listening
// on, or a consumer that stopped, each fail with a count rather than
// waiting for messages that are not coming. Generous, since the sender
// spaces the individual messages out and a loaded machine is not a failure.
let deadline = tokio::time::Instant::now() + Duration::from_secs(60);

while collected_messages.len() < EXPECTED_MESSAGES {
tokio::select! {
msg = rx.recv() => {
if let Some(unwrapped) = msg {
collected_messages.push(unwrapped);
}
msg = rx.recv() => match msg {
Some(message) => collected_messages.push(message),
// The consumer stopped, so nothing further is coming and
// waiting out the deadline would only delay the report.
None => break,
},
_ = tokio::time::sleep_until(deadline) => break,
}
i += 1
}

assert_eq!(collected_messages.len(), 300);
assert_eq!(
collected_messages.len(),
EXPECTED_MESSAGES,
"collected {} of {EXPECTED_MESSAGES} messages before the deadline, \
check the consumer is running and the queue it polls exists",
collected_messages.len(),
);
for (i, message) in collected_messages.into_iter().enumerate() {
// The first 200 messages were sent in batches,
// the last 100 were sent individually.
Expand Down
1 change: 0 additions & 1 deletion libs/runtime/core/src/consumer_handler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -936,7 +936,6 @@ mod tests {
EventResult {
event_id: event_id.to_string(),
data,
context: None,
}
}

Expand Down
1 change: 0 additions & 1 deletion libs/runtime/core/src/ipc_frames.rs
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,6 @@ pub fn event_result_from_frame(result: proto::Result, waiting_for: &EventType) -
EventResult {
event_id: result.id,
data,
context: None,
}
}

Expand Down
146 changes: 139 additions & 7 deletions libs/runtime/core/src/ipc_stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -640,16 +640,58 @@ async fn send_frame(
/// exactly those. Reporting only a single outcome for the batch would leave a
/// handler no choice but to resend all of it, delivering the successful ones
/// twice.
/// Delivers every message in a batch and reports what became of each.
///
/// Messages for one connection are sent in the order the handler listed them,
/// and different connections do not wait on each other. That matters most for
/// a message asking its client to acknowledge it, where sending the batch in
/// one line would have every later message wait out the round trip of the one
/// before, and a client that never answers hold up all of them.
async fn send_to_websockets(
stream_id: StreamId,
context: &StreamContext,
send: proto::WsSend,
) -> proto::WsSendAck {
let correlation_id = send.correlation_id;
let mut failures = Vec::new();

// Keyed by connection, holding the position each message had in the batch,
// which is what a failure names and what the handler knows it by.
let mut by_connection: HashMap<String, Vec<(u32, proto::WsOutbound)>> = HashMap::new();
for (index, outbound) in send.messages.into_iter().enumerate() {
let index = index as u32;
by_connection
.entry(outbound.connection_id.clone())
.or_default()
.push((index as u32, outbound));
}

let mut failures = futures::future::join_all(
by_connection
.into_values()
.map(|messages| send_to_one_connection(stream_id, context, messages)),
)
.await
.concat();

// Reported in the order the handler sent them, since it has nothing else to
// match them against and the connections were served in no order at all.
failures.sort_by_key(|failure| failure.index);

proto::WsSendAck {
correlation_id,
success: failures.is_empty(),
failures,
}
}

/// Sends one connection's share of a batch, in the order it was given.
async fn send_to_one_connection(
stream_id: StreamId,
context: &StreamContext,
messages: Vec<(u32, proto::WsOutbound)>,
) -> Vec<proto::WsSendFailure> {
let mut failures = Vec::new();

for (index, outbound) in messages {
let connection_id = outbound.connection_id.clone();
let (message_type, message) = match encode_outbound(&outbound) {
Ok(encoded) => encoded,
Expand Down Expand Up @@ -696,11 +738,7 @@ async fn send_to_websockets(
}
}

proto::WsSendAck {
correlation_id,
success: failures.is_empty(),
failures,
}
failures
}

/// Renders an outbound message the way the connection registry expects it.
Expand Down Expand Up @@ -858,6 +896,10 @@ mod tests {
/// acknowledgement the handler requested from one it did not.
contexts: std::sync::Mutex<Vec<Option<SendContext>>>,
unreachable: HashSet<String>,
/// A connection whose sends wait to be released, standing in for a
/// client that is slow to acknowledge.
held: Option<String>,
release: Arc<tokio::sync::Notify>,
}

impl std::fmt::Display for RecordingWsRegistry {
Expand All @@ -876,6 +918,9 @@ mod tests {
message: String,
ctx: Option<SendContext>,
) -> Result<(), celerity_ws_registry::errors::WebSocketConnError> {
if self.held.as_deref() == Some(connection_id.as_str()) {
self.release.notified().await;
}
self.contexts.lock().unwrap().push(ctx);
if self.unreachable.contains(&connection_id) {
return Err(
Expand Down Expand Up @@ -1327,6 +1372,93 @@ mod tests {
assert!(ctx.wait_for_ack);
}

/// A message asking its client to acknowledge it is not answered until that
/// client does, so a batch sent in one line would have every later message
/// wait out the round trip of the one before it.
#[tokio::test]
async fn one_connection_holding_up_a_batch_does_not_hold_up_the_others() {
let release = Arc::new(tokio::sync::Notify::new());
let mut harness = start_with_registry(
&["schedule::a"],
RecordingWsRegistry {
held: Some("connection-slow".to_string()),
release: release.clone(),
..RecordingWsRegistry::default()
},
);
let _registration = ready_stream(&mut harness).await;

harness
.inbound_tx
.send(Ok(ws_send(
"correlation-1",
vec![
outbound("connection-slow", b"{}", false),
outbound("connection-fast", b"{}", false),
],
)))
.await
.unwrap();

// The one behind it reaches its connection while the first is still
// waiting, which is the whole point of not sending them in sequence.
let sent_to_fast = tokio::time::timeout(Duration::from_secs(5), async {
loop {
if harness
.ws_registry
.sent
.lock()
.unwrap()
.iter()
.any(|(connection_id, ..)| connection_id == "connection-fast")
{
return;
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await;
assert!(
sent_to_fast.is_ok(),
"the second message waited for the first connection to answer"
);

release.notify_waiters();

let Some(proto::runtime_message::Frame::WsAck(ack)) =
next_frame(&mut harness.outbound_rx).await
else {
panic!("expected an acknowledgement for the batch");
};
assert!(ack.success, "failures = {:?}", ack.failures);
}

/// Two messages for one connection arrive in the order the handler listed
/// them, which is the one ordering a handler can reasonably expect.
#[tokio::test]
async fn messages_for_one_connection_keep_the_order_they_were_given() {
let mut harness = start(&["schedule::a"]);
let _registration = ready_stream(&mut harness).await;

harness
.inbound_tx
.send(Ok(ws_send(
"correlation-1",
vec![
outbound("connection-1", b"{\"n\":1}", false),
outbound("connection-1", b"{\"n\":2}", false),
],
)))
.await
.unwrap();

let _ack = next_frame(&mut harness.outbound_rx).await;

let sent = harness.ws_registry.sent.lock().unwrap();
let bodies: Vec<&str> = sent.iter().map(|(_, _, _, body)| body.as_str()).collect();
assert_eq!(bodies, vec!["{\"n\":1}", "{\"n\":2}"]);
}

/// Asking for neither an acknowledgement nor anyone informed is the common
/// case, and it settles on the write with nothing to carry.
#[tokio::test]
Expand Down
2 changes: 0 additions & 2 deletions libs/runtime/core/src/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -229,8 +229,6 @@ pub struct EventResult {
#[serde(rename = "eventId")]
pub event_id: String,
pub data: EventResultData,
#[serde(skip_serializing_if = "Option::is_none")]
pub context: Option<Value>,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
Expand Down
2 changes: 0 additions & 2 deletions libs/runtime/core/tests/integration_consumer_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ impl ConsumerEventHandler for MockConsumerEventHandler {
Ok(EventResult {
event_id: String::new(),
data: success_result_data(),
context: None,
})
}

Expand All @@ -81,7 +80,6 @@ impl ConsumerEventHandler for MockConsumerEventHandler {
Ok(EventResult {
event_id: String::new(),
data: success_schedule_result_data(),
context: None,
})
}
}
Expand Down
7 changes: 5 additions & 2 deletions libs/runtime/helpers/src/jsonpath.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@ use serde_json::Value;
///
/// An example of how you might use this function:
/// ```
/// # use celerity_helpers::jsonpath::jsonpath_inject_root;
/// # use jsonpath_rust::JsonPath;
/// # use serde_json::json;
///
/// # use std::str::FromStr;
/// let mut json = json!({
/// "name": "John Doe",
/// });
Expand All @@ -26,7 +28,8 @@ use serde_json::Value;
/// "planPrice": 99.99,
/// });
/// let result = jsonpath_inject_root(&path, &mut json, value);
/// assert_eq!(result, false);
/// // True, since the path names a field of the root object.
/// assert!(result);
/// assert_eq!(json, json!({
/// "name": "John Doe",
/// "planInfo": {
Expand Down
4 changes: 1 addition & 3 deletions libs/runtime/helpers/src/time.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,7 @@ impl Debug for dyn Clock + Send + Sync {
/// # Examples
///
/// ```
/// # use celerity_helpers::time::DefaultClock;
/// # use std::time::SystemTime;
///
/// # use celerity_helpers::time::{Clock, DefaultClock};
/// let clock = DefaultClock::new();
/// let now = clock.now();
/// ```
Expand Down
Loading
Loading