From d3fb8cfceb5b56e94539b9a6aa228b0f711cbfbf Mon Sep 17 00:00:00 2001 From: selarkin Date: Wed, 23 Sep 2026 18:42:12 -0700 Subject: [PATCH 1/9] [rush-daemon] Surface warm snapshot and daemon shutdown errors to the right client Fixes #6059 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- apps/rush-cli-client/src/daemonCommands.ts | 14 +++- apps/rush-cli-client/src/launchClient.ts | 9 +-- apps/rush-cli-client/src/resultDiagnostics.ts | 23 +++++++ .../src/test/resultDiagnostics.test.ts | 26 ++++++++ ...ushd-error-surfacing_2026-09-24-01-50.json | 11 ++++ ...ushd-error-surfacing_2026-09-24-01-50.json | 11 ++++ ...ushd-error-surfacing_2026-09-24-01-50.json | 11 ++++ ...ushd-error-surfacing_2026-09-24-01-50.json | 11 ++++ .../rush-client-core/src/DaemonClient.ts | 10 ++- .../src/test/DaemonClient.test.ts | 7 +- .../src/ControlMessageValidation.ts | 3 +- .../src/DaemonLifecycleControl.ts | 5 +- .../src/ShutdownAckValidation.ts | 17 +++++ .../src/test/LifecycleControl.test.ts | 7 ++ .../rush-daemon/src/DaemonControlSession.ts | 21 ++++-- .../rush-daemon/src/DaemonShutdownError.ts | 63 ++++++++++++++++++ .../rush-daemon/src/EngineTerminalProvider.ts | 15 ++++- .../rush-daemon/src/PhasedRequestRouter.ts | 5 +- .../src/ProductionDaemonRequestResolver.ts | 26 +++++++- libraries/rush-daemon/src/RushDaemonHost.ts | 30 ++++++--- libraries/rush-daemon/src/index.ts | 5 ++ libraries/rush-daemon/src/serveRushDaemon.ts | 13 +++- .../src/test/DaemonShutdown.test.ts | 2 +- .../src/test/EngineTerminalProvider.test.ts | 25 ++++++++ .../ProductionDaemonRequestResolver.test.ts | 64 +++++++++++++++++++ 25 files changed, 396 insertions(+), 38 deletions(-) create mode 100644 apps/rush-cli-client/src/resultDiagnostics.ts create mode 100644 apps/rush-cli-client/src/test/resultDiagnostics.test.ts create mode 100644 common/changes/@rushstack/rush-cli-client/fix-rushd-error-surfacing_2026-09-24-01-50.json create mode 100644 common/changes/@rushstack/rush-client-core/fix-rushd-error-surfacing_2026-09-24-01-50.json create mode 100644 common/changes/@rushstack/rush-daemon-protocol/fix-rushd-error-surfacing_2026-09-24-01-50.json create mode 100644 common/changes/@rushstack/rush-daemon/fix-rushd-error-surfacing_2026-09-24-01-50.json create mode 100644 libraries/rush-daemon-protocol/src/ShutdownAckValidation.ts create mode 100644 libraries/rush-daemon/src/DaemonShutdownError.ts create mode 100644 libraries/rush-daemon/src/test/EngineTerminalProvider.test.ts diff --git a/apps/rush-cli-client/src/daemonCommands.ts b/apps/rush-cli-client/src/daemonCommands.ts index d2c3e592b1..ae47a2bf42 100644 --- a/apps/rush-cli-client/src/daemonCommands.ts +++ b/apps/rush-cli-client/src/daemonCommands.ts @@ -82,10 +82,20 @@ export async function executeDaemonCommandAsync(options: IDaemonCommandOptions): : await DaemonClient.connectAsync({ socketPath: connectionOptions.paths.socketPath }); try { if (command === 'stop') { - await client.shutdownAsync(); + const { activeRequests } = await client.shutdownAsync(); + if (activeRequests) { + await writeStreamAsync( + process.stderr, + Buffer.from( + `rush-client: the daemon was running ${activeRequests} request(s); ` + + 'they were cancelled and their clients were told to re-run the command.\n' + ) + ); + } await writeStatusAsync({ state: 'shutdownAccepted', - socketPath: connectionOptions.paths.socketPath + socketPath: connectionOptions.paths.socketPath, + ...(activeRequests === undefined ? {} : { cancelledRequests: activeRequests }) }); return; } diff --git a/apps/rush-cli-client/src/launchClient.ts b/apps/rush-cli-client/src/launchClient.ts index b739b9fadf..5c9d13155b 100644 --- a/apps/rush-cli-client/src/launchClient.ts +++ b/apps/rush-cli-client/src/launchClient.ts @@ -29,6 +29,7 @@ import { executeDaemonCommandAsync } from './daemonCommands'; import { ClientOperationRenderer } from './ClientOperationRenderer'; import { getDaemonConnectionOptionsAsync } from './daemonConnectionOptions'; import { selectClientRoute, type IClientRoute } from './routing'; +import { getResultDiagnostic } from './resultDiagnostics'; import { writeStreamAsync } from './writeStreamAsync'; interface IWorkspaceJson { @@ -187,12 +188,8 @@ export async function launchClientAsync(rushx: boolean): Promise { } if (outcome.kind === 'result') { process.exitCode = outcome.result.exitCode; - if (outcome.result.admissionErrorCode) { - await writeStreamAsync( - process.stderr, - Buffer.from(`rush-client: daemon admission failed (${outcome.result.admissionErrorCode}).\n`) - ); - } + const diagnostic: string | undefined = getResultDiagnostic(outcome.result); + if (diagnostic) await writeStreamAsync(process.stderr, Buffer.from(diagnostic)); } else if (outcome.kind === 'rejected') { throw new Error(`Daemon rejected the request (${outcome.rejection.code}): ${outcome.rejection.message}`); } else if (abort.signal.aborted) { diff --git a/apps/rush-cli-client/src/resultDiagnostics.ts b/apps/rush-cli-client/src/resultDiagnostics.ts new file mode 100644 index 0000000000..d07db225b3 --- /dev/null +++ b/apps/rush-cli-client/src/resultDiagnostics.ts @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type { IDaemonCommandResult } from '@rushstack/rush-daemon-protocol'; + +/** + * Returns the stderr line that explains a daemon result, if any. + * + * @remarks + * A non-zero result's error message (for example, a daemon shutdown that aborted the request) is the only + * place the daemon reports failures that are not attributed to an operation, so it must not be dropped. + */ +export function getResultDiagnostic( + result: Pick +): string | undefined { + if (result.admissionErrorCode) { + return `rush-client: daemon admission failed (${result.admissionErrorCode}).\n`; + } + if (result.exitCode !== 0 && result.errorMessage) { + return `rush-client: ${result.errorMessage}\n`; + } + return undefined; +} diff --git a/apps/rush-cli-client/src/test/resultDiagnostics.test.ts b/apps/rush-cli-client/src/test/resultDiagnostics.test.ts new file mode 100644 index 0000000000..f06fce6967 --- /dev/null +++ b/apps/rush-cli-client/src/test/resultDiagnostics.test.ts @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { getResultDiagnostic } from '../resultDiagnostics'; + +describe(getResultDiagnostic.name, () => { + it('prints the error message of a failed result', () => { + expect( + getResultDiagnostic({ + exitCode: 1, + errorMessage: 'The Rush daemon was shut down (idle timeout) while this request was running.' + }) + ).toBe('rush-client: The Rush daemon was shut down (idle timeout) while this request was running.\n'); + }); + + it('prefers the typed admission failure', () => { + expect(getResultDiagnostic({ exitCode: 1, admissionErrorCode: 'wait-timeout', errorMessage: 'x' })).toBe( + 'rush-client: daemon admission failed (wait-timeout).\n' + ); + }); + + it('stays silent for successful results and failures without a message', () => { + expect(getResultDiagnostic({ exitCode: 0, errorMessage: 'ignored' })).toBeUndefined(); + expect(getResultDiagnostic({ exitCode: 1 })).toBeUndefined(); + }); +}); diff --git a/common/changes/@rushstack/rush-cli-client/fix-rushd-error-surfacing_2026-09-24-01-50.json b/common/changes/@rushstack/rush-cli-client/fix-rushd-error-surfacing_2026-09-24-01-50.json new file mode 100644 index 0000000000..925deeffab --- /dev/null +++ b/common/changes/@rushstack/rush-cli-client/fix-rushd-error-surfacing_2026-09-24-01-50.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-cli-client", + "comment": "Print the error message of a failed daemon result (for example, a daemon shutdown that cancelled the build), and report cancelled requests from \"rush-client daemon stop\".", + "type": "patch" + } + ], + "packageName": "@rushstack/rush-cli-client", + "email": "selarkin@microsoft.com" +} diff --git a/common/changes/@rushstack/rush-client-core/fix-rushd-error-surfacing_2026-09-24-01-50.json b/common/changes/@rushstack/rush-client-core/fix-rushd-error-surfacing_2026-09-24-01-50.json new file mode 100644 index 0000000000..fad92bb8f4 --- /dev/null +++ b/common/changes/@rushstack/rush-client-core/fix-rushd-error-surfacing_2026-09-24-01-50.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-client-core", + "comment": "Return the daemon shutdown acknowledgement, including its optional active request count, from DaemonClient.shutdownAsync().", + "type": "patch" + } + ], + "packageName": "@rushstack/rush-client-core", + "email": "selarkin@microsoft.com" +} diff --git a/common/changes/@rushstack/rush-daemon-protocol/fix-rushd-error-surfacing_2026-09-24-01-50.json b/common/changes/@rushstack/rush-daemon-protocol/fix-rushd-error-surfacing_2026-09-24-01-50.json new file mode 100644 index 0000000000..92fdcf2561 --- /dev/null +++ b/common/changes/@rushstack/rush-daemon-protocol/fix-rushd-error-surfacing_2026-09-24-01-50.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-daemon-protocol", + "comment": "Add an optional \"activeRequests\" count to the shutdownAck control message; older peers omit or ignore it.", + "type": "patch" + } + ], + "packageName": "@rushstack/rush-daemon-protocol", + "email": "selarkin@microsoft.com" +} diff --git a/common/changes/@rushstack/rush-daemon/fix-rushd-error-surfacing_2026-09-24-01-50.json b/common/changes/@rushstack/rush-daemon/fix-rushd-error-surfacing_2026-09-24-01-50.json new file mode 100644 index 0000000000..6af207cd00 --- /dev/null +++ b/common/changes/@rushstack/rush-daemon/fix-rushd-error-surfacing_2026-09-24-01-50.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-daemon", + "comment": "Report warm input-snapshot failures (with the engine diagnostics) to the failing request instead of replaying them into the next request, abort requests interrupted by a daemon shutdown with a typed DaemonShutdownError that names its initiator, and report running requests in the shutdown acknowledgement.", + "type": "patch" + } + ], + "packageName": "@rushstack/rush-daemon", + "email": "selarkin@microsoft.com" +} diff --git a/libraries/rush-client-core/src/DaemonClient.ts b/libraries/rush-client-core/src/DaemonClient.ts index 012b215cf3..54193ba64f 100644 --- a/libraries/rush-client-core/src/DaemonClient.ts +++ b/libraries/rush-client-core/src/DaemonClient.ts @@ -25,7 +25,8 @@ import { type IDaemonPongMessage, type IDaemonProtocolVersion, type IDaemonRequestEnvelope, - type IDaemonRequestRejectedMessage + type IDaemonRequestRejectedMessage, + type IDaemonShutdownAckMessage } from '@rushstack/rush-daemon-protocol'; import { connectDaemonAsync, type DaemonFrameConnection } from '@rushstack/rush-daemon-transport'; @@ -102,6 +103,7 @@ export class DaemonClient { #result: IDeferred | undefined; #shutdown: IDeferred | undefined; #shutdownAcknowledged: boolean = false; + #shutdownAck: IDaemonShutdownAckMessage['payload'] = {}; #execution: IDaemonClientExecuteOptions | undefined; #finished: boolean = false; #inputStarted: boolean = false; @@ -186,8 +188,10 @@ export class DaemonClient { * Requests shutdown on a fresh connection and waits for acknowledgement followed by EOF. * @remarks This confirms acceptance and connection closure, not successful workspace cleanup. * Requires protocol 0.6. The timeout defaults to 15000 milliseconds. + * @returns The acknowledgement, including the number of running requests the shutdown aborts when the + * daemon reports it. */ - public async shutdownAsync(timeoutMs: number = 15000): Promise { + public async shutdownAsync(timeoutMs: number = 15000): Promise { if (this.#used) throw new Error('Create a fresh DaemonClient for shutdown.'); this.#used = true; let timer: ReturnType | undefined; @@ -206,6 +210,7 @@ export class DaemonClient { ); }, timeoutMs); await Promise.all([this.#shutdown.promise, this.#sendControlAsync({ kind: 'shutdown', payload: {} })]); + return this.#shutdownAck; } finally { clearTimeout(timer); await this.closeAsync(); @@ -375,6 +380,7 @@ export class DaemonClient { throw new DaemonProtocolError('malformedControlMessage', 'Unexpected shutdown acknowledgement.'); } this.#shutdownAcknowledged = true; + this.#shutdownAck = message.payload; return; } const execution: IDaemonClientExecuteOptions = this.#requireExecution(); diff --git a/libraries/rush-client-core/src/test/DaemonClient.test.ts b/libraries/rush-client-core/src/test/DaemonClient.test.ts index d5aaf43094..9a2ccc6af8 100644 --- a/libraries/rush-client-core/src/test/DaemonClient.test.ts +++ b/libraries/rush-client-core/src/test/DaemonClient.test.ts @@ -521,21 +521,22 @@ describe('DaemonClient', () => { }); onRequest = async (message) => { if (message.kind === 'shutdown') { - await sendAsync({ kind: 'shutdownAck', payload: {} }); + await sendAsync({ kind: 'shutdownAck', payload: { activeRequests: 2 } }); acknowledged(); } }; const client = await DaemonClient.connectAsync({ socketPath: address }); expect(client.protocolVersion.minor).toBeGreaterThanOrEqual(6); let completed: boolean = false; - const shutdown: Promise = client.shutdownAsync().then(() => { + const shutdown: Promise = client.shutdownAsync().then((payload) => { completed = true; + return payload; }); await ack; await new Promise((resolve) => setTimeout(resolve, 20)); expect(completed).toBe(false); await connection!.closeAsync(); - await shutdown; + await expect(shutdown).resolves.toEqual({ activeRequests: 2 }); expect(completed).toBe(true); expect(controls.filter((message) => message.kind === 'shutdown')).toHaveLength(1); }); diff --git a/libraries/rush-daemon-protocol/src/ControlMessageValidation.ts b/libraries/rush-daemon-protocol/src/ControlMessageValidation.ts index 8022db1d85..2be3cf95f9 100644 --- a/libraries/rush-daemon-protocol/src/ControlMessageValidation.ts +++ b/libraries/rush-daemon-protocol/src/ControlMessageValidation.ts @@ -13,6 +13,7 @@ import { validateRequestResultControl, validateRequestStartControl } from './RequestControlValidation'; +import { validateShutdownAck } from './ShutdownAckValidation'; import { validateSubscribeControl } from './SubscribeControlValidation'; function fail(reason: string): never { throw new DaemonProtocolError('malformedControlMessage', reason); @@ -67,7 +68,7 @@ const VALIDATORS_BY_KIND: Record = { requestRejected: validateRequestRejectedControl, requestResult: validateRequestResultControl, shutdown: noopValidator, - shutdownAck: noopValidator, + shutdownAck: validateShutdownAck, stdinReady: validateRequestCancelControl, stdinEnd: validateRequestCancelControl }; diff --git a/libraries/rush-daemon-protocol/src/DaemonLifecycleControl.ts b/libraries/rush-daemon-protocol/src/DaemonLifecycleControl.ts index 31921111a2..e591e54844 100644 --- a/libraries/rush-daemon-protocol/src/DaemonLifecycleControl.ts +++ b/libraries/rush-daemon-protocol/src/DaemonLifecycleControl.ts @@ -10,5 +10,8 @@ export interface IDaemonShutdownMessage { /** Acknowledges shutdown before the host closes connections and releases its endpoint. @beta */ export interface IDaemonShutdownAckMessage { readonly kind: 'shutdownAck'; - readonly payload: Record; + readonly payload: { + /** Requests that were still running and will be aborted by this shutdown. Older daemons omit it. */ + readonly activeRequests?: number; + }; } diff --git a/libraries/rush-daemon-protocol/src/ShutdownAckValidation.ts b/libraries/rush-daemon-protocol/src/ShutdownAckValidation.ts new file mode 100644 index 0000000000..5bd6c80cbc --- /dev/null +++ b/libraries/rush-daemon-protocol/src/ShutdownAckValidation.ts @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { DaemonProtocolError } from './DaemonProtocolError'; + +const ZERO: number = 0; + +/** Validates the optional active request count of a shutdown acknowledgement. @internal */ +export function validateShutdownAck(payload: Record): void { + const value: unknown = payload.activeRequests; + if (value === undefined || isNonnegativeInteger(value)) return; + throw new DaemonProtocolError('malformedControlMessage', 'Invalid shutdownAck field "activeRequests".'); +} + +function isNonnegativeInteger(value: unknown): boolean { + return typeof value === 'number' && Number.isSafeInteger(value) && value >= ZERO; +} diff --git a/libraries/rush-daemon-protocol/src/test/LifecycleControl.test.ts b/libraries/rush-daemon-protocol/src/test/LifecycleControl.test.ts index 00c5363e2d..87d5cbceef 100644 --- a/libraries/rush-daemon-protocol/src/test/LifecycleControl.test.ts +++ b/libraries/rush-daemon-protocol/src/test/LifecycleControl.test.ts @@ -14,6 +14,8 @@ const FRACTION: number = 1.5; const MESSAGES: readonly DaemonControlMessage[] = [ { kind: 'shutdown', payload: {} }, { kind: 'shutdownAck', payload: {} }, + { kind: 'shutdownAck', payload: { activeRequests: ZERO } }, + { kind: 'shutdownAck', payload: { activeRequests: PID } }, { kind: 'pong', payload: { pid: PID, residentMemoryBytes: MEMORY_BYTES, uptimeMs: UPTIME_MS } } ]; @@ -21,6 +23,11 @@ it.each(MESSAGES)('round-trips lifecycle message $kind', (message: DaemonControl expect(decodeDaemonControlMessage(encodeDaemonControlMessage(message))).toEqual(message); }); +it.each([NEGATIVE, FRACTION, '1'])('rejects an invalid shutdown active request count %s', (value: unknown) => { + const json: string = JSON.stringify({ kind: 'shutdownAck', payload: { activeRequests: value } }); + expect(() => decodeDaemonControlMessage(new TextEncoder().encode(json))).toThrow('activeRequests'); +}); + it.each([ZERO, NEGATIVE, FRACTION, '42'])('rejects an invalid daemon PID %s', (pid: unknown) => { const json: string = JSON.stringify({ kind: 'pong', payload: { pid, uptimeMs: UPTIME_MS } }); expect(() => decodeDaemonControlMessage(new TextEncoder().encode(json))).toThrow('pid'); diff --git a/libraries/rush-daemon/src/DaemonControlSession.ts b/libraries/rush-daemon/src/DaemonControlSession.ts index 5189f6ffec..7917047955 100644 --- a/libraries/rush-daemon/src/DaemonControlSession.ts +++ b/libraries/rush-daemon/src/DaemonControlSession.ts @@ -32,6 +32,7 @@ import type { IDaemonInteractiveConnection } from './DaemonInteractiveConnection import { MAX_REQUESTS_PER_CONNECTION } from './DaemonConnectionLimits'; import { DaemonRequestDispatchError } from './DaemonRequestDispatcher'; import type { DaemonRequestDispatcher } from './DaemonRequestDispatcher'; +import type { DaemonShutdownError } from './DaemonShutdownError'; import { DaemonWireRequestClient } from './DaemonWireRequestClient'; import { InteractiveInputRoutingError, @@ -49,6 +50,8 @@ export interface IDaemonControlSessionOptions { readonly onError: (error: Error) => void; readonly onRequestStarted?: () => () => void; readonly onShutdownRequested: () => void; + /** Counts requests running on every connection, reported in the shutdown acknowledgement. */ + readonly getActiveRequestCount?: () => number; readonly getWorkspaceStatus?: () => IDaemonWorkspaceStatus; } @@ -103,11 +106,15 @@ export class DaemonControlSession { options.onInteractiveConnection?.(this.#interactiveConnection); } - public closeAsync(drainRequests: boolean = false): Promise { - this.#closePromise ??= this.#closeOnceAsync(drainRequests); + public closeAsync(drainRequests: boolean = false, reason?: DaemonShutdownError): Promise { + this.#closePromise ??= this.#closeOnceAsync(drainRequests, reason); return this.#closePromise; } + public get activeRequestCount(): number { + return this.#requestById.size; + } + async #handleFrameSafelyAsync(frame: IDaemonFrame): Promise { try { await this.#onFrameAsync(frame); @@ -240,7 +247,11 @@ export class DaemonControlSession { 'Daemon shutdown requires a lifecycle-capable protocol version.' ); } - await this.#enqueueControlAsync({ kind: 'shutdownAck', payload: {} }); + const activeRequests: number | undefined = this.#options.getActiveRequestCount?.(); + await this.#enqueueControlAsync({ + kind: 'shutdownAck', + payload: activeRequests === undefined ? {} : { activeRequests } + }); this.#options.onShutdownRequested(); } @@ -443,8 +454,8 @@ export class DaemonControlSession { } } - async #closeOnceAsync(drainRequests: boolean = false): Promise { - const closeReason: Error = new Error('The daemon control session is closing.'); + async #closeOnceAsync(drainRequests: boolean = false, reason?: DaemonShutdownError): Promise { + const closeReason: Error = reason ?? new Error('The daemon control session is closing.'); if (drainRequests) { const pending: Promise[]> = Promise.allSettled( Array.from(this.#requestById.values(), (state: IRequestState) => state.completion) diff --git a/libraries/rush-daemon/src/DaemonShutdownError.ts b/libraries/rush-daemon/src/DaemonShutdownError.ts new file mode 100644 index 0000000000..a3aa060a0e --- /dev/null +++ b/libraries/rush-daemon/src/DaemonShutdownError.ts @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +/** + * What initiated a daemon shutdown. + * + * @beta + */ +export type DaemonShutdownInitiator = 'controlClient' | 'signal' | 'idleTimeout' | 'restart' | 'host'; + +/** + * Options for {@link DaemonShutdownError}. + * + * @beta + */ +export interface IDaemonShutdownErrorOptions { + readonly initiator: DaemonShutdownInitiator; + /** The process signal name, when the initiator is `signal`. */ + readonly signal?: string; +} + +/** + * The typed reason used to abort requests that were still running when the daemon shut down. + * + * @remarks + * Its message is delivered to the affected clients as the request's error message. + * + * @beta + */ +export class DaemonShutdownError extends Error { + public readonly initiator: DaemonShutdownInitiator; + public readonly signal: string | undefined; + + public constructor(options: IDaemonShutdownErrorOptions) { + super( + `The Rush daemon was shut down (${describeInitiator(options)}) while this request was running; ` + + 're-run the command.' + ); + this.name = 'DaemonShutdownError'; + this.initiator = options.initiator; + this.signal = options.signal; + } +} + +function describeInitiator(options: IDaemonShutdownErrorOptions): string { + switch (options.initiator) { + case 'controlClient': + return 'requested by "rush-client daemon stop" or "daemon restart"'; + case 'signal': + return `the daemon process received ${options.signal ?? 'a termination signal'}`; + case 'idleTimeout': + return 'idle timeout'; + case 'restart': + return 'the daemon restarted to apply workspace changes'; + case 'host': + return 'the daemon host was closed'; + } +} + +/** Returns the shutdown reason if the signal was aborted because the daemon shut down. */ +export function getDaemonShutdownReason(signal: AbortSignal): DaemonShutdownError | undefined { + return signal.aborted && signal.reason instanceof DaemonShutdownError ? signal.reason : undefined; +} diff --git a/libraries/rush-daemon/src/EngineTerminalProvider.ts b/libraries/rush-daemon/src/EngineTerminalProvider.ts index 13c8b48d5e..a64309ce23 100644 --- a/libraries/rush-daemon/src/EngineTerminalProvider.ts +++ b/libraries/rush-daemon/src/EngineTerminalProvider.ts @@ -16,13 +16,26 @@ export class EngineTerminalProvider implements ITerminalProvider { else this.#messages.push({ text, severity }); } + /** + * Drains buffered diagnostics into the failure description, so that they belong to the failing request + * and are never replayed into a later request. + */ public describeError(error: unknown): string { return [ - ...this.#messages.map(({ text }) => text), + ...this.#messages.splice(0).map(({ text }) => text), error instanceof Error ? error.message : String(error) ].join('\n'); } + public get hasBufferedMessages(): boolean { + return this.#messages.length > 0; + } + + /** Discards diagnostics buffered by an earlier request before a new request starts using this terminal. */ + public discardBufferedMessages(): void { + this.#messages.length = 0; + } + public attach(graph: IOperationGraph): void { if (!('eventSink' in graph)) throw new Error('The native graph does not expose its operation event sink.'); diff --git a/libraries/rush-daemon/src/PhasedRequestRouter.ts b/libraries/rush-daemon/src/PhasedRequestRouter.ts index 0903eaf136..b8981dd64c 100644 --- a/libraries/rush-daemon/src/PhasedRequestRouter.ts +++ b/libraries/rush-daemon/src/PhasedRequestRouter.ts @@ -21,6 +21,7 @@ import { PhasedRequestEventSink } from './PhasedRequestEventSink'; import { PhasedRequestEventMultiplexer } from './PhasedRequestEventMultiplexer'; import type { IPhasedRequestClient } from './PhasedRequestClient'; import { DaemonRequiresInProcessError, evaluateDaemonTerminalPolicy } from './DaemonTerminalPolicy'; +import { getDaemonShutdownReason } from './DaemonShutdownError'; import type { IInteractiveRequestSession } from './InteractiveRequestInputRouter'; import { classifyRushCommand } from './RushCommandRequestPolicy'; import { @@ -563,7 +564,7 @@ class PhasedRequestBatchCoordinator { : []; const result: IDaemonPhasedRequestResult = createPhasedCommandResult({ aborted, - error: combineErrors(executionError, cleanupErrors), + error: combineErrors(executionError ?? getDaemonShutdownReason(entry.client.abortSignal), cleanupErrors), graphStatus: getClientGraphStatus(aborted, operationOutcomes), operationOutcomes, requestId: entry.request.requestId, @@ -928,7 +929,7 @@ async function writeAbortedResultAsync( const result: IDaemonPhasedRequestResult = { ...createPhasedCommandResult({ aborted: true, - error: combineErrors(undefined, cleanupErrors), + error: combineErrors(getDaemonShutdownReason(client.abortSignal), cleanupErrors), graphStatus: OperationStatus.Aborted, operationOutcomes: [], requestId, diff --git a/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts b/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts index 7a1793510a..cecdb1772f 100644 --- a/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts +++ b/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts @@ -216,11 +216,20 @@ export class ProductionDaemonRequestResolver implements IDaemonRequestResolver { getChangedOperations(invalidationOptions) }); const components: IWorkspaceSessionComponents = await factory.createAsync(options); + let hasReconciled: boolean = false; return { ...components, reconcileInvalidationsAsync: async () => { - const result: IWorkspaceInvalidationReconciliation = - await components.reconcileInvalidationsAsync!(); + // The graph keeps the binding request's terminal. Diagnostics buffered before a later request + // starts belong to an earlier request; the binding request's own diagnostics are kept. + if (hasReconciled) terminal.discardBufferedMessages(); + hasReconciled = true; + let result: IWorkspaceInvalidationReconciliation; + try { + result = await components.reconcileInvalidationsAsync!(); + } catch (error) { + throw attachBufferedDiagnostics(terminal, error); + } if (!engine.isIncremental) engine.operationGraph.invalidateOperations(undefined, 'rebuild'); return result; } @@ -240,6 +249,19 @@ export class ProductionDaemonRequestResolver implements IDaemonRequestResolver { } } +function attachBufferedDiagnostics(terminal: EngineTerminalProvider, error: unknown): unknown { + if (error instanceof WorkspaceEngineRecreationRequiredError) { + // The replacement engine gets a fresh terminal; the stale diagnostics must not reach a later request. + terminal.discardBufferedMessages(); + return error; + } + if (!terminal.hasBufferedMessages) return error; + if (!(error instanceof Error)) return new Error(terminal.describeError(error), { cause: error }); + // Keep the error's identity and type, which callers use for classification. + error.message = terminal.describeError(error); + return error; +} + function environmentIdentity(environment: Readonly>): string { return JSON.stringify( Object.entries(environment) diff --git a/libraries/rush-daemon/src/RushDaemonHost.ts b/libraries/rush-daemon/src/RushDaemonHost.ts index 6aa4989ac3..17ff4a7bed 100644 --- a/libraries/rush-daemon/src/RushDaemonHost.ts +++ b/libraries/rush-daemon/src/RushDaemonHost.ts @@ -18,6 +18,7 @@ import { DaemonIdleTimer } from './DaemonIdleTimer'; import type { IDaemonInteractiveConnection } from './DaemonInteractiveConnection'; import { DaemonRequestDispatcher } from './DaemonRequestDispatcher'; import type { IDaemonRequestResolver } from './DaemonRequestDispatcher'; +import { DaemonShutdownError, type DaemonShutdownInitiator } from './DaemonShutdownError'; import { WorkspaceSession } from './WorkspaceSession'; import type { IWorkspaceSession, WorkspaceSessionFactory } from './WorkspaceSession'; import { WorkspaceSessionProvider } from './WorkspaceSessionProvider'; @@ -180,7 +181,12 @@ export class RushDaemonHost { }, onError: (error: Error) => options.onError?.(error), onRequestStarted: () => idleTimer.acquire(), - onShutdownRequested: requestShutdown + onShutdownRequested: () => requestShutdown('controlClient'), + getActiveRequestCount: () => { + let count: number = 0; + for (const activeSession of sessions) count += activeSession.activeRequestCount; + return count; + } }); sessions.add(session); if (lifecycle.closing) { @@ -223,13 +229,13 @@ export class RushDaemonHost { function requestRestart(plan: IWorkspaceProcessRestartPlan): void { host.#requestRestart(plan); } - function requestShutdown(): void { - void host.closeAsync().catch((error: Error) => { + function requestShutdown(initiator: DaemonShutdownInitiator): void { + void host.closeAsync(new DaemonShutdownError({ initiator })).catch((error: Error) => { if (options.onError) options.onError(error); else process.emitWarning(error); }); } - idleTimer.start(requestShutdown); + idleTimer.start(() => requestShutdown('idleTimeout')); return host; } @@ -248,9 +254,13 @@ export class RushDaemonHost { return this.#readWorkspaceStatus(); } - /** Closes active connections, stops listening, and removes transport artifacts. */ - public closeAsync(): Promise { - this.#closePromise ??= this.#closeOnceAsync().finally(() => { + /** + * Closes active connections, stops listening, and removes transport artifacts. + * + * @param reason - Delivered to requests that are still running; only the first close call's reason is used. + */ + public closeAsync(reason?: DaemonShutdownError): Promise { + this.#closePromise ??= this.#closeOnceAsync(reason).finally(() => { this.#notifyClosed?.(); if (!this.#restartPromise) this.#resolveRestart?.(undefined); }); @@ -272,7 +282,7 @@ export class RushDaemonHost { } async #restartOnceAsync(plan: IWorkspaceProcessRestartPlan): Promise { - await this.closeAsync(); + await this.closeAsync(new DaemonShutdownError({ initiator: 'restart' })); if (plan.failure) throw plan.failure; if (!plan.launch) throw new Error('A successor was not selected.'); const paths: IDaemonPaths = resolveDaemonPathsFromProcess( @@ -297,7 +307,7 @@ export class RushDaemonHost { } } - async #closeOnceAsync(): Promise { + async #closeOnceAsync(reason: DaemonShutdownError | undefined): Promise { this.#idleTimer[Symbol.dispose](); this.#lifecycle.closing = true; const errors: unknown[] = []; @@ -305,7 +315,7 @@ export class RushDaemonHost { // A failed standalone host must not exit naturally and become reclaimable over unjoined children. const sessionSettlements: PromiseSettledResult[] = await Promise.allSettled( Array.from(this.#sessions, (session: DaemonControlSession) => - session.closeAsync(!!this.#restartPromise) + session.closeAsync(!!this.#restartPromise, reason ?? new DaemonShutdownError({ initiator: 'host' })) ) ); for (const settlement of sessionSettlements) { diff --git a/libraries/rush-daemon/src/index.ts b/libraries/rush-daemon/src/index.ts index 89039a0c43..b895226872 100644 --- a/libraries/rush-daemon/src/index.ts +++ b/libraries/rush-daemon/src/index.ts @@ -57,6 +57,11 @@ export { type IGlobalCommandRequestResult } from './GlobalCommandRequestRouter'; export { RushDaemonHost, type IRushDaemonHostOptions } from './RushDaemonHost'; +export { + DaemonShutdownError, + type DaemonShutdownInitiator, + type IDaemonShutdownErrorOptions +} from './DaemonShutdownError'; export { serveRushDaemonAsync, type IRushDaemonServeOptions } from './serveRushDaemon'; export { WorkspaceEngineComponentFactory, diff --git a/libraries/rush-daemon/src/serveRushDaemon.ts b/libraries/rush-daemon/src/serveRushDaemon.ts index 929b28bbf0..1baec925df 100644 --- a/libraries/rush-daemon/src/serveRushDaemon.ts +++ b/libraries/rush-daemon/src/serveRushDaemon.ts @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import { DaemonShutdownError } from './DaemonShutdownError'; import { RushDaemonHost } from './RushDaemonHost'; import type { IRushDaemonHostOptions } from './RushDaemonHost'; import { getInstalledWorkspaceSuccessorLaunchAsync } from './WorkspaceProcessRestart'; @@ -41,7 +42,7 @@ export async function serveRushDaemonAsync(options: IRushDaemonServeOptions): Pr }); await options.onReady?.(host); await waitForShutdownAsync(host, signalRegistration.signal); - await host.closeAsync(); + await host.closeAsync(getShutdownReason(signalRegistration.signal)); await host.restartCompleted; } finally { signalRegistration.dispose(); @@ -49,6 +50,13 @@ export async function serveRushDaemonAsync(options: IRushDaemonServeOptions): Pr } } +function getShutdownReason(signal: AbortSignal): DaemonShutdownError | undefined { + if (!signal.aborted) return undefined; + return signal.reason instanceof DaemonShutdownError + ? signal.reason + : new DaemonShutdownError({ initiator: 'host' }); +} + interface IShutdownSignalRegistration { readonly signal: AbortSignal; readonly dispose: () => void; @@ -56,7 +64,8 @@ interface IShutdownSignalRegistration { function createProcessShutdownSignal(): IShutdownSignalRegistration { const controller: AbortController = new AbortController(); - const onSignal: () => void = () => controller.abort(); + const onSignal: (signal: NodeJS.Signals) => void = (signal: NodeJS.Signals) => + controller.abort(new DaemonShutdownError({ initiator: 'signal', signal })); process.once('SIGINT', onSignal); process.once('SIGTERM', onSignal); return { diff --git a/libraries/rush-daemon/src/test/DaemonShutdown.test.ts b/libraries/rush-daemon/src/test/DaemonShutdown.test.ts index 284d26a697..c2c8c91e3d 100644 --- a/libraries/rush-daemon/src/test/DaemonShutdown.test.ts +++ b/libraries/rush-daemon/src/test/DaemonShutdown.test.ts @@ -38,7 +38,7 @@ describe('daemon management shutdown', () => { await client.sendControlAsync(createDaemonHello(DAEMON_PROTOCOL_VERSION)); expect((await client.readControlAsync()).kind).toBe('helloAck'); await client.sendControlAsync({ kind: 'shutdown', payload: {} }); - expect(await client.readControlAsync()).toEqual({ kind: 'shutdownAck', payload: {} }); + expect(await client.readControlAsync()).toEqual({ kind: 'shutdownAck', payload: { activeRequests: 0 } }); await client.closed; await host.closed; expect(readDaemonLockfile(host.paths.lockfilePath)).toBeUndefined(); diff --git a/libraries/rush-daemon/src/test/EngineTerminalProvider.test.ts b/libraries/rush-daemon/src/test/EngineTerminalProvider.test.ts new file mode 100644 index 0000000000..b2db248093 --- /dev/null +++ b/libraries/rush-daemon/src/test/EngineTerminalProvider.test.ts @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { TerminalProviderSeverity } from '@rushstack/terminal'; + +import { EngineTerminalProvider } from '../EngineTerminalProvider'; + +describe(EngineTerminalProvider.name, () => { + it('drains buffered diagnostics into the failure description so a later request cannot replay them', () => { + const terminal: EngineTerminalProvider = new EngineTerminalProvider(); + terminal.write('Permission denied', TerminalProviderSeverity.error); + expect(terminal.hasBufferedMessages).toBe(true); + expect(terminal.describeError(new Error('snapshot failed'))).toBe('Permission denied\nsnapshot failed'); + expect(terminal.hasBufferedMessages).toBe(false); + expect(terminal.describeError(new Error('next request'))).toBe('next request'); + }); + + it('discards diagnostics buffered by an earlier request', () => { + const terminal: EngineTerminalProvider = new EngineTerminalProvider(); + terminal.write('stale', TerminalProviderSeverity.warning); + terminal.discardBufferedMessages(); + expect(terminal.hasBufferedMessages).toBe(false); + expect(terminal.describeError('failure')).toBe('failure'); + }); +}); diff --git a/libraries/rush-daemon/src/test/ProductionDaemonRequestResolver.test.ts b/libraries/rush-daemon/src/test/ProductionDaemonRequestResolver.test.ts index b0d02fd298..01f38bfd80 100644 --- a/libraries/rush-daemon/src/test/ProductionDaemonRequestResolver.test.ts +++ b/libraries/rush-daemon/src/test/ProductionDaemonRequestResolver.test.ts @@ -34,6 +34,7 @@ import { stopSuccessorAsync } from './WorkspaceLifecycleTestProcess'; import { removeTestFolderAsync } from './TestProcessExit'; import { readDaemonLockfile } from '@rushstack/rush-daemon-transport'; import { EngineTerminalProvider } from '../EngineTerminalProvider'; +import { DaemonShutdownError } from '../DaemonShutdownError'; import { getInstalledWorkspaceSuccessorLaunchAsync } from '../WorkspaceProcessRestart'; import type { GetWorkspaceSuccessorLaunchAsync, @@ -1436,4 +1437,67 @@ process.exit(23); await fixture[Symbol.asyncDispose](); } }); + + const canRevokeReadAccess: boolean = process.platform !== 'win32' && process.getuid?.() !== 0; + (canRevokeReadAccess ? it : it.skip)( + 'reports a warm snapshot failure to the failing request and never replays it into the next request', + async () => { + const fixture: IFixture = await createFixtureAsync(); + const inputPath: string = path.join(fixture.repoRoot, 'projects/a/input.txt'); + try { + await runAsync(fixture, 'initial', ['build', '--only', 'a']); + fs.writeFileSync(inputPath, 'unreadable'); + fs.chmodSync(inputPath, 0); + const failed: ITerminalExchange = await runAsync(fixture, 'unreadable', ['build', '--only', 'a']); + expect(failed.terminal).toMatchObject({ + kind: 'requestRejected', + payload: { + message: expect.stringMatching( + /Permission denied[\s\S]*Rush could not capture the next workspace inputs snapshot\./ + ) + } + }); + fs.chmodSync(inputPath, 0o644); + const recovered: ITerminalExchange = await runAsync(fixture, 'recovered', ['build', '--only', 'a']); + expect(recovered.terminal).toMatchObject({ kind: 'requestResult', payload: { exitCode: 0 } }); + const output: string = [ + logText(recovered), + ...recovered.frames + .filter((frame) => frame.kind === DaemonFrameType.event) + .map((frame) => JSON.stringify(decodeDaemonEventFrame(frame.payload))) + ].join('\n'); + expect(output).not.toContain('Permission denied'); + expect(output).not.toContain('state of the repo'); + } finally { + if (fs.existsSync(inputPath)) fs.chmodSync(inputPath, 0o644); + await fixture[Symbol.asyncDispose](); + } + } + ); + + it('aborts an in-flight build with the typed daemon shutdown reason', async () => { + const fixture: IFixture = await createFixtureAsync(); + const gate: INativeScriptGate = await createNativeScriptGateAsync(fixture.repoRoot, 'a'); + try { + const victim: Promise = runAsync(fixture, 'victim', ['build', '--only', 'a']); + await gate.entered; + const closing: Promise = fixture.host.closeAsync( + new DaemonShutdownError({ initiator: 'signal', signal: 'SIGTERM' }) + ); + await gate.releaseAsync(); + expect((await victim).terminal).toMatchObject({ + kind: 'requestResult', + payload: { + aborted: true, + errorMessage: expect.stringMatching( + /^The Rush daemon was shut down \(the daemon process received SIGTERM\) while this request was running; re-run the command\.$/ + ) + } + }); + await closing; + } finally { + await gate.releaseAsync(); + await fixture[Symbol.asyncDispose](); + } + }); }); From 1e987f93115d6518afaf09bb445d3ff47d618e61 Mon Sep 17 00:00:00 2001 From: selarkin Date: Wed, 23 Sep 2026 19:19:29 -0700 Subject: [PATCH 2/9] Update API reports Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- common/reviews/api/rush-client-core.api.md | 3 ++- .../reviews/api/rush-daemon-protocol.api.md | 4 +++- common/reviews/api/rush-daemon.api.md | 21 ++++++++++++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/common/reviews/api/rush-client-core.api.md b/common/reviews/api/rush-client-core.api.md index 2305d7bad2..47815ef352 100644 --- a/common/reviews/api/rush-client-core.api.md +++ b/common/reviews/api/rush-client-core.api.md @@ -13,6 +13,7 @@ import { IDaemonPongMessage } from '@rushstack/rush-daemon-protocol'; import { IDaemonProtocolVersion } from '@rushstack/rush-daemon-protocol'; import { IDaemonRequestEnvelope } from '@rushstack/rush-daemon-protocol'; import { IDaemonRequestRejectedMessage } from '@rushstack/rush-daemon-protocol'; +import { IDaemonShutdownAckMessage } from '@rushstack/rush-daemon-protocol'; import type { Readable } from 'node:stream'; // @beta @@ -29,7 +30,7 @@ export class DaemonClient { static connectAsync(options: IDaemonClientConnectOptions): Promise; executeAsync(options: IDaemonClientExecuteOptions): Promise; get protocolVersion(): IDaemonProtocolVersion; - shutdownAsync(timeoutMs?: number): Promise; + shutdownAsync(timeoutMs?: number): Promise; get status(): Promise; } diff --git a/common/reviews/api/rush-daemon-protocol.api.md b/common/reviews/api/rush-daemon-protocol.api.md index 409700cfaf..9beeebd56a 100644 --- a/common/reviews/api/rush-daemon-protocol.api.md +++ b/common/reviews/api/rush-daemon-protocol.api.md @@ -591,7 +591,9 @@ export interface IDaemonShutdownAckMessage { // (undocumented) readonly kind: 'shutdownAck'; // (undocumented) - readonly payload: Record; + readonly payload: { + readonly activeRequests?: number; + }; } // @beta diff --git a/common/reviews/api/rush-daemon.api.md b/common/reviews/api/rush-daemon.api.md index bbad30a4b9..3c95a6c87e 100644 --- a/common/reviews/api/rush-daemon.api.md +++ b/common/reviews/api/rush-daemon.api.md @@ -66,6 +66,18 @@ export class DaemonRequiresInProcessError extends Error { readonly policy: IDaemonTerminalPolicyResult; } +// @beta +export class DaemonShutdownError extends Error { + constructor(options: IDaemonShutdownErrorOptions); + // (undocumented) + readonly initiator: DaemonShutdownInitiator; + // (undocumented) + readonly signal: string | undefined; +} + +// @beta +export type DaemonShutdownInitiator = 'controlClient' | 'signal' | 'idleTimeout' | 'restart' | 'host'; + // @beta export type DispatchWorkspaceRequestAsync = (options: IDispatchWorkspaceRequestOptions) => Promise; @@ -177,6 +189,13 @@ export interface IDaemonRequestResolver { readonly workspaceLifecycle?: IWorkspaceResolverLifecycle; } +// @beta +export interface IDaemonShutdownErrorOptions { + // (undocumented) + readonly initiator: DaemonShutdownInitiator; + readonly signal?: string; +} + // @beta export interface IDispatchWorkspaceRequestOptions { // (undocumented) @@ -723,7 +742,7 @@ export type ResolvedDaemonRequest = IResolvedDaemonPhasedRequest | IResolvedDaem // @beta export class RushDaemonHost { - closeAsync(): Promise; + closeAsync(reason?: DaemonShutdownError): Promise; readonly closed: Promise; getWorkspaceSessionAsync(): Promise; // (undocumented) From a107cd68e91dc9011e23e9799ffd44c80df5faf2 Mon Sep 17 00:00:00 2001 From: selarkin Date: Wed, 23 Sep 2026 19:24:26 -0700 Subject: [PATCH 3/9] [rush-daemon] Report the shutdown reason for requests aborted before engine initialization Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts b/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts index cecdb1772f..9dd7fe0baf 100644 --- a/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts +++ b/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts @@ -31,6 +31,7 @@ import { } from './WorkspaceEngineComponentFactory'; import type { IWorkspaceSession, IWorkspaceSessionComponents } from './WorkspaceSession'; import { EngineTerminalProvider } from './EngineTerminalProvider'; +import { getDaemonShutdownReason } from './DaemonShutdownError'; import type { IWorkspaceResolverLifecycle } from './WorkspaceResolverLifecycle'; /** @@ -170,7 +171,8 @@ export class ProductionDaemonRequestResolver implements IDaemonRequestResolver { if (abortSignal.aborted) throw new DaemonRequestDispatchError( 'routingFailed', - 'The request was cancelled before engine initialization.' + getDaemonShutdownReason(abortSignal)?.message ?? + 'The request was cancelled before engine initialization.' ); return command; } From 09ec313ae0da4e8c3d2ba9cc8f65e99baee04dcd Mon Sep 17 00:00:00 2001 From: selarkin Date: Wed, 23 Sep 2026 19:50:56 -0700 Subject: [PATCH 4/9] [rush-daemon] Move request-scoped reconcile diagnostics into EngineTerminalProvider Keeps the ProductionDaemonRequestResolver change to a one-line call so it composes with other open daemon PRs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../rush-daemon/src/EngineTerminalProvider.ts | 31 ++++++++++++++++++ .../src/ProductionDaemonRequestResolver.ts | 27 ++-------------- .../src/test/EngineTerminalProvider.test.ts | 32 +++++++++++++++++++ 3 files changed, 66 insertions(+), 24 deletions(-) diff --git a/libraries/rush-daemon/src/EngineTerminalProvider.ts b/libraries/rush-daemon/src/EngineTerminalProvider.ts index a64309ce23..88345d0372 100644 --- a/libraries/rush-daemon/src/EngineTerminalProvider.ts +++ b/libraries/rush-daemon/src/EngineTerminalProvider.ts @@ -4,12 +4,15 @@ import type { IOperationGraph, _IOperationGraphEventSink } from '@microsoft/rush-lib'; import { TerminalProviderSeverity, type ITerminalProvider } from '@rushstack/terminal'; +import { WorkspaceEngineRecreationRequiredError } from './WorkspaceEngineComponentFactory'; + export class EngineTerminalProvider implements ITerminalProvider { public readonly supportsColor: boolean = false; public readonly eolCharacter: string = '\n'; readonly #messages: Array<{ text: string; severity: TerminalProviderSeverity }> = []; #graph: (IOperationGraph & { eventSink?: _IOperationGraphEventSink }) | undefined; #executing: boolean = false; + #hasReconciled: boolean = false; public write(text: string, severity: TerminalProviderSeverity): void { if (this.#executing) this.#emit(text, severity); @@ -36,6 +39,34 @@ export class EngineTerminalProvider implements ITerminalProvider { this.#messages.length = 0; } + /** + * Runs a warm reconcile with request-scoped diagnostics. The graph keeps the binding request's terminal, so + * diagnostics buffered before a later request's reconcile belong to an earlier request and are discarded; the + * binding request's own diagnostics are kept. A failure carries the diagnostics buffered while reconciling. + */ + public async reconcileWithRequestDiagnosticsAsync(reconcileAsync: () => Promise): Promise { + if (this.#hasReconciled) this.discardBufferedMessages(); + this.#hasReconciled = true; + try { + return await reconcileAsync(); + } catch (error) { + throw this.#attachBufferedDiagnostics(error); + } + } + + #attachBufferedDiagnostics(error: unknown): unknown { + if (error instanceof WorkspaceEngineRecreationRequiredError) { + // The replacement engine gets a fresh terminal; the stale diagnostics must not reach a later request. + this.discardBufferedMessages(); + return error; + } + if (!this.hasBufferedMessages) return error; + if (!(error instanceof Error)) return new Error(this.describeError(error), { cause: error }); + // Keep the error's identity and type, which callers use for classification. + error.message = this.describeError(error); + return error; + } + public attach(graph: IOperationGraph): void { if (!('eventSink' in graph)) throw new Error('The native graph does not expose its operation event sink.'); diff --git a/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts b/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts index 9dd7fe0baf..0978b254d7 100644 --- a/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts +++ b/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts @@ -218,20 +218,12 @@ export class ProductionDaemonRequestResolver implements IDaemonRequestResolver { getChangedOperations(invalidationOptions) }); const components: IWorkspaceSessionComponents = await factory.createAsync(options); - let hasReconciled: boolean = false; return { ...components, reconcileInvalidationsAsync: async () => { - // The graph keeps the binding request's terminal. Diagnostics buffered before a later request - // starts belong to an earlier request; the binding request's own diagnostics are kept. - if (hasReconciled) terminal.discardBufferedMessages(); - hasReconciled = true; - let result: IWorkspaceInvalidationReconciliation; - try { - result = await components.reconcileInvalidationsAsync!(); - } catch (error) { - throw attachBufferedDiagnostics(terminal, error); - } + const result: IWorkspaceInvalidationReconciliation = await terminal.reconcileWithRequestDiagnosticsAsync( + () => components.reconcileInvalidationsAsync!() + ); if (!engine.isIncremental) engine.operationGraph.invalidateOperations(undefined, 'rebuild'); return result; } @@ -251,19 +243,6 @@ export class ProductionDaemonRequestResolver implements IDaemonRequestResolver { } } -function attachBufferedDiagnostics(terminal: EngineTerminalProvider, error: unknown): unknown { - if (error instanceof WorkspaceEngineRecreationRequiredError) { - // The replacement engine gets a fresh terminal; the stale diagnostics must not reach a later request. - terminal.discardBufferedMessages(); - return error; - } - if (!terminal.hasBufferedMessages) return error; - if (!(error instanceof Error)) return new Error(terminal.describeError(error), { cause: error }); - // Keep the error's identity and type, which callers use for classification. - error.message = terminal.describeError(error); - return error; -} - function environmentIdentity(environment: Readonly>): string { return JSON.stringify( Object.entries(environment) diff --git a/libraries/rush-daemon/src/test/EngineTerminalProvider.test.ts b/libraries/rush-daemon/src/test/EngineTerminalProvider.test.ts index b2db248093..2c6fcbfdf7 100644 --- a/libraries/rush-daemon/src/test/EngineTerminalProvider.test.ts +++ b/libraries/rush-daemon/src/test/EngineTerminalProvider.test.ts @@ -4,6 +4,7 @@ import { TerminalProviderSeverity } from '@rushstack/terminal'; import { EngineTerminalProvider } from '../EngineTerminalProvider'; +import { WorkspaceEngineRecreationRequiredError } from '../WorkspaceEngineComponentFactory'; describe(EngineTerminalProvider.name, () => { it('drains buffered diagnostics into the failure description so a later request cannot replay them', () => { @@ -22,4 +23,35 @@ describe(EngineTerminalProvider.name, () => { expect(terminal.hasBufferedMessages).toBe(false); expect(terminal.describeError('failure')).toBe('failure'); }); + + it('scopes reconcile diagnostics to the request whose reconcile produced them', async () => { + const terminal: EngineTerminalProvider = new EngineTerminalProvider(); + terminal.write('binding request diagnostic', TerminalProviderSeverity.warning); + const failure: RangeError = new RangeError('could not capture'); + await expect( + terminal.reconcileWithRequestDiagnosticsAsync(async () => { + terminal.write('Permission denied', TerminalProviderSeverity.error); + throw failure; + }) + ).rejects.toBe(failure); + expect(failure.message).toBe('binding request diagnostic\nPermission denied\ncould not capture'); + + terminal.write('stale', TerminalProviderSeverity.warning); + await expect(terminal.reconcileWithRequestDiagnosticsAsync(async () => 'ok')).resolves.toBe('ok'); + expect(terminal.hasBufferedMessages).toBe(false); + }); + + it('drops diagnostics when the engine must be recreated', async () => { + const terminal: EngineTerminalProvider = new EngineTerminalProvider(); + const recreate: WorkspaceEngineRecreationRequiredError = new WorkspaceEngineRecreationRequiredError(); + const message: string = recreate.message; + await expect( + terminal.reconcileWithRequestDiagnosticsAsync(async () => { + terminal.write('stale', TerminalProviderSeverity.error); + throw recreate; + }) + ).rejects.toBe(recreate); + expect(recreate.message).toBe(message); + expect(terminal.hasBufferedMessages).toBe(false); + }); }); From d2b27bf4a3f4a3bd21bcd5ca7586ebafde046445 Mon Sep 17 00:00:00 2001 From: selarkin Date: Thu, 24 Sep 2026 11:18:02 -0700 Subject: [PATCH 5/9] [rush-daemon] Address review: shutdown reason for queued and raw-mode requests, protocol minor 11 - Report the shutdown reason for requests aborted while waiting for admission, and prefer it on the client. - Keep the shutdown reason when restoring raw mode fails with it. - Begin shutdown before awaiting the acknowledgement write so the reported count matches the aborted set. - Advertise protocol minor 11 for the shutdownAck activeRequests field. - Keep the daemon stop note accurate for every request kind. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- apps/rush-cli-client/src/daemonCommands.ts | 3 +- apps/rush-cli-client/src/launchClient.ts | 8 ++-- apps/rush-cli-client/src/resultDiagnostics.ts | 10 ++-- .../src/test/resultDiagnostics.test.ts | 14 ++++++ .../src/DaemonLifecycleControl.ts | 5 +- .../src/DaemonProtocolVersion.ts | 5 +- libraries/rush-daemon-protocol/src/index.ts | 1 + .../rush-daemon/src/DaemonControlSession.ts | 5 +- .../rush-daemon/src/PhasedRequestRouter.ts | 10 +++- .../src/WorkspaceRequestLifecycle.ts | 3 +- .../src/test/DaemonRequestWireGlobal.test.ts | 46 +++++++++++++++++++ .../src/test/PhasedRequestInteractive.test.ts | 30 ++++++++++++ 12 files changed, 125 insertions(+), 15 deletions(-) diff --git a/apps/rush-cli-client/src/daemonCommands.ts b/apps/rush-cli-client/src/daemonCommands.ts index ae47a2bf42..de9ee23d84 100644 --- a/apps/rush-cli-client/src/daemonCommands.ts +++ b/apps/rush-cli-client/src/daemonCommands.ts @@ -87,8 +87,7 @@ export async function executeDaemonCommandAsync(options: IDaemonCommandOptions): await writeStreamAsync( process.stderr, Buffer.from( - `rush-client: the daemon was running ${activeRequests} request(s); ` + - 'they were cancelled and their clients were told to re-run the command.\n' + `rush-client: the daemon was running ${activeRequests} request(s); they were cancelled.\n` ) ); } diff --git a/apps/rush-cli-client/src/launchClient.ts b/apps/rush-cli-client/src/launchClient.ts index 55a27bd2e5..73a1370a6d 100644 --- a/apps/rush-cli-client/src/launchClient.ts +++ b/apps/rush-cli-client/src/launchClient.ts @@ -196,14 +196,14 @@ export async function launchClientAsync(rushx: boolean): Promise { } if (outcome.kind === 'result') { process.exitCode = outcome.result.exitCode; - if (outcome.result.admissionErrorCode) { + const diagnostic: string | undefined = getResultDiagnostic(outcome.result); + if (diagnostic) { + await writeStreamAsync(process.stderr, Buffer.from(diagnostic)); + } else if (outcome.result.admissionErrorCode) { await writeStreamAsync( process.stderr, Buffer.from(formatAdmissionFailure(outcome.result.admissionErrorCode, request.admission)) ); - } else { - const diagnostic: string | undefined = getResultDiagnostic(outcome.result); - if (diagnostic) await writeStreamAsync(process.stderr, Buffer.from(diagnostic)); } } else if (outcome.kind === 'rejected') { throw new Error(`Daemon rejected the request (${outcome.rejection.code}): ${outcome.rejection.message}`); diff --git a/apps/rush-cli-client/src/resultDiagnostics.ts b/apps/rush-cli-client/src/resultDiagnostics.ts index e5745617a9..e65be24364 100644 --- a/apps/rush-cli-client/src/resultDiagnostics.ts +++ b/apps/rush-cli-client/src/resultDiagnostics.ts @@ -4,16 +4,20 @@ import type { IDaemonCommandResult } from '@rushstack/rush-daemon-protocol'; /** - * Returns the stderr line that explains a failed daemon result that is not an admission failure, if any. + * Returns the stderr line that explains a failed daemon result, if any. * * @remarks * A non-zero result's error message (for example, a daemon shutdown that aborted the request) is the only * place the daemon reports failures that are not attributed to an operation, so it must not be dropped. - * Admission failures are reported separately by `formatAdmissionFailure`. + * Returns `undefined` for `no-wait` and `wait-timeout` admission failures, which `formatAdmissionFailure` + * explains. */ export function getResultDiagnostic( - result: Pick + result: Pick ): string | undefined { + // A request aborted while waiting for admission carries the reason (such as a daemon shutdown) in its + // error message; other admission failures are explained by `formatAdmissionFailure`. + if (result.admissionErrorCode !== undefined && result.admissionErrorCode !== 'aborted') return undefined; if (result.exitCode !== 0 && result.errorMessage) { return `rush-client: ${result.errorMessage}\n`; } diff --git a/apps/rush-cli-client/src/test/resultDiagnostics.test.ts b/apps/rush-cli-client/src/test/resultDiagnostics.test.ts index 27d2930a40..a1dbec266f 100644 --- a/apps/rush-cli-client/src/test/resultDiagnostics.test.ts +++ b/apps/rush-cli-client/src/test/resultDiagnostics.test.ts @@ -13,6 +13,20 @@ describe(getResultDiagnostic.name, () => { ).toBe('rush-client: The Rush daemon was shut down (idle timeout) while this request was running.\n'); }); + it('prefers the reason of a request aborted while waiting for admission', () => { + expect( + getResultDiagnostic({ exitCode: 1, admissionErrorCode: 'aborted', errorMessage: 'daemon shut down' }) + ).toBe('rush-client: daemon shut down\n'); + }); + + it('leaves no-wait and wait-timeout admission failures to the admission formatter', () => { + expect( + getResultDiagnostic({ exitCode: 1, admissionErrorCode: 'wait-timeout', errorMessage: 'x' }) + ).toBeUndefined(); + expect(getResultDiagnostic({ exitCode: 1, admissionErrorCode: 'no-wait', errorMessage: 'x' })).toBeUndefined(); + expect(getResultDiagnostic({ exitCode: 1, admissionErrorCode: 'aborted' })).toBeUndefined(); + }); + it('stays silent for successful results and failures without a message', () => { expect(getResultDiagnostic({ exitCode: 0, errorMessage: 'ignored' })).toBeUndefined(); expect(getResultDiagnostic({ exitCode: 1 })).toBeUndefined(); diff --git a/libraries/rush-daemon-protocol/src/DaemonLifecycleControl.ts b/libraries/rush-daemon-protocol/src/DaemonLifecycleControl.ts index e591e54844..a7c084ed11 100644 --- a/libraries/rush-daemon-protocol/src/DaemonLifecycleControl.ts +++ b/libraries/rush-daemon-protocol/src/DaemonLifecycleControl.ts @@ -11,7 +11,10 @@ export interface IDaemonShutdownMessage { export interface IDaemonShutdownAckMessage { readonly kind: 'shutdownAck'; readonly payload: { - /** Requests that were still running and will be aborted by this shutdown. Older daemons omit it. */ + /** + * Requests that were still running and will be aborted by this shutdown. Daemons older than + * `DAEMON_SHUTDOWN_ACTIVE_REQUESTS_PROTOCOL_MINOR` omit it. + */ readonly activeRequests?: number; }; } diff --git a/libraries/rush-daemon-protocol/src/DaemonProtocolVersion.ts b/libraries/rush-daemon-protocol/src/DaemonProtocolVersion.ts index a687dae853..b35d1d6883 100644 --- a/libraries/rush-daemon-protocol/src/DaemonProtocolVersion.ts +++ b/libraries/rush-daemon-protocol/src/DaemonProtocolVersion.ts @@ -25,6 +25,9 @@ export const DAEMON_INVOCATION_KIND_PROTOCOL_MINOR: number = 8; /** The first minor supporting native mutations and guaranteed pre-execution restart outcomes. @beta */ export const DAEMON_WORKSPACE_RESTART_PROTOCOL_MINOR: number = 10; +/** The first additive protocol minor whose shutdown acknowledgement reports the active request count. @beta */ +export const DAEMON_SHUTDOWN_ACTIVE_REQUESTS_PROTOCOL_MINOR: number = 11; + /** * A rushd wire protocol version. * @@ -58,7 +61,7 @@ export interface IDaemonProtocolVersion { */ export const DAEMON_PROTOCOL_VERSION: IDaemonProtocolVersion = { major: 0, - minor: DAEMON_WORKSPACE_RESTART_PROTOCOL_MINOR + minor: DAEMON_SHUTDOWN_ACTIVE_REQUESTS_PROTOCOL_MINOR }; /** diff --git a/libraries/rush-daemon-protocol/src/index.ts b/libraries/rush-daemon-protocol/src/index.ts index b170c7e899..1537163377 100644 --- a/libraries/rush-daemon-protocol/src/index.ts +++ b/libraries/rush-daemon-protocol/src/index.ts @@ -29,6 +29,7 @@ export { DAEMON_INVOCATION_KIND_PROTOCOL_MINOR } from './DaemonProtocolVersion'; export { DAEMON_LIFECYCLE_PROTOCOL_MINOR } from './DaemonProtocolVersion'; export { DAEMON_REQUEST_ADMISSION_PROTOCOL_MINOR } from './DaemonProtocolVersion'; export { DAEMON_REQUEST_LIFECYCLE_PROTOCOL_MINOR, DAEMON_PROTOCOL_VERSION } from './DaemonProtocolVersion'; +export { DAEMON_SHUTDOWN_ACTIVE_REQUESTS_PROTOCOL_MINOR } from './DaemonProtocolVersion'; export { DAEMON_WORKSPACE_RESTART_PROTOCOL_MINOR } from './DaemonProtocolVersion'; export { isDaemonProtocolCompatible } from './DaemonProtocolVersion'; export type { IDaemonProtocolVersion } from './DaemonProtocolVersion'; diff --git a/libraries/rush-daemon/src/DaemonControlSession.ts b/libraries/rush-daemon/src/DaemonControlSession.ts index 7917047955..51f9bff38a 100644 --- a/libraries/rush-daemon/src/DaemonControlSession.ts +++ b/libraries/rush-daemon/src/DaemonControlSession.ts @@ -248,11 +248,14 @@ export class DaemonControlSession { ); } const activeRequests: number | undefined = this.#options.getActiveRequestCount?.(); - await this.#enqueueControlAsync({ + // Queue the acknowledgement, then begin shutdown synchronously so the reported count is the set that + // shutdown aborts; closing drains the send queue, so the acknowledgement is still delivered first. + const ackPromise: Promise = this.#enqueueControlAsync({ kind: 'shutdownAck', payload: activeRequests === undefined ? {} : { activeRequests } }); this.#options.onShutdownRequested(); + await ackPromise; } #startRequest(envelope: IDaemonRequestEnvelope): void { diff --git a/libraries/rush-daemon/src/PhasedRequestRouter.ts b/libraries/rush-daemon/src/PhasedRequestRouter.ts index 78f5c0a7b8..938183bb9b 100644 --- a/libraries/rush-daemon/src/PhasedRequestRouter.ts +++ b/libraries/rush-daemon/src/PhasedRequestRouter.ts @@ -21,7 +21,7 @@ import { PhasedRequestEventSink } from './PhasedRequestEventSink'; import { PhasedRequestEventMultiplexer } from './PhasedRequestEventMultiplexer'; import type { IPhasedRequestClient } from './PhasedRequestClient'; import { DaemonRequiresInProcessError, evaluateDaemonTerminalPolicy } from './DaemonTerminalPolicy'; -import { getDaemonShutdownReason } from './DaemonShutdownError'; +import { DaemonShutdownError, getDaemonShutdownReason } from './DaemonShutdownError'; import type { IInteractiveRequestSession } from './InteractiveRequestInputRouter'; import { classifyRushCommand } from './RushCommandRequestPolicy'; import { @@ -1031,7 +1031,13 @@ async function finishAfterAdmissionErrorAsync( return result; } -function combineErrors(executionError: unknown, cleanupErrors: unknown[]): unknown { +function combineErrors(executionError: unknown, allCleanupErrors: unknown[]): unknown { + // Cleanup that fails with the same daemon shutdown reason (for example, restoring raw mode after the + // interactive connection closed) must not hide that reason from the client. + const cleanupErrors: unknown[] = + executionError instanceof DaemonShutdownError + ? allCleanupErrors.filter((error: unknown) => !(error instanceof DaemonShutdownError)) + : allCleanupErrors; if (executionError !== undefined && cleanupErrors.length > 0) { return new AggregateError( [executionError, ...cleanupErrors], diff --git a/libraries/rush-daemon/src/WorkspaceRequestLifecycle.ts b/libraries/rush-daemon/src/WorkspaceRequestLifecycle.ts index da5ca8230a..d3ebbd01cf 100644 --- a/libraries/rush-daemon/src/WorkspaceRequestLifecycle.ts +++ b/libraries/rush-daemon/src/WorkspaceRequestLifecycle.ts @@ -40,6 +40,7 @@ import { getWorkspaceRequestScheduler } from './WorkspaceRequestAdmission'; import { WorkspaceEngineRecreationRequiredError } from './WorkspaceEngineComponentFactory'; +import { getDaemonShutdownReason } from './DaemonShutdownError'; import type { IWorkspaceSession } from './WorkspaceSession'; import type { WorkspaceSessionProvider } from './WorkspaceSessionProvider'; import { assertWorkspaceRequestResourcesHealthy } from './WorkspaceRequestResources'; @@ -259,7 +260,7 @@ export class WorkspaceRequestLifecycle implements IDaemonRequestLifecycle { if (error instanceof RequestSchedulerError && !state.began && !state.terminalAttempted) { await client.interactiveSession.finishAsync(); await client.writeResultAsync({ - ...preExecutionFailure(envelope.requestId, error), + ...preExecutionFailure(envelope.requestId, getDaemonShutdownReason(client.abortSignal) ?? error), aborted: client.abortSignal.aborted, admissionErrorCode: getRequestAdmissionErrorCode(error) }); diff --git a/libraries/rush-daemon/src/test/DaemonRequestWireGlobal.test.ts b/libraries/rush-daemon/src/test/DaemonRequestWireGlobal.test.ts index 86d984c61b..e570988009 100644 --- a/libraries/rush-daemon/src/test/DaemonRequestWireGlobal.test.ts +++ b/libraries/rush-daemon/src/test/DaemonRequestWireGlobal.test.ts @@ -10,6 +10,7 @@ import type { DaemonControlMessage, IDaemonRequestEnvelope } from '@rushstack/ru import type { GlobalCommandExecutor, IDaemonRequestResolver } from '../index'; import { MAX_REQUESTS_PER_CONNECTION } from '../DaemonConnectionLimits'; +import { DaemonShutdownError } from '../DaemonShutdownError'; import { RushDaemonHost } from '../RushDaemonHost'; import type { IRushDaemonHostOptions } from '../RushDaemonHost'; import { TestWorkspaceSession } from './TestWorkspaceSession'; @@ -303,6 +304,51 @@ describe('daemon global request wire integration', () => { } }); + it('tells a request queued for admission that the daemon shut down', async () => { + const repoRoot: string = createRepoRoot(); + const holderStarted: IDeferred = createDeferred(); + const releaseHolder: IDeferred = createDeferred(); + const resolver: IDaemonRequestResolver = new CallbackDaemonRequestResolver(async ({ envelope }) => { + const executorAsync: GlobalCommandExecutor = async () => { + if (envelope.requestId === 'holder') { + holderStarted.resolve(); + await releaseHolder.promise; + } + return { exitCode: 0 }; + }; + return { executor: executorAsync, kind: 'global' }; + }); + const host: RushDaemonHost = await RushDaemonHost.startAsync(createHostOptions(repoRoot, resolver)); + const clients: DaemonRequestWireClient[] = await Promise.all([connectAsync(host), connectAsync(host)]); + const shutdown: DaemonShutdownError = new DaemonShutdownError({ initiator: 'controlClient' }); + try { + await clients[0].sendControlAsync({ + kind: 'requestStart', + payload: createWireEnvelope('holder', 'custom', repoRoot) + }); + await holderStarted.promise; + await clients[1].sendControlAsync({ + kind: 'requestStart', + payload: createWireEnvelope('queued', 'custom', repoRoot) + }); + expect(await clients[1].readControlAsync()).toMatchObject({ + kind: 'queuePosition', + payload: { requestId: 'queued' } + }); + const closePromise: Promise = host.closeAsync(shutdown); + releaseHolder.resolve(); + expect((await clients[1].readTerminalAsync('queued')).terminal).toMatchObject({ + kind: 'requestResult', + payload: { aborted: true, admissionErrorCode: 'aborted', errorMessage: shutdown.message } + }); + await closePromise; + } finally { + releaseHolder.resolve(); + await Promise.all(clients.map((client: DaemonRequestWireClient) => client.closeAsync())); + await host.closeAsync(); + } + }); + it('rejects a second active request on one connection without cancelling the first', async () => { const repoRoot: string = createRepoRoot(); const started: IDeferred = createDeferred(); diff --git a/libraries/rush-daemon/src/test/PhasedRequestInteractive.test.ts b/libraries/rush-daemon/src/test/PhasedRequestInteractive.test.ts index 7cc2ab8b51..6c915f8b5c 100644 --- a/libraries/rush-daemon/src/test/PhasedRequestInteractive.test.ts +++ b/libraries/rush-daemon/src/test/PhasedRequestInteractive.test.ts @@ -3,9 +3,11 @@ import type { IDaemonPhasedRequest, + IDaemonPhasedRequestResult, IDaemonSetRawModeMessage } from '@rushstack/rush-daemon-protocol'; +import { DaemonShutdownError } from '../DaemonShutdownError'; import { DaemonRequiresInProcessError } from '../DaemonTerminalPolicy'; import { InteractiveRequestInputRouter } from '../InteractiveRequestInputRouter'; import { PhasedRequestRouter } from '../PhasedRequestRouter'; @@ -68,6 +70,34 @@ it('restores phased-request raw mode before publishing the command result', asyn expect(lifecycleOrder).toEqual(['raw:true', 'raw:false', 'result']); }); +it('keeps the daemon shutdown reason when restoring raw mode fails with it', async () => { + const fixture: ITestRoutingFixture = createFixture(); + const client: TestPhasedRequestClient = new TestPhasedRequestClient(); + const shutdown: DaemonShutdownError = new DaemonShutdownError({ initiator: 'signal', signal: 'SIGTERM' }); + client.interactiveSession = new InteractiveRequestInputRouter().register({ + acceptsStdin: true, + client: { + abortSignal: client.abortSignal, + writeRawModeControlAsync: (message: IDaemonSetRawModeMessage): Promise => + message.payload.enabled ? Promise.resolve() : Promise.reject(shutdown) + }, + onFailure: (error: Error) => client.abortController.abort(error), + requestId: 'interactive-request' + }); + client.interactiveInputSink = { + writeInputAsync: (): Promise => Promise.resolve() + }; + await client.interactiveSession.setRawModeAsync(true); + client.abortController.abort(shutdown); + + await new PhasedRequestRouter(fixture.session) + .executeAsync(createRequest({ acceptsStdin: true, terminalRequirement: 'interactiveInput' }), client) + .catch(() => undefined); + + const result: IDaemonPhasedRequestResult | undefined = client.writes.find((write) => write.result)?.result; + expect(result).toMatchObject({ aborted: true, errorMessage: shutdown.message }); +}); + it('signals requiresInProcess without scheduling a PTY-only phased request', async () => { const fixture: ITestRoutingFixture = createFixture(); const client: TestPhasedRequestClient = new TestPhasedRequestClient(); From 2b7000b745ffd4c62a891e8a9fe7db7aee63c96a Mon Sep 17 00:00:00 2001 From: selarkin Date: Thu, 24 Sep 2026 11:23:13 -0700 Subject: [PATCH 6/9] [rush-daemon] Report the shutdown reason from global and graph request aborts Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../rush-daemon/src/DaemonGraphRequestRouter.ts | 6 +++++- libraries/rush-daemon/src/DaemonShutdownError.ts | 10 ++++++++++ .../rush-daemon/src/GlobalCommandRequestRouter.ts | 14 ++++++++++++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/libraries/rush-daemon/src/DaemonGraphRequestRouter.ts b/libraries/rush-daemon/src/DaemonGraphRequestRouter.ts index 567ac94026..b1659e33c8 100644 --- a/libraries/rush-daemon/src/DaemonGraphRequestRouter.ts +++ b/libraries/rush-daemon/src/DaemonGraphRequestRouter.ts @@ -37,6 +37,7 @@ import { import type { IWorkspaceSession } from './WorkspaceSession'; import { setPauseNextIteration } from './PhasedRequestRouter'; import { getWorkspaceGenerationToken } from './WorkspaceGeneration'; +import { type DaemonShutdownError, getDaemonShutdownReason } from './DaemonShutdownError'; const DAEMON_PACKAGE_VERSION: string = PackageJsonLookup.loadOwnPackageJson(__dirname).version; @@ -86,7 +87,10 @@ export class DaemonGraphRequestRouter { admissionErrorCode: getRequestAdmissionErrorCode(error) }; } - await client.writeResultAsync(result); + const shutdownReason: DaemonShutdownError | undefined = result.aborted + ? getDaemonShutdownReason(client.abortSignal) + : undefined; + await client.writeResultAsync(shutdownReason ? { ...result, errorMessage: shutdownReason.message } : result); } private async _mutateAsync( diff --git a/libraries/rush-daemon/src/DaemonShutdownError.ts b/libraries/rush-daemon/src/DaemonShutdownError.ts index a3aa060a0e..f50a30252f 100644 --- a/libraries/rush-daemon/src/DaemonShutdownError.ts +++ b/libraries/rush-daemon/src/DaemonShutdownError.ts @@ -61,3 +61,13 @@ function describeInitiator(options: IDaemonShutdownErrorOptions): string { export function getDaemonShutdownReason(signal: AbortSignal): DaemonShutdownError | undefined { return signal.aborted && signal.reason instanceof DaemonShutdownError ? signal.reason : undefined; } + +/** + * Returns the cleanup error unless it repeats a shutdown reason that is already the primary error, for example when + * restoring raw mode fails because shutdown closed the connection. + */ +export function withoutRepeatedShutdownReason(primary: unknown, cleanupError: unknown): unknown { + return primary instanceof DaemonShutdownError && cleanupError instanceof DaemonShutdownError + ? undefined + : cleanupError; +} diff --git a/libraries/rush-daemon/src/GlobalCommandRequestRouter.ts b/libraries/rush-daemon/src/GlobalCommandRequestRouter.ts index 256564c846..8ce2d4a61f 100644 --- a/libraries/rush-daemon/src/GlobalCommandRequestRouter.ts +++ b/libraries/rush-daemon/src/GlobalCommandRequestRouter.ts @@ -24,6 +24,7 @@ import { import { type IRequestLease, RequestSchedulerError, RequestSchedulerErrorCode } from './RequestScheduler'; import type { IWorkspaceSession } from './WorkspaceSession'; import { assertWorkspaceRequestResourcesHealthy } from './WorkspaceRequestResources'; +import { getDaemonShutdownReason, withoutRepeatedShutdownReason } from './DaemonShutdownError'; /** * Executes caller-resolved global command logic. @@ -179,7 +180,12 @@ async function executeAdmittedAsync( cleanupError = combineExecutionAndCleanupErrors(cleanupError, error); } aborted ||= context.requestAborted; - const combinedError: unknown = combineExecutionAndCleanupErrors(executionError, cleanupError); + const primaryError: unknown = + executionError ?? (aborted ? getDaemonShutdownReason(client.abortSignal) : undefined); + const combinedError: unknown = combineExecutionAndCleanupErrors( + primaryError, + withoutRepeatedShutdownReason(primaryError, cleanupError) + ); let result: IDaemonCommandResult; try { result = createGlobalCommandResult({ @@ -259,8 +265,12 @@ async function finishAfterAdmissionErrorAsync( throw combineExecutionAndCleanupErrors(admissionError, cleanupError); } const aborted: boolean = admissionError.code === RequestSchedulerErrorCode.Aborted; + const shutdownReason: unknown = aborted ? getDaemonShutdownReason(client.abortSignal) : undefined; const error: unknown = aborted - ? cleanupError + ? combineExecutionAndCleanupErrors( + shutdownReason, + withoutRepeatedShutdownReason(shutdownReason, cleanupError) + ) : combineExecutionAndCleanupErrors(admissionError, cleanupError); const result: IDaemonCommandResult = { ...createGlobalCommandResult({ From 9e87079f7e2ab0f6a5d339296e8f0f21e18fc08c Mon Sep 17 00:00:00 2001 From: selarkin Date: Thu, 24 Sep 2026 11:25:48 -0700 Subject: [PATCH 7/9] Format with prettier and update the protocol API report Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../src/test/resultDiagnostics.test.ts | 4 +++- common/reviews/api/rush-daemon-protocol.api.md | 3 +++ .../src/test/LifecycleControl.test.ts | 11 +++++++---- libraries/rush-daemon/src/DaemonGraphRequestRouter.ts | 4 +++- libraries/rush-daemon/src/PhasedRequestRouter.ts | 5 ++++- .../src/ProductionDaemonRequestResolver.ts | 7 ++++--- .../rush-daemon/src/WorkspaceRequestLifecycle.ts | 5 ++++- 7 files changed, 28 insertions(+), 11 deletions(-) diff --git a/apps/rush-cli-client/src/test/resultDiagnostics.test.ts b/apps/rush-cli-client/src/test/resultDiagnostics.test.ts index a1dbec266f..46c0ad5814 100644 --- a/apps/rush-cli-client/src/test/resultDiagnostics.test.ts +++ b/apps/rush-cli-client/src/test/resultDiagnostics.test.ts @@ -23,7 +23,9 @@ describe(getResultDiagnostic.name, () => { expect( getResultDiagnostic({ exitCode: 1, admissionErrorCode: 'wait-timeout', errorMessage: 'x' }) ).toBeUndefined(); - expect(getResultDiagnostic({ exitCode: 1, admissionErrorCode: 'no-wait', errorMessage: 'x' })).toBeUndefined(); + expect( + getResultDiagnostic({ exitCode: 1, admissionErrorCode: 'no-wait', errorMessage: 'x' }) + ).toBeUndefined(); expect(getResultDiagnostic({ exitCode: 1, admissionErrorCode: 'aborted' })).toBeUndefined(); }); diff --git a/common/reviews/api/rush-daemon-protocol.api.md b/common/reviews/api/rush-daemon-protocol.api.md index 92764aac5d..0584bcb366 100644 --- a/common/reviews/api/rush-daemon-protocol.api.md +++ b/common/reviews/api/rush-daemon-protocol.api.md @@ -79,6 +79,9 @@ export const DAEMON_REQUEST_ADMISSION_PROTOCOL_MINOR: number; // @beta export const DAEMON_REQUEST_LIFECYCLE_PROTOCOL_MINOR: number; +// @beta +export const DAEMON_SHUTDOWN_ACTIVE_REQUESTS_PROTOCOL_MINOR: number; + // @beta export const DAEMON_WORKSPACE_RESTART_PROTOCOL_MINOR: number; diff --git a/libraries/rush-daemon-protocol/src/test/LifecycleControl.test.ts b/libraries/rush-daemon-protocol/src/test/LifecycleControl.test.ts index 87d5cbceef..b6487b245a 100644 --- a/libraries/rush-daemon-protocol/src/test/LifecycleControl.test.ts +++ b/libraries/rush-daemon-protocol/src/test/LifecycleControl.test.ts @@ -23,10 +23,13 @@ it.each(MESSAGES)('round-trips lifecycle message $kind', (message: DaemonControl expect(decodeDaemonControlMessage(encodeDaemonControlMessage(message))).toEqual(message); }); -it.each([NEGATIVE, FRACTION, '1'])('rejects an invalid shutdown active request count %s', (value: unknown) => { - const json: string = JSON.stringify({ kind: 'shutdownAck', payload: { activeRequests: value } }); - expect(() => decodeDaemonControlMessage(new TextEncoder().encode(json))).toThrow('activeRequests'); -}); +it.each([NEGATIVE, FRACTION, '1'])( + 'rejects an invalid shutdown active request count %s', + (value: unknown) => { + const json: string = JSON.stringify({ kind: 'shutdownAck', payload: { activeRequests: value } }); + expect(() => decodeDaemonControlMessage(new TextEncoder().encode(json))).toThrow('activeRequests'); + } +); it.each([ZERO, NEGATIVE, FRACTION, '42'])('rejects an invalid daemon PID %s', (pid: unknown) => { const json: string = JSON.stringify({ kind: 'pong', payload: { pid, uptimeMs: UPTIME_MS } }); diff --git a/libraries/rush-daemon/src/DaemonGraphRequestRouter.ts b/libraries/rush-daemon/src/DaemonGraphRequestRouter.ts index b1659e33c8..10b8f818f3 100644 --- a/libraries/rush-daemon/src/DaemonGraphRequestRouter.ts +++ b/libraries/rush-daemon/src/DaemonGraphRequestRouter.ts @@ -90,7 +90,9 @@ export class DaemonGraphRequestRouter { const shutdownReason: DaemonShutdownError | undefined = result.aborted ? getDaemonShutdownReason(client.abortSignal) : undefined; - await client.writeResultAsync(shutdownReason ? { ...result, errorMessage: shutdownReason.message } : result); + await client.writeResultAsync( + shutdownReason ? { ...result, errorMessage: shutdownReason.message } : result + ); } private async _mutateAsync( diff --git a/libraries/rush-daemon/src/PhasedRequestRouter.ts b/libraries/rush-daemon/src/PhasedRequestRouter.ts index 938183bb9b..8710bea3b4 100644 --- a/libraries/rush-daemon/src/PhasedRequestRouter.ts +++ b/libraries/rush-daemon/src/PhasedRequestRouter.ts @@ -564,7 +564,10 @@ class PhasedRequestBatchCoordinator { : []; const result: IDaemonPhasedRequestResult = createPhasedCommandResult({ aborted, - error: combineErrors(executionError ?? getDaemonShutdownReason(entry.client.abortSignal), cleanupErrors), + error: combineErrors( + executionError ?? getDaemonShutdownReason(entry.client.abortSignal), + cleanupErrors + ), graphStatus: getClientGraphStatus(aborted, operationOutcomes), operationOutcomes, requestId: entry.request.requestId, diff --git a/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts b/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts index 2388b3f844..b0cd07ce69 100644 --- a/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts +++ b/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts @@ -222,9 +222,10 @@ export class ProductionDaemonRequestResolver implements IDaemonRequestResolver { return { ...components, reconcileInvalidationsAsync: async () => { - const result: IWorkspaceInvalidationReconciliation = await terminal.reconcileWithRequestDiagnosticsAsync( - () => components.reconcileInvalidationsAsync!() - ); + const result: IWorkspaceInvalidationReconciliation = + await terminal.reconcileWithRequestDiagnosticsAsync(() => + components.reconcileInvalidationsAsync!() + ); if (!engine.isIncremental) engine.operationGraph.invalidateOperations(undefined, 'rebuild'); return result; } diff --git a/libraries/rush-daemon/src/WorkspaceRequestLifecycle.ts b/libraries/rush-daemon/src/WorkspaceRequestLifecycle.ts index d3ebbd01cf..553ab9dedb 100644 --- a/libraries/rush-daemon/src/WorkspaceRequestLifecycle.ts +++ b/libraries/rush-daemon/src/WorkspaceRequestLifecycle.ts @@ -260,7 +260,10 @@ export class WorkspaceRequestLifecycle implements IDaemonRequestLifecycle { if (error instanceof RequestSchedulerError && !state.began && !state.terminalAttempted) { await client.interactiveSession.finishAsync(); await client.writeResultAsync({ - ...preExecutionFailure(envelope.requestId, getDaemonShutdownReason(client.abortSignal) ?? error), + ...preExecutionFailure( + envelope.requestId, + getDaemonShutdownReason(client.abortSignal) ?? error + ), aborted: client.abortSignal.aborted, admissionErrorCode: getRequestAdmissionErrorCode(error) }); From f1f3e5bb372e71f7a153843f045566ad42573439 Mon Sep 17 00:00:00 2001 From: selarkin Date: Thu, 24 Sep 2026 11:26:30 -0700 Subject: [PATCH 8/9] Update the protocol change file for minor 11 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../fix-rushd-error-surfacing_2026-09-24-01-50.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/changes/@rushstack/rush-daemon-protocol/fix-rushd-error-surfacing_2026-09-24-01-50.json b/common/changes/@rushstack/rush-daemon-protocol/fix-rushd-error-surfacing_2026-09-24-01-50.json index 92fdcf2561..5aebbe2a85 100644 --- a/common/changes/@rushstack/rush-daemon-protocol/fix-rushd-error-surfacing_2026-09-24-01-50.json +++ b/common/changes/@rushstack/rush-daemon-protocol/fix-rushd-error-surfacing_2026-09-24-01-50.json @@ -2,7 +2,7 @@ "changes": [ { "packageName": "@rushstack/rush-daemon-protocol", - "comment": "Add an optional \"activeRequests\" count to the shutdownAck control message; older peers omit or ignore it.", + "comment": "Add an optional \"activeRequests\" count to the shutdownAck control message; advertised as protocol minor 11 (`DAEMON_SHUTDOWN_ACTIVE_REQUESTS_PROTOCOL_MINOR`); older peers omit or ignore it.", "type": "patch" } ], From ac46fb04c45b19e830829dcc43103382f1eb94ed Mon Sep 17 00:00:00 2001 From: selarkin Date: Thu, 24 Sep 2026 12:17:04 -0700 Subject: [PATCH 9/9] [rush-cli-client] Expect the cancelled request count from daemon stop --force Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- apps/rush-cli-client/src/test/launchClient.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/rush-cli-client/src/test/launchClient.test.ts b/apps/rush-cli-client/src/test/launchClient.test.ts index b3ec694fca..d454c99c4e 100644 --- a/apps/rush-cli-client/src/test/launchClient.test.ts +++ b/apps/rush-cli-client/src/test/launchClient.test.ts @@ -288,6 +288,7 @@ describe('standalone rushx fallback', () => { expect(JSON.parse(result.stdout)).toEqual({ state: 'shutdownAccepted', socketPath: paths.socketPath, + cancelledRequests: 0, removedPaths: [reservation] }); expect(fs.existsSync(reservation)).toBe(false);