diff --git a/apps/rush-cli-client/src/daemonCommands.ts b/apps/rush-cli-client/src/daemonCommands.ts index b6ad07750d..fd19088343 100644 --- a/apps/rush-cli-client/src/daemonCommands.ts +++ b/apps/rush-cli-client/src/daemonCommands.ts @@ -119,7 +119,17 @@ export async function executeDaemonCommandAsync(options: IDaemonCommandOptions): } try { if (command === 'stop') { - await client.shutdownAsync(); + const { activeRequests } = await client.shutdownAsync(); + if (activeRequests) { + await writeStreamAsync( + process.stderr, + Buffer.from( + `rush-client: the daemon was running ${activeRequests} request(s); they were cancelled.\n` + ) + ); + } + const cancelled: { cancelledRequests?: number } = + activeRequests === undefined ? {} : { cancelledRequests: activeRequests }; if (options.argv[1] === '--force') { // Wait for the acknowledged daemon to release its listener and record, then clear leftovers // such as an abandoned startup reservation in the same invocation. @@ -129,13 +139,15 @@ export async function executeDaemonCommandAsync(options: IDaemonCommandOptions): await writeStatusAsync({ state: 'shutdownAccepted', socketPath: connectionOptions.paths.socketPath, + ...cancelled, removedPaths }); return; } await writeStatusAsync({ state: 'shutdownAccepted', - socketPath: connectionOptions.paths.socketPath + socketPath: connectionOptions.paths.socketPath, + ...cancelled }); return; } diff --git a/apps/rush-cli-client/src/launchClient.ts b/apps/rush-cli-client/src/launchClient.ts index af3cadb9c1..102c9b732b 100644 --- a/apps/rush-cli-client/src/launchClient.ts +++ b/apps/rush-cli-client/src/launchClient.ts @@ -27,6 +27,7 @@ import { formatAdmissionFailure, getConfiguredAdmission } from './ClientAdmissio import { ClientOperationRenderer } from './ClientOperationRenderer'; import { getDaemonConnectionOptionsAsync } from './daemonConnectionOptions'; import { selectClientRoute, type IClientRoute } from './routing'; +import { getResultDiagnostic } from './resultDiagnostics'; import { writeStreamAsync } from './writeStreamAsync'; import { getBundledRushVersion, @@ -206,7 +207,10 @@ export async function launchClientAsync(rushx: boolean): Promise { } if (outcome.kind === 'result') { process.exitCode = outcome.result.exitCode; - if (outcome.result.admissionErrorCode) { + const diagnostic: string | undefined = getResultDiagnostic(outcome.result); + if (diagnostic) { + await writeStreamAsync(process.stderr, Buffer.from(diagnostic)); + } else if (outcome.result.admissionErrorCode) { await writeStreamAsync( process.stderr, Buffer.from(formatAdmissionFailure(outcome.result.admissionErrorCode, request.admission)) diff --git a/apps/rush-cli-client/src/resultDiagnostics.ts b/apps/rush-cli-client/src/resultDiagnostics.ts new file mode 100644 index 0000000000..e65be24364 --- /dev/null +++ b/apps/rush-cli-client/src/resultDiagnostics.ts @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type { IDaemonCommandResult } from '@rushstack/rush-daemon-protocol'; + +/** + * Returns the stderr line that explains a failed daemon result, if any. + * + * @remarks + * A non-zero result's error message (for example, a daemon shutdown that aborted the request) is the only + * place the daemon reports failures that are not attributed to an operation, so it must not be dropped. + * Returns `undefined` for `no-wait` and `wait-timeout` admission failures, which `formatAdmissionFailure` + * explains. + */ +export function getResultDiagnostic( + result: Pick +): string | undefined { + // A request aborted while waiting for admission carries the reason (such as a daemon shutdown) in its + // error message; other admission failures are explained by `formatAdmissionFailure`. + if (result.admissionErrorCode !== undefined && result.admissionErrorCode !== 'aborted') return undefined; + if (result.exitCode !== 0 && result.errorMessage) { + return `rush-client: ${result.errorMessage}\n`; + } + return undefined; +} diff --git a/apps/rush-cli-client/src/test/launchClient.test.ts b/apps/rush-cli-client/src/test/launchClient.test.ts index b3ec694fca..d454c99c4e 100644 --- a/apps/rush-cli-client/src/test/launchClient.test.ts +++ b/apps/rush-cli-client/src/test/launchClient.test.ts @@ -288,6 +288,7 @@ describe('standalone rushx fallback', () => { expect(JSON.parse(result.stdout)).toEqual({ state: 'shutdownAccepted', socketPath: paths.socketPath, + cancelledRequests: 0, removedPaths: [reservation] }); expect(fs.existsSync(reservation)).toBe(false); diff --git a/apps/rush-cli-client/src/test/resultDiagnostics.test.ts b/apps/rush-cli-client/src/test/resultDiagnostics.test.ts new file mode 100644 index 0000000000..46c0ad5814 --- /dev/null +++ b/apps/rush-cli-client/src/test/resultDiagnostics.test.ts @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { getResultDiagnostic } from '../resultDiagnostics'; + +describe(getResultDiagnostic.name, () => { + it('prints the error message of a failed result', () => { + expect( + getResultDiagnostic({ + exitCode: 1, + errorMessage: 'The Rush daemon was shut down (idle timeout) while this request was running.' + }) + ).toBe('rush-client: The Rush daemon was shut down (idle timeout) while this request was running.\n'); + }); + + it('prefers the reason of a request aborted while waiting for admission', () => { + expect( + getResultDiagnostic({ exitCode: 1, admissionErrorCode: 'aborted', errorMessage: 'daemon shut down' }) + ).toBe('rush-client: daemon shut down\n'); + }); + + it('leaves no-wait and wait-timeout admission failures to the admission formatter', () => { + expect( + getResultDiagnostic({ exitCode: 1, admissionErrorCode: 'wait-timeout', errorMessage: 'x' }) + ).toBeUndefined(); + expect( + getResultDiagnostic({ exitCode: 1, admissionErrorCode: 'no-wait', errorMessage: 'x' }) + ).toBeUndefined(); + expect(getResultDiagnostic({ exitCode: 1, admissionErrorCode: 'aborted' })).toBeUndefined(); + }); + + it('stays silent for successful results and failures without a message', () => { + expect(getResultDiagnostic({ exitCode: 0, errorMessage: 'ignored' })).toBeUndefined(); + expect(getResultDiagnostic({ exitCode: 1 })).toBeUndefined(); + }); +}); diff --git a/common/changes/@rushstack/rush-cli-client/fix-rushd-error-surfacing_2026-09-24-01-50.json b/common/changes/@rushstack/rush-cli-client/fix-rushd-error-surfacing_2026-09-24-01-50.json new file mode 100644 index 0000000000..925deeffab --- /dev/null +++ b/common/changes/@rushstack/rush-cli-client/fix-rushd-error-surfacing_2026-09-24-01-50.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-cli-client", + "comment": "Print the error message of a failed daemon result (for example, a daemon shutdown that cancelled the build), and report cancelled requests from \"rush-client daemon stop\".", + "type": "patch" + } + ], + "packageName": "@rushstack/rush-cli-client", + "email": "selarkin@microsoft.com" +} diff --git a/common/changes/@rushstack/rush-client-core/fix-rushd-error-surfacing_2026-09-24-01-50.json b/common/changes/@rushstack/rush-client-core/fix-rushd-error-surfacing_2026-09-24-01-50.json new file mode 100644 index 0000000000..fad92bb8f4 --- /dev/null +++ b/common/changes/@rushstack/rush-client-core/fix-rushd-error-surfacing_2026-09-24-01-50.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-client-core", + "comment": "Return the daemon shutdown acknowledgement, including its optional active request count, from DaemonClient.shutdownAsync().", + "type": "patch" + } + ], + "packageName": "@rushstack/rush-client-core", + "email": "selarkin@microsoft.com" +} diff --git a/common/changes/@rushstack/rush-daemon-protocol/fix-rushd-error-surfacing_2026-09-24-01-50.json b/common/changes/@rushstack/rush-daemon-protocol/fix-rushd-error-surfacing_2026-09-24-01-50.json new file mode 100644 index 0000000000..5aebbe2a85 --- /dev/null +++ b/common/changes/@rushstack/rush-daemon-protocol/fix-rushd-error-surfacing_2026-09-24-01-50.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-daemon-protocol", + "comment": "Add an optional \"activeRequests\" count to the shutdownAck control message; advertised as protocol minor 11 (`DAEMON_SHUTDOWN_ACTIVE_REQUESTS_PROTOCOL_MINOR`); older peers omit or ignore it.", + "type": "patch" + } + ], + "packageName": "@rushstack/rush-daemon-protocol", + "email": "selarkin@microsoft.com" +} diff --git a/common/changes/@rushstack/rush-daemon/fix-rushd-error-surfacing_2026-09-24-01-50.json b/common/changes/@rushstack/rush-daemon/fix-rushd-error-surfacing_2026-09-24-01-50.json new file mode 100644 index 0000000000..6af207cd00 --- /dev/null +++ b/common/changes/@rushstack/rush-daemon/fix-rushd-error-surfacing_2026-09-24-01-50.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-daemon", + "comment": "Report warm input-snapshot failures (with the engine diagnostics) to the failing request instead of replaying them into the next request, abort requests interrupted by a daemon shutdown with a typed DaemonShutdownError that names its initiator, and report running requests in the shutdown acknowledgement.", + "type": "patch" + } + ], + "packageName": "@rushstack/rush-daemon", + "email": "selarkin@microsoft.com" +} diff --git a/common/reviews/api/rush-client-core.api.md b/common/reviews/api/rush-client-core.api.md index a731a4a1d2..f3ac64a463 100644 --- a/common/reviews/api/rush-client-core.api.md +++ b/common/reviews/api/rush-client-core.api.md @@ -13,6 +13,7 @@ import { IDaemonPongMessage } from '@rushstack/rush-daemon-protocol'; import { IDaemonProtocolVersion } from '@rushstack/rush-daemon-protocol'; import { IDaemonRequestEnvelope } from '@rushstack/rush-daemon-protocol'; import { IDaemonRequestRejectedMessage } from '@rushstack/rush-daemon-protocol'; +import { IDaemonShutdownAckMessage } from '@rushstack/rush-daemon-protocol'; import type { Readable } from 'node:stream'; // @beta @@ -29,7 +30,7 @@ export class DaemonClient { static connectAsync(options: IDaemonClientConnectOptions): Promise; executeAsync(options: IDaemonClientExecuteOptions): Promise; get protocolVersion(): IDaemonProtocolVersion; - shutdownAsync(timeoutMs?: number): Promise; + shutdownAsync(timeoutMs?: number): Promise; get status(): Promise; } diff --git a/common/reviews/api/rush-daemon-protocol.api.md b/common/reviews/api/rush-daemon-protocol.api.md index 8ca51ef110..0584bcb366 100644 --- a/common/reviews/api/rush-daemon-protocol.api.md +++ b/common/reviews/api/rush-daemon-protocol.api.md @@ -79,6 +79,9 @@ export const DAEMON_REQUEST_ADMISSION_PROTOCOL_MINOR: number; // @beta export const DAEMON_REQUEST_LIFECYCLE_PROTOCOL_MINOR: number; +// @beta +export const DAEMON_SHUTDOWN_ACTIVE_REQUESTS_PROTOCOL_MINOR: number; + // @beta export const DAEMON_WORKSPACE_RESTART_PROTOCOL_MINOR: number; @@ -592,7 +595,9 @@ export interface IDaemonShutdownAckMessage { // (undocumented) readonly kind: 'shutdownAck'; // (undocumented) - readonly payload: Record; + readonly payload: { + readonly activeRequests?: number; + }; } // @beta diff --git a/common/reviews/api/rush-daemon.api.md b/common/reviews/api/rush-daemon.api.md index 99cc2ca6a5..534d4c3973 100644 --- a/common/reviews/api/rush-daemon.api.md +++ b/common/reviews/api/rush-daemon.api.md @@ -66,6 +66,18 @@ export class DaemonRequiresInProcessError extends Error { readonly policy: IDaemonTerminalPolicyResult; } +// @beta +export class DaemonShutdownError extends Error { + constructor(options: IDaemonShutdownErrorOptions); + // (undocumented) + readonly initiator: DaemonShutdownInitiator; + // (undocumented) + readonly signal: string | undefined; +} + +// @beta +export type DaemonShutdownInitiator = 'controlClient' | 'signal' | 'idleTimeout' | 'restart' | 'host'; + // @beta export type DispatchWorkspaceRequestAsync = (options: IDispatchWorkspaceRequestOptions) => Promise; @@ -177,6 +189,13 @@ export interface IDaemonRequestResolver { readonly workspaceLifecycle?: IWorkspaceResolverLifecycle; } +// @beta +export interface IDaemonShutdownErrorOptions { + // (undocumented) + readonly initiator: DaemonShutdownInitiator; + readonly signal?: string; +} + // @beta export interface IDispatchWorkspaceRequestOptions { // (undocumented) @@ -723,7 +742,7 @@ export type ResolvedDaemonRequest = IResolvedDaemonPhasedRequest | IResolvedDaem // @beta export class RushDaemonHost { - closeAsync(): Promise; + closeAsync(reason?: DaemonShutdownError): Promise; readonly closed: Promise; getWorkspaceSessionAsync(): Promise; // (undocumented) diff --git a/libraries/rush-client-core/src/DaemonClient.ts b/libraries/rush-client-core/src/DaemonClient.ts index 012b215cf3..54193ba64f 100644 --- a/libraries/rush-client-core/src/DaemonClient.ts +++ b/libraries/rush-client-core/src/DaemonClient.ts @@ -25,7 +25,8 @@ import { type IDaemonPongMessage, type IDaemonProtocolVersion, type IDaemonRequestEnvelope, - type IDaemonRequestRejectedMessage + type IDaemonRequestRejectedMessage, + type IDaemonShutdownAckMessage } from '@rushstack/rush-daemon-protocol'; import { connectDaemonAsync, type DaemonFrameConnection } from '@rushstack/rush-daemon-transport'; @@ -102,6 +103,7 @@ export class DaemonClient { #result: IDeferred | undefined; #shutdown: IDeferred | undefined; #shutdownAcknowledged: boolean = false; + #shutdownAck: IDaemonShutdownAckMessage['payload'] = {}; #execution: IDaemonClientExecuteOptions | undefined; #finished: boolean = false; #inputStarted: boolean = false; @@ -186,8 +188,10 @@ export class DaemonClient { * Requests shutdown on a fresh connection and waits for acknowledgement followed by EOF. * @remarks This confirms acceptance and connection closure, not successful workspace cleanup. * Requires protocol 0.6. The timeout defaults to 15000 milliseconds. + * @returns The acknowledgement, including the number of running requests the shutdown aborts when the + * daemon reports it. */ - public async shutdownAsync(timeoutMs: number = 15000): Promise { + public async shutdownAsync(timeoutMs: number = 15000): Promise { if (this.#used) throw new Error('Create a fresh DaemonClient for shutdown.'); this.#used = true; let timer: ReturnType | undefined; @@ -206,6 +210,7 @@ export class DaemonClient { ); }, timeoutMs); await Promise.all([this.#shutdown.promise, this.#sendControlAsync({ kind: 'shutdown', payload: {} })]); + return this.#shutdownAck; } finally { clearTimeout(timer); await this.closeAsync(); @@ -375,6 +380,7 @@ export class DaemonClient { throw new DaemonProtocolError('malformedControlMessage', 'Unexpected shutdown acknowledgement.'); } this.#shutdownAcknowledged = true; + this.#shutdownAck = message.payload; return; } const execution: IDaemonClientExecuteOptions = this.#requireExecution(); diff --git a/libraries/rush-client-core/src/test/DaemonClient.test.ts b/libraries/rush-client-core/src/test/DaemonClient.test.ts index d5aaf43094..9a2ccc6af8 100644 --- a/libraries/rush-client-core/src/test/DaemonClient.test.ts +++ b/libraries/rush-client-core/src/test/DaemonClient.test.ts @@ -521,21 +521,22 @@ describe('DaemonClient', () => { }); onRequest = async (message) => { if (message.kind === 'shutdown') { - await sendAsync({ kind: 'shutdownAck', payload: {} }); + await sendAsync({ kind: 'shutdownAck', payload: { activeRequests: 2 } }); acknowledged(); } }; const client = await DaemonClient.connectAsync({ socketPath: address }); expect(client.protocolVersion.minor).toBeGreaterThanOrEqual(6); let completed: boolean = false; - const shutdown: Promise = client.shutdownAsync().then(() => { + const shutdown: Promise = client.shutdownAsync().then((payload) => { completed = true; + return payload; }); await ack; await new Promise((resolve) => setTimeout(resolve, 20)); expect(completed).toBe(false); await connection!.closeAsync(); - await shutdown; + await expect(shutdown).resolves.toEqual({ activeRequests: 2 }); expect(completed).toBe(true); expect(controls.filter((message) => message.kind === 'shutdown')).toHaveLength(1); }); diff --git a/libraries/rush-daemon-protocol/src/ControlMessageValidation.ts b/libraries/rush-daemon-protocol/src/ControlMessageValidation.ts index 8022db1d85..2be3cf95f9 100644 --- a/libraries/rush-daemon-protocol/src/ControlMessageValidation.ts +++ b/libraries/rush-daemon-protocol/src/ControlMessageValidation.ts @@ -13,6 +13,7 @@ import { validateRequestResultControl, validateRequestStartControl } from './RequestControlValidation'; +import { validateShutdownAck } from './ShutdownAckValidation'; import { validateSubscribeControl } from './SubscribeControlValidation'; function fail(reason: string): never { throw new DaemonProtocolError('malformedControlMessage', reason); @@ -67,7 +68,7 @@ const VALIDATORS_BY_KIND: Record = { requestRejected: validateRequestRejectedControl, requestResult: validateRequestResultControl, shutdown: noopValidator, - shutdownAck: noopValidator, + shutdownAck: validateShutdownAck, stdinReady: validateRequestCancelControl, stdinEnd: validateRequestCancelControl }; diff --git a/libraries/rush-daemon-protocol/src/DaemonLifecycleControl.ts b/libraries/rush-daemon-protocol/src/DaemonLifecycleControl.ts index 31921111a2..a7c084ed11 100644 --- a/libraries/rush-daemon-protocol/src/DaemonLifecycleControl.ts +++ b/libraries/rush-daemon-protocol/src/DaemonLifecycleControl.ts @@ -10,5 +10,11 @@ export interface IDaemonShutdownMessage { /** Acknowledges shutdown before the host closes connections and releases its endpoint. @beta */ export interface IDaemonShutdownAckMessage { readonly kind: 'shutdownAck'; - readonly payload: Record; + readonly payload: { + /** + * Requests that were still running and will be aborted by this shutdown. Daemons older than + * `DAEMON_SHUTDOWN_ACTIVE_REQUESTS_PROTOCOL_MINOR` omit it. + */ + readonly activeRequests?: number; + }; } diff --git a/libraries/rush-daemon-protocol/src/DaemonProtocolVersion.ts b/libraries/rush-daemon-protocol/src/DaemonProtocolVersion.ts index a687dae853..b35d1d6883 100644 --- a/libraries/rush-daemon-protocol/src/DaemonProtocolVersion.ts +++ b/libraries/rush-daemon-protocol/src/DaemonProtocolVersion.ts @@ -25,6 +25,9 @@ export const DAEMON_INVOCATION_KIND_PROTOCOL_MINOR: number = 8; /** The first minor supporting native mutations and guaranteed pre-execution restart outcomes. @beta */ export const DAEMON_WORKSPACE_RESTART_PROTOCOL_MINOR: number = 10; +/** The first additive protocol minor whose shutdown acknowledgement reports the active request count. @beta */ +export const DAEMON_SHUTDOWN_ACTIVE_REQUESTS_PROTOCOL_MINOR: number = 11; + /** * A rushd wire protocol version. * @@ -58,7 +61,7 @@ export interface IDaemonProtocolVersion { */ export const DAEMON_PROTOCOL_VERSION: IDaemonProtocolVersion = { major: 0, - minor: DAEMON_WORKSPACE_RESTART_PROTOCOL_MINOR + minor: DAEMON_SHUTDOWN_ACTIVE_REQUESTS_PROTOCOL_MINOR }; /** diff --git a/libraries/rush-daemon-protocol/src/ShutdownAckValidation.ts b/libraries/rush-daemon-protocol/src/ShutdownAckValidation.ts new file mode 100644 index 0000000000..5bd6c80cbc --- /dev/null +++ b/libraries/rush-daemon-protocol/src/ShutdownAckValidation.ts @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { DaemonProtocolError } from './DaemonProtocolError'; + +const ZERO: number = 0; + +/** Validates the optional active request count of a shutdown acknowledgement. @internal */ +export function validateShutdownAck(payload: Record): void { + const value: unknown = payload.activeRequests; + if (value === undefined || isNonnegativeInteger(value)) return; + throw new DaemonProtocolError('malformedControlMessage', 'Invalid shutdownAck field "activeRequests".'); +} + +function isNonnegativeInteger(value: unknown): boolean { + return typeof value === 'number' && Number.isSafeInteger(value) && value >= ZERO; +} diff --git a/libraries/rush-daemon-protocol/src/index.ts b/libraries/rush-daemon-protocol/src/index.ts index b170c7e899..1537163377 100644 --- a/libraries/rush-daemon-protocol/src/index.ts +++ b/libraries/rush-daemon-protocol/src/index.ts @@ -29,6 +29,7 @@ export { DAEMON_INVOCATION_KIND_PROTOCOL_MINOR } from './DaemonProtocolVersion'; export { DAEMON_LIFECYCLE_PROTOCOL_MINOR } from './DaemonProtocolVersion'; export { DAEMON_REQUEST_ADMISSION_PROTOCOL_MINOR } from './DaemonProtocolVersion'; export { DAEMON_REQUEST_LIFECYCLE_PROTOCOL_MINOR, DAEMON_PROTOCOL_VERSION } from './DaemonProtocolVersion'; +export { DAEMON_SHUTDOWN_ACTIVE_REQUESTS_PROTOCOL_MINOR } from './DaemonProtocolVersion'; export { DAEMON_WORKSPACE_RESTART_PROTOCOL_MINOR } from './DaemonProtocolVersion'; export { isDaemonProtocolCompatible } from './DaemonProtocolVersion'; export type { IDaemonProtocolVersion } from './DaemonProtocolVersion'; diff --git a/libraries/rush-daemon-protocol/src/test/LifecycleControl.test.ts b/libraries/rush-daemon-protocol/src/test/LifecycleControl.test.ts index 00c5363e2d..b6487b245a 100644 --- a/libraries/rush-daemon-protocol/src/test/LifecycleControl.test.ts +++ b/libraries/rush-daemon-protocol/src/test/LifecycleControl.test.ts @@ -14,6 +14,8 @@ const FRACTION: number = 1.5; const MESSAGES: readonly DaemonControlMessage[] = [ { kind: 'shutdown', payload: {} }, { kind: 'shutdownAck', payload: {} }, + { kind: 'shutdownAck', payload: { activeRequests: ZERO } }, + { kind: 'shutdownAck', payload: { activeRequests: PID } }, { kind: 'pong', payload: { pid: PID, residentMemoryBytes: MEMORY_BYTES, uptimeMs: UPTIME_MS } } ]; @@ -21,6 +23,14 @@ it.each(MESSAGES)('round-trips lifecycle message $kind', (message: DaemonControl expect(decodeDaemonControlMessage(encodeDaemonControlMessage(message))).toEqual(message); }); +it.each([NEGATIVE, FRACTION, '1'])( + 'rejects an invalid shutdown active request count %s', + (value: unknown) => { + const json: string = JSON.stringify({ kind: 'shutdownAck', payload: { activeRequests: value } }); + expect(() => decodeDaemonControlMessage(new TextEncoder().encode(json))).toThrow('activeRequests'); + } +); + it.each([ZERO, NEGATIVE, FRACTION, '42'])('rejects an invalid daemon PID %s', (pid: unknown) => { const json: string = JSON.stringify({ kind: 'pong', payload: { pid, uptimeMs: UPTIME_MS } }); expect(() => decodeDaemonControlMessage(new TextEncoder().encode(json))).toThrow('pid'); diff --git a/libraries/rush-daemon/src/DaemonControlSession.ts b/libraries/rush-daemon/src/DaemonControlSession.ts index 5189f6ffec..51f9bff38a 100644 --- a/libraries/rush-daemon/src/DaemonControlSession.ts +++ b/libraries/rush-daemon/src/DaemonControlSession.ts @@ -32,6 +32,7 @@ import type { IDaemonInteractiveConnection } from './DaemonInteractiveConnection import { MAX_REQUESTS_PER_CONNECTION } from './DaemonConnectionLimits'; import { DaemonRequestDispatchError } from './DaemonRequestDispatcher'; import type { DaemonRequestDispatcher } from './DaemonRequestDispatcher'; +import type { DaemonShutdownError } from './DaemonShutdownError'; import { DaemonWireRequestClient } from './DaemonWireRequestClient'; import { InteractiveInputRoutingError, @@ -49,6 +50,8 @@ export interface IDaemonControlSessionOptions { readonly onError: (error: Error) => void; readonly onRequestStarted?: () => () => void; readonly onShutdownRequested: () => void; + /** Counts requests running on every connection, reported in the shutdown acknowledgement. */ + readonly getActiveRequestCount?: () => number; readonly getWorkspaceStatus?: () => IDaemonWorkspaceStatus; } @@ -103,11 +106,15 @@ export class DaemonControlSession { options.onInteractiveConnection?.(this.#interactiveConnection); } - public closeAsync(drainRequests: boolean = false): Promise { - this.#closePromise ??= this.#closeOnceAsync(drainRequests); + public closeAsync(drainRequests: boolean = false, reason?: DaemonShutdownError): Promise { + this.#closePromise ??= this.#closeOnceAsync(drainRequests, reason); return this.#closePromise; } + public get activeRequestCount(): number { + return this.#requestById.size; + } + async #handleFrameSafelyAsync(frame: IDaemonFrame): Promise { try { await this.#onFrameAsync(frame); @@ -240,8 +247,15 @@ export class DaemonControlSession { 'Daemon shutdown requires a lifecycle-capable protocol version.' ); } - await this.#enqueueControlAsync({ kind: 'shutdownAck', payload: {} }); + const activeRequests: number | undefined = this.#options.getActiveRequestCount?.(); + // Queue the acknowledgement, then begin shutdown synchronously so the reported count is the set that + // shutdown aborts; closing drains the send queue, so the acknowledgement is still delivered first. + const ackPromise: Promise = this.#enqueueControlAsync({ + kind: 'shutdownAck', + payload: activeRequests === undefined ? {} : { activeRequests } + }); this.#options.onShutdownRequested(); + await ackPromise; } #startRequest(envelope: IDaemonRequestEnvelope): void { @@ -443,8 +457,8 @@ export class DaemonControlSession { } } - async #closeOnceAsync(drainRequests: boolean = false): Promise { - const closeReason: Error = new Error('The daemon control session is closing.'); + async #closeOnceAsync(drainRequests: boolean = false, reason?: DaemonShutdownError): Promise { + const closeReason: Error = reason ?? new Error('The daemon control session is closing.'); if (drainRequests) { const pending: Promise[]> = Promise.allSettled( Array.from(this.#requestById.values(), (state: IRequestState) => state.completion) diff --git a/libraries/rush-daemon/src/DaemonGraphRequestRouter.ts b/libraries/rush-daemon/src/DaemonGraphRequestRouter.ts index 567ac94026..10b8f818f3 100644 --- a/libraries/rush-daemon/src/DaemonGraphRequestRouter.ts +++ b/libraries/rush-daemon/src/DaemonGraphRequestRouter.ts @@ -37,6 +37,7 @@ import { import type { IWorkspaceSession } from './WorkspaceSession'; import { setPauseNextIteration } from './PhasedRequestRouter'; import { getWorkspaceGenerationToken } from './WorkspaceGeneration'; +import { type DaemonShutdownError, getDaemonShutdownReason } from './DaemonShutdownError'; const DAEMON_PACKAGE_VERSION: string = PackageJsonLookup.loadOwnPackageJson(__dirname).version; @@ -86,7 +87,12 @@ export class DaemonGraphRequestRouter { admissionErrorCode: getRequestAdmissionErrorCode(error) }; } - await client.writeResultAsync(result); + const shutdownReason: DaemonShutdownError | undefined = result.aborted + ? getDaemonShutdownReason(client.abortSignal) + : undefined; + await client.writeResultAsync( + shutdownReason ? { ...result, errorMessage: shutdownReason.message } : result + ); } private async _mutateAsync( diff --git a/libraries/rush-daemon/src/DaemonShutdownError.ts b/libraries/rush-daemon/src/DaemonShutdownError.ts new file mode 100644 index 0000000000..f50a30252f --- /dev/null +++ b/libraries/rush-daemon/src/DaemonShutdownError.ts @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +/** + * What initiated a daemon shutdown. + * + * @beta + */ +export type DaemonShutdownInitiator = 'controlClient' | 'signal' | 'idleTimeout' | 'restart' | 'host'; + +/** + * Options for {@link DaemonShutdownError}. + * + * @beta + */ +export interface IDaemonShutdownErrorOptions { + readonly initiator: DaemonShutdownInitiator; + /** The process signal name, when the initiator is `signal`. */ + readonly signal?: string; +} + +/** + * The typed reason used to abort requests that were still running when the daemon shut down. + * + * @remarks + * Its message is delivered to the affected clients as the request's error message. + * + * @beta + */ +export class DaemonShutdownError extends Error { + public readonly initiator: DaemonShutdownInitiator; + public readonly signal: string | undefined; + + public constructor(options: IDaemonShutdownErrorOptions) { + super( + `The Rush daemon was shut down (${describeInitiator(options)}) while this request was running; ` + + 're-run the command.' + ); + this.name = 'DaemonShutdownError'; + this.initiator = options.initiator; + this.signal = options.signal; + } +} + +function describeInitiator(options: IDaemonShutdownErrorOptions): string { + switch (options.initiator) { + case 'controlClient': + return 'requested by "rush-client daemon stop" or "daemon restart"'; + case 'signal': + return `the daemon process received ${options.signal ?? 'a termination signal'}`; + case 'idleTimeout': + return 'idle timeout'; + case 'restart': + return 'the daemon restarted to apply workspace changes'; + case 'host': + return 'the daemon host was closed'; + } +} + +/** Returns the shutdown reason if the signal was aborted because the daemon shut down. */ +export function getDaemonShutdownReason(signal: AbortSignal): DaemonShutdownError | undefined { + return signal.aborted && signal.reason instanceof DaemonShutdownError ? signal.reason : undefined; +} + +/** + * Returns the cleanup error unless it repeats a shutdown reason that is already the primary error, for example when + * restoring raw mode fails because shutdown closed the connection. + */ +export function withoutRepeatedShutdownReason(primary: unknown, cleanupError: unknown): unknown { + return primary instanceof DaemonShutdownError && cleanupError instanceof DaemonShutdownError + ? undefined + : cleanupError; +} diff --git a/libraries/rush-daemon/src/EngineTerminalProvider.ts b/libraries/rush-daemon/src/EngineTerminalProvider.ts index 13c8b48d5e..88345d0372 100644 --- a/libraries/rush-daemon/src/EngineTerminalProvider.ts +++ b/libraries/rush-daemon/src/EngineTerminalProvider.ts @@ -4,25 +4,69 @@ import type { IOperationGraph, _IOperationGraphEventSink } from '@microsoft/rush-lib'; import { TerminalProviderSeverity, type ITerminalProvider } from '@rushstack/terminal'; +import { WorkspaceEngineRecreationRequiredError } from './WorkspaceEngineComponentFactory'; + export class EngineTerminalProvider implements ITerminalProvider { public readonly supportsColor: boolean = false; public readonly eolCharacter: string = '\n'; readonly #messages: Array<{ text: string; severity: TerminalProviderSeverity }> = []; #graph: (IOperationGraph & { eventSink?: _IOperationGraphEventSink }) | undefined; #executing: boolean = false; + #hasReconciled: boolean = false; public write(text: string, severity: TerminalProviderSeverity): void { if (this.#executing) this.#emit(text, severity); else this.#messages.push({ text, severity }); } + /** + * Drains buffered diagnostics into the failure description, so that they belong to the failing request + * and are never replayed into a later request. + */ public describeError(error: unknown): string { return [ - ...this.#messages.map(({ text }) => text), + ...this.#messages.splice(0).map(({ text }) => text), error instanceof Error ? error.message : String(error) ].join('\n'); } + public get hasBufferedMessages(): boolean { + return this.#messages.length > 0; + } + + /** Discards diagnostics buffered by an earlier request before a new request starts using this terminal. */ + public discardBufferedMessages(): void { + this.#messages.length = 0; + } + + /** + * Runs a warm reconcile with request-scoped diagnostics. The graph keeps the binding request's terminal, so + * diagnostics buffered before a later request's reconcile belong to an earlier request and are discarded; the + * binding request's own diagnostics are kept. A failure carries the diagnostics buffered while reconciling. + */ + public async reconcileWithRequestDiagnosticsAsync(reconcileAsync: () => Promise): Promise { + if (this.#hasReconciled) this.discardBufferedMessages(); + this.#hasReconciled = true; + try { + return await reconcileAsync(); + } catch (error) { + throw this.#attachBufferedDiagnostics(error); + } + } + + #attachBufferedDiagnostics(error: unknown): unknown { + if (error instanceof WorkspaceEngineRecreationRequiredError) { + // The replacement engine gets a fresh terminal; the stale diagnostics must not reach a later request. + this.discardBufferedMessages(); + return error; + } + if (!this.hasBufferedMessages) return error; + if (!(error instanceof Error)) return new Error(this.describeError(error), { cause: error }); + // Keep the error's identity and type, which callers use for classification. + error.message = this.describeError(error); + return error; + } + public attach(graph: IOperationGraph): void { if (!('eventSink' in graph)) throw new Error('The native graph does not expose its operation event sink.'); diff --git a/libraries/rush-daemon/src/GlobalCommandRequestRouter.ts b/libraries/rush-daemon/src/GlobalCommandRequestRouter.ts index 256564c846..8ce2d4a61f 100644 --- a/libraries/rush-daemon/src/GlobalCommandRequestRouter.ts +++ b/libraries/rush-daemon/src/GlobalCommandRequestRouter.ts @@ -24,6 +24,7 @@ import { import { type IRequestLease, RequestSchedulerError, RequestSchedulerErrorCode } from './RequestScheduler'; import type { IWorkspaceSession } from './WorkspaceSession'; import { assertWorkspaceRequestResourcesHealthy } from './WorkspaceRequestResources'; +import { getDaemonShutdownReason, withoutRepeatedShutdownReason } from './DaemonShutdownError'; /** * Executes caller-resolved global command logic. @@ -179,7 +180,12 @@ async function executeAdmittedAsync( cleanupError = combineExecutionAndCleanupErrors(cleanupError, error); } aborted ||= context.requestAborted; - const combinedError: unknown = combineExecutionAndCleanupErrors(executionError, cleanupError); + const primaryError: unknown = + executionError ?? (aborted ? getDaemonShutdownReason(client.abortSignal) : undefined); + const combinedError: unknown = combineExecutionAndCleanupErrors( + primaryError, + withoutRepeatedShutdownReason(primaryError, cleanupError) + ); let result: IDaemonCommandResult; try { result = createGlobalCommandResult({ @@ -259,8 +265,12 @@ async function finishAfterAdmissionErrorAsync( throw combineExecutionAndCleanupErrors(admissionError, cleanupError); } const aborted: boolean = admissionError.code === RequestSchedulerErrorCode.Aborted; + const shutdownReason: unknown = aborted ? getDaemonShutdownReason(client.abortSignal) : undefined; const error: unknown = aborted - ? cleanupError + ? combineExecutionAndCleanupErrors( + shutdownReason, + withoutRepeatedShutdownReason(shutdownReason, cleanupError) + ) : combineExecutionAndCleanupErrors(admissionError, cleanupError); const result: IDaemonCommandResult = { ...createGlobalCommandResult({ diff --git a/libraries/rush-daemon/src/PhasedRequestRouter.ts b/libraries/rush-daemon/src/PhasedRequestRouter.ts index e6a8416e45..0d28934251 100644 --- a/libraries/rush-daemon/src/PhasedRequestRouter.ts +++ b/libraries/rush-daemon/src/PhasedRequestRouter.ts @@ -21,6 +21,7 @@ import { PhasedRequestEventSink } from './PhasedRequestEventSink'; import { PhasedRequestEventMultiplexer } from './PhasedRequestEventMultiplexer'; import type { IPhasedRequestClient } from './PhasedRequestClient'; import { DaemonRequiresInProcessError, evaluateDaemonTerminalPolicy } from './DaemonTerminalPolicy'; +import { DaemonShutdownError, getDaemonShutdownReason } from './DaemonShutdownError'; import type { IInteractiveRequestSession } from './InteractiveRequestInputRouter'; import { classifyRushCommand } from './RushCommandRequestPolicy'; import { @@ -628,7 +629,10 @@ class PhasedRequestBatchCoordinator { : []; const result: IDaemonPhasedRequestResult = createPhasedCommandResult({ aborted, - error: combineErrors(executionError, cleanupErrors), + error: combineErrors( + executionError ?? getDaemonShutdownReason(entry.client.abortSignal), + cleanupErrors + ), graphStatus: getClientGraphStatus(aborted, operationOutcomes), operationOutcomes, requestId: entry.request.requestId, @@ -1005,7 +1009,7 @@ async function writeAbortedResultAsync( const result: IDaemonPhasedRequestResult = { ...createPhasedCommandResult({ aborted: true, - error: combineErrors(undefined, cleanupErrors), + error: combineErrors(getDaemonShutdownReason(client.abortSignal), cleanupErrors), graphStatus: OperationStatus.Aborted, operationOutcomes: [], requestId, @@ -1107,7 +1111,13 @@ async function finishAfterAdmissionErrorAsync( return result; } -function combineErrors(executionError: unknown, cleanupErrors: unknown[]): unknown { +function combineErrors(executionError: unknown, allCleanupErrors: unknown[]): unknown { + // Cleanup that fails with the same daemon shutdown reason (for example, restoring raw mode after the + // interactive connection closed) must not hide that reason from the client. + const cleanupErrors: unknown[] = + executionError instanceof DaemonShutdownError + ? allCleanupErrors.filter((error: unknown) => !(error instanceof DaemonShutdownError)) + : allCleanupErrors; if (executionError !== undefined && cleanupErrors.length > 0) { return new AggregateError( [executionError, ...cleanupErrors], diff --git a/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts b/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts index 4bc77ac193..b0cd07ce69 100644 --- a/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts +++ b/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts @@ -32,6 +32,7 @@ import { } from './WorkspaceEngineComponentFactory'; import type { IWorkspaceSession, IWorkspaceSessionComponents } from './WorkspaceSession'; import { EngineTerminalProvider } from './EngineTerminalProvider'; +import { getDaemonShutdownReason } from './DaemonShutdownError'; import type { IWorkspaceResolverLifecycle } from './WorkspaceResolverLifecycle'; /** @@ -171,7 +172,8 @@ export class ProductionDaemonRequestResolver implements IDaemonRequestResolver { if (abortSignal.aborted) throw new DaemonRequestDispatchError( 'routingFailed', - 'The request was cancelled before engine initialization.' + getDaemonShutdownReason(abortSignal)?.message ?? + 'The request was cancelled before engine initialization.' ); return command; } @@ -221,7 +223,9 @@ export class ProductionDaemonRequestResolver implements IDaemonRequestResolver { ...components, reconcileInvalidationsAsync: async () => { const result: IWorkspaceInvalidationReconciliation = - await components.reconcileInvalidationsAsync!(); + await terminal.reconcileWithRequestDiagnosticsAsync(() => + components.reconcileInvalidationsAsync!() + ); if (!engine.isIncremental) engine.operationGraph.invalidateOperations(undefined, 'rebuild'); return result; } diff --git a/libraries/rush-daemon/src/RushDaemonHost.ts b/libraries/rush-daemon/src/RushDaemonHost.ts index 6aa4989ac3..17ff4a7bed 100644 --- a/libraries/rush-daemon/src/RushDaemonHost.ts +++ b/libraries/rush-daemon/src/RushDaemonHost.ts @@ -18,6 +18,7 @@ import { DaemonIdleTimer } from './DaemonIdleTimer'; import type { IDaemonInteractiveConnection } from './DaemonInteractiveConnection'; import { DaemonRequestDispatcher } from './DaemonRequestDispatcher'; import type { IDaemonRequestResolver } from './DaemonRequestDispatcher'; +import { DaemonShutdownError, type DaemonShutdownInitiator } from './DaemonShutdownError'; import { WorkspaceSession } from './WorkspaceSession'; import type { IWorkspaceSession, WorkspaceSessionFactory } from './WorkspaceSession'; import { WorkspaceSessionProvider } from './WorkspaceSessionProvider'; @@ -180,7 +181,12 @@ export class RushDaemonHost { }, onError: (error: Error) => options.onError?.(error), onRequestStarted: () => idleTimer.acquire(), - onShutdownRequested: requestShutdown + onShutdownRequested: () => requestShutdown('controlClient'), + getActiveRequestCount: () => { + let count: number = 0; + for (const activeSession of sessions) count += activeSession.activeRequestCount; + return count; + } }); sessions.add(session); if (lifecycle.closing) { @@ -223,13 +229,13 @@ export class RushDaemonHost { function requestRestart(plan: IWorkspaceProcessRestartPlan): void { host.#requestRestart(plan); } - function requestShutdown(): void { - void host.closeAsync().catch((error: Error) => { + function requestShutdown(initiator: DaemonShutdownInitiator): void { + void host.closeAsync(new DaemonShutdownError({ initiator })).catch((error: Error) => { if (options.onError) options.onError(error); else process.emitWarning(error); }); } - idleTimer.start(requestShutdown); + idleTimer.start(() => requestShutdown('idleTimeout')); return host; } @@ -248,9 +254,13 @@ export class RushDaemonHost { return this.#readWorkspaceStatus(); } - /** Closes active connections, stops listening, and removes transport artifacts. */ - public closeAsync(): Promise { - this.#closePromise ??= this.#closeOnceAsync().finally(() => { + /** + * Closes active connections, stops listening, and removes transport artifacts. + * + * @param reason - Delivered to requests that are still running; only the first close call's reason is used. + */ + public closeAsync(reason?: DaemonShutdownError): Promise { + this.#closePromise ??= this.#closeOnceAsync(reason).finally(() => { this.#notifyClosed?.(); if (!this.#restartPromise) this.#resolveRestart?.(undefined); }); @@ -272,7 +282,7 @@ export class RushDaemonHost { } async #restartOnceAsync(plan: IWorkspaceProcessRestartPlan): Promise { - await this.closeAsync(); + await this.closeAsync(new DaemonShutdownError({ initiator: 'restart' })); if (plan.failure) throw plan.failure; if (!plan.launch) throw new Error('A successor was not selected.'); const paths: IDaemonPaths = resolveDaemonPathsFromProcess( @@ -297,7 +307,7 @@ export class RushDaemonHost { } } - async #closeOnceAsync(): Promise { + async #closeOnceAsync(reason: DaemonShutdownError | undefined): Promise { this.#idleTimer[Symbol.dispose](); this.#lifecycle.closing = true; const errors: unknown[] = []; @@ -305,7 +315,7 @@ export class RushDaemonHost { // A failed standalone host must not exit naturally and become reclaimable over unjoined children. const sessionSettlements: PromiseSettledResult[] = await Promise.allSettled( Array.from(this.#sessions, (session: DaemonControlSession) => - session.closeAsync(!!this.#restartPromise) + session.closeAsync(!!this.#restartPromise, reason ?? new DaemonShutdownError({ initiator: 'host' })) ) ); for (const settlement of sessionSettlements) { diff --git a/libraries/rush-daemon/src/WorkspaceRequestLifecycle.ts b/libraries/rush-daemon/src/WorkspaceRequestLifecycle.ts index da5ca8230a..553ab9dedb 100644 --- a/libraries/rush-daemon/src/WorkspaceRequestLifecycle.ts +++ b/libraries/rush-daemon/src/WorkspaceRequestLifecycle.ts @@ -40,6 +40,7 @@ import { getWorkspaceRequestScheduler } from './WorkspaceRequestAdmission'; import { WorkspaceEngineRecreationRequiredError } from './WorkspaceEngineComponentFactory'; +import { getDaemonShutdownReason } from './DaemonShutdownError'; import type { IWorkspaceSession } from './WorkspaceSession'; import type { WorkspaceSessionProvider } from './WorkspaceSessionProvider'; import { assertWorkspaceRequestResourcesHealthy } from './WorkspaceRequestResources'; @@ -259,7 +260,10 @@ export class WorkspaceRequestLifecycle implements IDaemonRequestLifecycle { if (error instanceof RequestSchedulerError && !state.began && !state.terminalAttempted) { await client.interactiveSession.finishAsync(); await client.writeResultAsync({ - ...preExecutionFailure(envelope.requestId, error), + ...preExecutionFailure( + envelope.requestId, + getDaemonShutdownReason(client.abortSignal) ?? error + ), aborted: client.abortSignal.aborted, admissionErrorCode: getRequestAdmissionErrorCode(error) }); diff --git a/libraries/rush-daemon/src/index.ts b/libraries/rush-daemon/src/index.ts index 89039a0c43..b895226872 100644 --- a/libraries/rush-daemon/src/index.ts +++ b/libraries/rush-daemon/src/index.ts @@ -57,6 +57,11 @@ export { type IGlobalCommandRequestResult } from './GlobalCommandRequestRouter'; export { RushDaemonHost, type IRushDaemonHostOptions } from './RushDaemonHost'; +export { + DaemonShutdownError, + type DaemonShutdownInitiator, + type IDaemonShutdownErrorOptions +} from './DaemonShutdownError'; export { serveRushDaemonAsync, type IRushDaemonServeOptions } from './serveRushDaemon'; export { WorkspaceEngineComponentFactory, diff --git a/libraries/rush-daemon/src/serveRushDaemon.ts b/libraries/rush-daemon/src/serveRushDaemon.ts index 929b28bbf0..1baec925df 100644 --- a/libraries/rush-daemon/src/serveRushDaemon.ts +++ b/libraries/rush-daemon/src/serveRushDaemon.ts @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import { DaemonShutdownError } from './DaemonShutdownError'; import { RushDaemonHost } from './RushDaemonHost'; import type { IRushDaemonHostOptions } from './RushDaemonHost'; import { getInstalledWorkspaceSuccessorLaunchAsync } from './WorkspaceProcessRestart'; @@ -41,7 +42,7 @@ export async function serveRushDaemonAsync(options: IRushDaemonServeOptions): Pr }); await options.onReady?.(host); await waitForShutdownAsync(host, signalRegistration.signal); - await host.closeAsync(); + await host.closeAsync(getShutdownReason(signalRegistration.signal)); await host.restartCompleted; } finally { signalRegistration.dispose(); @@ -49,6 +50,13 @@ export async function serveRushDaemonAsync(options: IRushDaemonServeOptions): Pr } } +function getShutdownReason(signal: AbortSignal): DaemonShutdownError | undefined { + if (!signal.aborted) return undefined; + return signal.reason instanceof DaemonShutdownError + ? signal.reason + : new DaemonShutdownError({ initiator: 'host' }); +} + interface IShutdownSignalRegistration { readonly signal: AbortSignal; readonly dispose: () => void; @@ -56,7 +64,8 @@ interface IShutdownSignalRegistration { function createProcessShutdownSignal(): IShutdownSignalRegistration { const controller: AbortController = new AbortController(); - const onSignal: () => void = () => controller.abort(); + const onSignal: (signal: NodeJS.Signals) => void = (signal: NodeJS.Signals) => + controller.abort(new DaemonShutdownError({ initiator: 'signal', signal })); process.once('SIGINT', onSignal); process.once('SIGTERM', onSignal); return { diff --git a/libraries/rush-daemon/src/test/DaemonRequestWireGlobal.test.ts b/libraries/rush-daemon/src/test/DaemonRequestWireGlobal.test.ts index 86d984c61b..e570988009 100644 --- a/libraries/rush-daemon/src/test/DaemonRequestWireGlobal.test.ts +++ b/libraries/rush-daemon/src/test/DaemonRequestWireGlobal.test.ts @@ -10,6 +10,7 @@ import type { DaemonControlMessage, IDaemonRequestEnvelope } from '@rushstack/ru import type { GlobalCommandExecutor, IDaemonRequestResolver } from '../index'; import { MAX_REQUESTS_PER_CONNECTION } from '../DaemonConnectionLimits'; +import { DaemonShutdownError } from '../DaemonShutdownError'; import { RushDaemonHost } from '../RushDaemonHost'; import type { IRushDaemonHostOptions } from '../RushDaemonHost'; import { TestWorkspaceSession } from './TestWorkspaceSession'; @@ -303,6 +304,51 @@ describe('daemon global request wire integration', () => { } }); + it('tells a request queued for admission that the daemon shut down', async () => { + const repoRoot: string = createRepoRoot(); + const holderStarted: IDeferred = createDeferred(); + const releaseHolder: IDeferred = createDeferred(); + const resolver: IDaemonRequestResolver = new CallbackDaemonRequestResolver(async ({ envelope }) => { + const executorAsync: GlobalCommandExecutor = async () => { + if (envelope.requestId === 'holder') { + holderStarted.resolve(); + await releaseHolder.promise; + } + return { exitCode: 0 }; + }; + return { executor: executorAsync, kind: 'global' }; + }); + const host: RushDaemonHost = await RushDaemonHost.startAsync(createHostOptions(repoRoot, resolver)); + const clients: DaemonRequestWireClient[] = await Promise.all([connectAsync(host), connectAsync(host)]); + const shutdown: DaemonShutdownError = new DaemonShutdownError({ initiator: 'controlClient' }); + try { + await clients[0].sendControlAsync({ + kind: 'requestStart', + payload: createWireEnvelope('holder', 'custom', repoRoot) + }); + await holderStarted.promise; + await clients[1].sendControlAsync({ + kind: 'requestStart', + payload: createWireEnvelope('queued', 'custom', repoRoot) + }); + expect(await clients[1].readControlAsync()).toMatchObject({ + kind: 'queuePosition', + payload: { requestId: 'queued' } + }); + const closePromise: Promise = host.closeAsync(shutdown); + releaseHolder.resolve(); + expect((await clients[1].readTerminalAsync('queued')).terminal).toMatchObject({ + kind: 'requestResult', + payload: { aborted: true, admissionErrorCode: 'aborted', errorMessage: shutdown.message } + }); + await closePromise; + } finally { + releaseHolder.resolve(); + await Promise.all(clients.map((client: DaemonRequestWireClient) => client.closeAsync())); + await host.closeAsync(); + } + }); + it('rejects a second active request on one connection without cancelling the first', async () => { const repoRoot: string = createRepoRoot(); const started: IDeferred = createDeferred(); diff --git a/libraries/rush-daemon/src/test/DaemonShutdown.test.ts b/libraries/rush-daemon/src/test/DaemonShutdown.test.ts index 284d26a697..c2c8c91e3d 100644 --- a/libraries/rush-daemon/src/test/DaemonShutdown.test.ts +++ b/libraries/rush-daemon/src/test/DaemonShutdown.test.ts @@ -38,7 +38,7 @@ describe('daemon management shutdown', () => { await client.sendControlAsync(createDaemonHello(DAEMON_PROTOCOL_VERSION)); expect((await client.readControlAsync()).kind).toBe('helloAck'); await client.sendControlAsync({ kind: 'shutdown', payload: {} }); - expect(await client.readControlAsync()).toEqual({ kind: 'shutdownAck', payload: {} }); + expect(await client.readControlAsync()).toEqual({ kind: 'shutdownAck', payload: { activeRequests: 0 } }); await client.closed; await host.closed; expect(readDaemonLockfile(host.paths.lockfilePath)).toBeUndefined(); diff --git a/libraries/rush-daemon/src/test/EngineTerminalProvider.test.ts b/libraries/rush-daemon/src/test/EngineTerminalProvider.test.ts new file mode 100644 index 0000000000..2c6fcbfdf7 --- /dev/null +++ b/libraries/rush-daemon/src/test/EngineTerminalProvider.test.ts @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { TerminalProviderSeverity } from '@rushstack/terminal'; + +import { EngineTerminalProvider } from '../EngineTerminalProvider'; +import { WorkspaceEngineRecreationRequiredError } from '../WorkspaceEngineComponentFactory'; + +describe(EngineTerminalProvider.name, () => { + it('drains buffered diagnostics into the failure description so a later request cannot replay them', () => { + const terminal: EngineTerminalProvider = new EngineTerminalProvider(); + terminal.write('Permission denied', TerminalProviderSeverity.error); + expect(terminal.hasBufferedMessages).toBe(true); + expect(terminal.describeError(new Error('snapshot failed'))).toBe('Permission denied\nsnapshot failed'); + expect(terminal.hasBufferedMessages).toBe(false); + expect(terminal.describeError(new Error('next request'))).toBe('next request'); + }); + + it('discards diagnostics buffered by an earlier request', () => { + const terminal: EngineTerminalProvider = new EngineTerminalProvider(); + terminal.write('stale', TerminalProviderSeverity.warning); + terminal.discardBufferedMessages(); + expect(terminal.hasBufferedMessages).toBe(false); + expect(terminal.describeError('failure')).toBe('failure'); + }); + + it('scopes reconcile diagnostics to the request whose reconcile produced them', async () => { + const terminal: EngineTerminalProvider = new EngineTerminalProvider(); + terminal.write('binding request diagnostic', TerminalProviderSeverity.warning); + const failure: RangeError = new RangeError('could not capture'); + await expect( + terminal.reconcileWithRequestDiagnosticsAsync(async () => { + terminal.write('Permission denied', TerminalProviderSeverity.error); + throw failure; + }) + ).rejects.toBe(failure); + expect(failure.message).toBe('binding request diagnostic\nPermission denied\ncould not capture'); + + terminal.write('stale', TerminalProviderSeverity.warning); + await expect(terminal.reconcileWithRequestDiagnosticsAsync(async () => 'ok')).resolves.toBe('ok'); + expect(terminal.hasBufferedMessages).toBe(false); + }); + + it('drops diagnostics when the engine must be recreated', async () => { + const terminal: EngineTerminalProvider = new EngineTerminalProvider(); + const recreate: WorkspaceEngineRecreationRequiredError = new WorkspaceEngineRecreationRequiredError(); + const message: string = recreate.message; + await expect( + terminal.reconcileWithRequestDiagnosticsAsync(async () => { + terminal.write('stale', TerminalProviderSeverity.error); + throw recreate; + }) + ).rejects.toBe(recreate); + expect(recreate.message).toBe(message); + expect(terminal.hasBufferedMessages).toBe(false); + }); +}); diff --git a/libraries/rush-daemon/src/test/PhasedRequestInteractive.test.ts b/libraries/rush-daemon/src/test/PhasedRequestInteractive.test.ts index 7cc2ab8b51..6c915f8b5c 100644 --- a/libraries/rush-daemon/src/test/PhasedRequestInteractive.test.ts +++ b/libraries/rush-daemon/src/test/PhasedRequestInteractive.test.ts @@ -3,9 +3,11 @@ import type { IDaemonPhasedRequest, + IDaemonPhasedRequestResult, IDaemonSetRawModeMessage } from '@rushstack/rush-daemon-protocol'; +import { DaemonShutdownError } from '../DaemonShutdownError'; import { DaemonRequiresInProcessError } from '../DaemonTerminalPolicy'; import { InteractiveRequestInputRouter } from '../InteractiveRequestInputRouter'; import { PhasedRequestRouter } from '../PhasedRequestRouter'; @@ -68,6 +70,34 @@ it('restores phased-request raw mode before publishing the command result', asyn expect(lifecycleOrder).toEqual(['raw:true', 'raw:false', 'result']); }); +it('keeps the daemon shutdown reason when restoring raw mode fails with it', async () => { + const fixture: ITestRoutingFixture = createFixture(); + const client: TestPhasedRequestClient = new TestPhasedRequestClient(); + const shutdown: DaemonShutdownError = new DaemonShutdownError({ initiator: 'signal', signal: 'SIGTERM' }); + client.interactiveSession = new InteractiveRequestInputRouter().register({ + acceptsStdin: true, + client: { + abortSignal: client.abortSignal, + writeRawModeControlAsync: (message: IDaemonSetRawModeMessage): Promise => + message.payload.enabled ? Promise.resolve() : Promise.reject(shutdown) + }, + onFailure: (error: Error) => client.abortController.abort(error), + requestId: 'interactive-request' + }); + client.interactiveInputSink = { + writeInputAsync: (): Promise => Promise.resolve() + }; + await client.interactiveSession.setRawModeAsync(true); + client.abortController.abort(shutdown); + + await new PhasedRequestRouter(fixture.session) + .executeAsync(createRequest({ acceptsStdin: true, terminalRequirement: 'interactiveInput' }), client) + .catch(() => undefined); + + const result: IDaemonPhasedRequestResult | undefined = client.writes.find((write) => write.result)?.result; + expect(result).toMatchObject({ aborted: true, errorMessage: shutdown.message }); +}); + it('signals requiresInProcess without scheduling a PTY-only phased request', async () => { const fixture: ITestRoutingFixture = createFixture(); const client: TestPhasedRequestClient = new TestPhasedRequestClient(); diff --git a/libraries/rush-daemon/src/test/ProductionDaemonRequestResolver.test.ts b/libraries/rush-daemon/src/test/ProductionDaemonRequestResolver.test.ts index b0d02fd298..01f38bfd80 100644 --- a/libraries/rush-daemon/src/test/ProductionDaemonRequestResolver.test.ts +++ b/libraries/rush-daemon/src/test/ProductionDaemonRequestResolver.test.ts @@ -34,6 +34,7 @@ import { stopSuccessorAsync } from './WorkspaceLifecycleTestProcess'; import { removeTestFolderAsync } from './TestProcessExit'; import { readDaemonLockfile } from '@rushstack/rush-daemon-transport'; import { EngineTerminalProvider } from '../EngineTerminalProvider'; +import { DaemonShutdownError } from '../DaemonShutdownError'; import { getInstalledWorkspaceSuccessorLaunchAsync } from '../WorkspaceProcessRestart'; import type { GetWorkspaceSuccessorLaunchAsync, @@ -1436,4 +1437,67 @@ process.exit(23); await fixture[Symbol.asyncDispose](); } }); + + const canRevokeReadAccess: boolean = process.platform !== 'win32' && process.getuid?.() !== 0; + (canRevokeReadAccess ? it : it.skip)( + 'reports a warm snapshot failure to the failing request and never replays it into the next request', + async () => { + const fixture: IFixture = await createFixtureAsync(); + const inputPath: string = path.join(fixture.repoRoot, 'projects/a/input.txt'); + try { + await runAsync(fixture, 'initial', ['build', '--only', 'a']); + fs.writeFileSync(inputPath, 'unreadable'); + fs.chmodSync(inputPath, 0); + const failed: ITerminalExchange = await runAsync(fixture, 'unreadable', ['build', '--only', 'a']); + expect(failed.terminal).toMatchObject({ + kind: 'requestRejected', + payload: { + message: expect.stringMatching( + /Permission denied[\s\S]*Rush could not capture the next workspace inputs snapshot\./ + ) + } + }); + fs.chmodSync(inputPath, 0o644); + const recovered: ITerminalExchange = await runAsync(fixture, 'recovered', ['build', '--only', 'a']); + expect(recovered.terminal).toMatchObject({ kind: 'requestResult', payload: { exitCode: 0 } }); + const output: string = [ + logText(recovered), + ...recovered.frames + .filter((frame) => frame.kind === DaemonFrameType.event) + .map((frame) => JSON.stringify(decodeDaemonEventFrame(frame.payload))) + ].join('\n'); + expect(output).not.toContain('Permission denied'); + expect(output).not.toContain('state of the repo'); + } finally { + if (fs.existsSync(inputPath)) fs.chmodSync(inputPath, 0o644); + await fixture[Symbol.asyncDispose](); + } + } + ); + + it('aborts an in-flight build with the typed daemon shutdown reason', async () => { + const fixture: IFixture = await createFixtureAsync(); + const gate: INativeScriptGate = await createNativeScriptGateAsync(fixture.repoRoot, 'a'); + try { + const victim: Promise = runAsync(fixture, 'victim', ['build', '--only', 'a']); + await gate.entered; + const closing: Promise = fixture.host.closeAsync( + new DaemonShutdownError({ initiator: 'signal', signal: 'SIGTERM' }) + ); + await gate.releaseAsync(); + expect((await victim).terminal).toMatchObject({ + kind: 'requestResult', + payload: { + aborted: true, + errorMessage: expect.stringMatching( + /^The Rush daemon was shut down \(the daemon process received SIGTERM\) while this request was running; re-run the command\.$/ + ) + } + }); + await closing; + } finally { + await gate.releaseAsync(); + await fixture[Symbol.asyncDispose](); + } + }); });