From dcfc42699386758b1f4fbf05142e198cc2afb7a5 Mon Sep 17 00:00:00 2001 From: roblourens Date: Sat, 15 Aug 2026 16:11:33 -0700 Subject: [PATCH 1/3] agentHost: Don't split the final response at the reconnect boundary (#331045) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * agentHost: Don't split the final response at the reconnect boundary Reconnecting to an active turn ran two independent converters over the same response parts: the one-shot `activeTurnToProgress` snapshot and the always-on `_observeTurn` graph. They de-duplicate via `adoptInvocations`, which is keyed on live `ChatToolInvocation` instances — so a tool call that had already settled, and which the snapshot renders as a `toolInvocationSerialized` part, could not be adopted and was emitted a second time as a live invocation. That duplicate lands between the restored markdown prefix and the markdown still streaming into the same response part. The response model only merges a markdown update into an immediately preceding markdown part, so the final answer was split in two — in the observed case mid-word, with the prefix folded into the collapsed activity section and the remainder rendered as a separate response. Record what the snapshot emitted per tool call instead of only the adoptable subset, so per-tool setup can skip a settled tool call that is already fully rendered. Subagent tools are excluded because their setup is what streams the child session's inner tool calls into the response. (Written by Copilot) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * agentHost: Don't re-emit a settled subagent parent card on reconnect Code review caught that the subagent exception left the reconnect split in place for completed subagent calls: the guard declined to skip them so their child session would still be observed, but that fell through to `_setupServerToolCall`, which sinks a second live parent invocation. That part lands between the restored markdown prefix and its continuation — the very split the guard exists to prevent. Separate child-session observation from emitting the parent invocation. The invocation is still built so subagent observation has something to drive, but a tool call the snapshot already rendered as a serialized part is no longer emitted a second time. Also trims the inline commentary flagged in review. (Written by Copilot) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../agentHost/agentHostSessionHandler.ts | 80 ++++++++++++---- .../agentHostChatContribution.test.ts | 95 +++++++++++++++++++ 2 files changed, 155 insertions(+), 20 deletions(-) diff --git a/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostSessionHandler.ts b/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostSessionHandler.ts index f0a66d4aaa5273..0e16beea20f14f 100644 --- a/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostSessionHandler.ts +++ b/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostSessionHandler.ts @@ -75,7 +75,7 @@ import { type IImageVariableEntry } from '../../../common/attachments/chatVariableEntries.js'; import { coerceImageBuffer } from '../../../common/chatImageExtraction.js'; -import { ChatRequestQueueKind, ConfirmedReason, ElicitationState, IChatProgress, IChatQuestionAnswers, IChatService, IChatToolInvocation, IRemotePendingRequest, ToolConfirmKind, type IChatAutoModeResolutionPart, type IChatMcpAuthenticationRequired, type IChatMcpAuthenticationRequiredServer, type IChatMcpStartingServer, type IChatMultiSelectAnswer, type IChatPlanReviewResult, type IChatResponseErrorDetails, type IChatSingleSelectAnswer, type IChatTerminalToolInvocationData } from '../../../common/chatService/chatService.js'; +import { ChatRequestQueueKind, ConfirmedReason, ElicitationState, IChatProgress, IChatQuestionAnswers, IChatService, IChatToolInvocation, IRemotePendingRequest, ToolConfirmKind, type IChatAutoModeResolutionPart, type IChatMcpAuthenticationRequired, type IChatMcpAuthenticationRequiredServer, type IChatMcpStartingServer, type IChatMultiSelectAnswer, type IChatPlanReviewResult, type IChatResponseErrorDetails, type IChatSingleSelectAnswer, type IChatTerminalToolInvocationData, type IChatToolInvocationSerialized } from '../../../common/chatService/chatService.js'; import { IChatSession, IChatSessionContentProvider, IChatSessionHistoryItem, IChatSessionItem, IChatSessionRequestHistoryItem, isTerminalCommandPrompt, SessionType, type IChatInputCompletionItem, type IChatInputCompletionsParams, type IChatInputCompletionsResult, type IChatSessionServerRequest } from '../../../common/chatSessionsService.js'; import { IChatEntitlementService } from '../../../../../services/chat/common/chatEntitlementService.js'; import { IWorkingCopyService } from '../../../../../services/workingCopy/common/workingCopyService.js'; @@ -170,9 +170,10 @@ type AgentHostInvocationFailedClassification = { * - {@link sink} routes emitted progress to either the agent invoke * callback (live) or `chatSession.appendProgress` (reconnect / * server-initiated). - * - {@link adoptInvocations} carries `ChatToolInvocation` instances that - * `activeTurnToProgress` already produced so per-tool setup adopts them - * rather than recreating UI handles. + * - {@link snapshotToolCalls} carries whatever the snapshot already emitted + * for each tool call: a live `ChatToolInvocation` that per-tool setup adopts + * rather than recreating a UI handle, or a serialized part for a tool call + * that had already settled, which per-tool setup must not emit again. * - {@link seedEmittedLengths} prevents the always-on graph from re-emitting * markdown / reasoning prefixes already covered by the snapshot. * - {@link onTurnEnded} fires once when the turn reaches a terminal state. @@ -191,7 +192,13 @@ interface IObserveTurnOptions { readonly turnId: string; readonly sink: (parts: IChatProgress[]) => void; readonly cancellationToken: CancellationToken; - readonly adoptInvocations?: ReadonlyMap; + /** + * What `activeTurnToProgress` already emitted for each tool call in the + * reconnect snapshot, keyed by tool call id. A live `ChatToolInvocation` is + * adopted by per-tool setup; a serialized part means the tool call had + * already settled and is fully rendered, so per-tool setup emits nothing. + */ + readonly snapshotToolCalls?: ReadonlyMap; readonly seedEmittedLengths?: ReadonlyMap; readonly initialResponsePartCount?: number; readonly onTurnEnded?: (lastTurn: Turn | undefined) => void; @@ -581,6 +588,16 @@ function inputRequestResponsePartKey(part: InputRequestResponsePart): string { return `ir:${part.request.id}:${JSON.stringify({ ...part.request, answers: undefined })}`; } +/** + * The live invocation the reconnect snapshot emitted for this tool call, if + * any. A tool call the snapshot rendered as a serialized part has no live + * handle to adopt. + */ +function snapshotInvocationToAdopt(opts: IObserveTurnOptions, toolCallId: string): ChatToolInvocation | undefined { + const emitted = opts.snapshotToolCalls?.get(toolCallId); + return emitted instanceof ChatToolInvocation ? emitted : undefined; +} + // ============================================================================= // Chat session // ============================================================================= @@ -2906,7 +2923,7 @@ export class AgentHostSessionHandler extends Disposable implements IChatSessionC * - server-initiated turns detected by {@link _watchForServerInitiatedTurns}. * * Differences are captured in {@link IObserveTurnOptions.sink} (where - * progress is delivered) and {@link IObserveTurnOptions.adoptInvocations} / + * progress is delivered) and {@link IObserveTurnOptions.snapshotToolCalls} / * {@link IObserveTurnOptions.seedEmittedLengths} (snapshot continuity for * the reconnect case). * @@ -3474,18 +3491,28 @@ export class AgentHostSessionHandler extends Disposable implements IChatSessionC subagentContext: ISubagentContext, ): void { const initial = part$.get().toolCall; + // The snapshot renders a settled tool call as a serialized part, which + // cannot be adopted. A live invocation for it would duplicate the card + // and land a tool part between the restored markdown prefix and the + // markdown still streaming into the same response part, splitting the + // answer at the reconnect boundary. + const renderedBySnapshot = !!opts.snapshotToolCalls?.has(initial.toolCallId) + && !snapshotInvocationToAdopt(opts, initial.toolCallId); + if (renderedBySnapshot && !shouldObserveSubagentChat(initial)) { + return; + } const contributor = initial.contributor; if (contributor?.kind === ToolCallContributorKind.Client && contributor.clientId === this._config.connection.clientId) { // Set up before claiming: the claim is what tells the session-level // watcher it may execute this call, and it must find the shared // invocation already created when it does. - this._setupClientToolCall(initial, part$, store, opts, subagentContext); + this._setupClientToolCall(initial, part$, store, opts, subagentContext, renderedBySnapshot); store.add(this._markToolCallRendered(opts.chatURI, opts.turnId, initial.toolCallId, opts.sessionResource)); } else if (contributor?.kind === ToolCallContributorKind.Client) { this._setupOtherClientToolCall(initial, part$, store, opts); } else { store.add(this._markToolCallRendered(opts.chatURI, opts.turnId, initial.toolCallId, opts.sessionResource)); - this._setupServerToolCall(initial, part$, store, opts, subagentContext); + this._setupServerToolCall(initial, part$, store, opts, subagentContext, renderedBySnapshot); } } @@ -3561,7 +3588,7 @@ export class AgentHostSessionHandler extends Disposable implements IChatSessionC opts: IObserveTurnOptions, ): void { const toolCallId = initial.toolCallId; - const adopted = opts.adoptInvocations?.get(toolCallId); + const adopted = snapshotInvocationToAdopt(opts, toolCallId); const invocation = adopted ?? toolCallStateToInvocation( initial, opts.subAgentInvocationId, @@ -3624,6 +3651,11 @@ export class AgentHostSessionHandler extends Disposable implements IChatSessionC * {@link ChatToolInvocation} when present (reconnect parity); otherwise * emits a fresh one. Reacts to status transitions for re-confirmation, * terminal revival, finalization, and subagent observation. + * + * `renderedBySnapshot` marks a settled tool call the reconnect snapshot + * already rendered as a serialized part. The invocation is still built so + * subagent observation has something to drive, but it is not emitted — + * the snapshot's part is the one on screen. */ private _setupServerToolCall( initial: ToolCallState, @@ -3631,10 +3663,11 @@ export class AgentHostSessionHandler extends Disposable implements IChatSessionC store: DisposableStore, opts: IObserveTurnOptions, subagentContext: ISubagentContext, + renderedBySnapshot = false, ): void { const toolCallId = initial.toolCallId; const subAgentInvocationId = opts.subAgentInvocationId; - const adopted = opts.adoptInvocations?.get(toolCallId); + const adopted = snapshotInvocationToAdopt(opts, toolCallId); let confirmationOptions = initial.status === ToolCallStatus.PendingConfirmation ? initial.options : undefined; // Tools that stream their arguments (reliably: terminal/bash commands) // are first observed in `Streaming`. Represent them with a native @@ -3647,10 +3680,14 @@ export class AgentHostSessionHandler extends Disposable implements IChatSessionC invocation = adopted; } else if (initial.status === ToolCallStatus.Streaming) { invocation = toolCallStateToStreamingInvocation(initial, subAgentInvocationId, opts.backendSession, this._config.connectionAuthority, opts.sessionResource.authority); - opts.sink([invocation]); + if (!renderedBySnapshot) { + opts.sink([invocation]); + } } else { invocation = toolCallStateToInvocation(initial, subAgentInvocationId, opts.backendSession, this._config.connectionAuthority, opts.sessionResource.authority); - opts.sink([invocation]); + if (!renderedBySnapshot) { + opts.sink([invocation]); + } } // Hook up a tool first observed after it already entered confirmation. @@ -3864,6 +3901,7 @@ export class AgentHostSessionHandler extends Disposable implements IChatSessionC store: DisposableStore, opts: IObserveTurnOptions, subagentContext: ISubagentContext, + renderedBySnapshot = false, ): void { const toolCallId = initial.toolCallId; const toolName = initial.toolName; @@ -3871,7 +3909,7 @@ export class AgentHostSessionHandler extends Disposable implements IChatSessionC // Reconnect adoption: settle any snapshot invocation so the shared // invocation can take over the UI slot rather than leaving the old // instance orphaned. - const adopted = opts.adoptInvocations?.get(toolCallId); + const adopted = snapshotInvocationToAdopt(opts, toolCallId); if (adopted && !IChatToolInvocation.isComplete(adopted)) { adopted.didExecuteTool(undefined); } @@ -3920,7 +3958,9 @@ export class AgentHostSessionHandler extends Disposable implements IChatSessionC // The shared invocation is created with no `sessionResource`, so it // does not `appendProgress` into a chat model. Emit it explicitly so it // renders in this chat / subagent group (mirrors `_setupServerToolCall`). - opts.sink([invocation]); + if (!renderedBySnapshot) { + opts.sink([invocation]); + } let confirmationDispatched = false; @@ -4726,12 +4766,12 @@ export class AgentHostSessionHandler extends Disposable implements IChatSessionC const sessionKey = backendSession.toString(); const chatURI = this._getChatURI(chatSession.sessionResource); - // Extract live ChatToolInvocation objects from the initial progress - // array so per-tool setup adopts the same instances the chat UI holds. - const adoptInvocations = new Map(); + // Live invocations are adopted by per-tool setup; serialized parts mark + // a settled tool call it must not emit again. + const snapshotToolCalls = new Map(); for (const item of initialProgress) { - if (item instanceof ChatToolInvocation) { - adoptInvocations.set(item.toolCallId, item); + if (item instanceof ChatToolInvocation || item.kind === 'toolInvocationSerialized') { + snapshotToolCalls.set(item.toolCallId, item); } } @@ -4758,7 +4798,7 @@ export class AgentHostSessionHandler extends Disposable implements IChatSessionC turnId, sink: parts => chatSession.appendProgress(parts), cancellationToken: cts.token, - adoptInvocations, + snapshotToolCalls, seedEmittedLengths, initialResponsePartCount, onTurnEnded: () => { diff --git a/src/vs/workbench/contrib/chat/test/browser/agentSessions/agentHostChatContribution.test.ts b/src/vs/workbench/contrib/chat/test/browser/agentSessions/agentHostChatContribution.test.ts index 919d87d69e0e6e..093f7877c59147 100644 --- a/src/vs/workbench/contrib/chat/test/browser/agentSessions/agentHostChatContribution.test.ts +++ b/src/vs/workbench/contrib/chat/test/browser/agentSessions/agentHostChatContribution.test.ts @@ -9701,6 +9701,101 @@ suite('AgentHostChatContribution', () => { assert.strictEqual(toolInvocation!.toolCallId, 'tc-running'); }); + test('replays a settled tool call once, keeping the streamed markdown in one part', () => runWithFakedTimers({ useFakeTimers: true }, async () => { + // A tool part between the two markdown fragments stops the response + // model from merging them, splitting the answer mid-word. + const { sessionHandler, agentHostService } = createContribution(disposables); + + const sessionUri = AgentSession.uri('copilot', 'reconnect-settled-tool'); + const sessionState = makeSessionStateWithActiveTurn(sessionUri.toString(), { streamingText: 'that fallback fails to' }); + sessionState.activeTurn!.responseParts.unshift({ + kind: ResponsePartKind.ToolCall, + toolCall: { + toolCallId: 'tc-done', + toolName: 'bash', + displayName: 'Bash', + invocationMessage: 'Ran command', + pastTenseMessage: 'Ran command', + status: ToolCallStatus.Completed, + confirmed: ToolCallConfirmationReason.NotNeeded, + success: true, + content: [], + }, + }); + agentHostService.sessionStates.set(sessionUri.toString(), sessionState); + + const sessionResource = URI.from({ scheme: 'agent-host-copilot', path: '/reconnect-settled-tool' }); + const session = await sessionHandler.provideChatSessionContent(sessionResource, CancellationToken.None); + disposables.add(toDisposable(() => session.dispose())); + + agentHostService.fireAction({ + channel: sessionUri.toString(), + action: { type: 'chat/delta', turnId: 'turn-active', partId: 'md-active', content: 'ward destroying history.' } as ChatAction, + serverSeq: 1, + origin: undefined, + }); + await timeout(10); + + assert.deepStrictEqual( + (session.progressObs?.get() ?? []).map(part => part.kind === 'markdownContent' ? `markdown:${part.content.value}` : part.kind), + ['toolInvocationSerialized', 'markdown:that fallback fails to', 'markdown:ward destroying history.'], + ); + })); + + test('replays a settled subagent tool call once, keeping the streamed markdown in one part', () => runWithFakedTimers({ useFakeTimers: true }, async () => { + // The child session still needs observing, but that must not cost a + // second parent card between the two markdown fragments. + const { sessionHandler, agentHostService } = createContribution(disposables); + + const sessionUri = AgentSession.uri('copilot', 'reconnect-settled-subagent'); + const sessionState = makeSessionStateWithActiveTurn(sessionUri.toString(), { streamingText: 'that fallback fails to' }); + sessionState.activeTurn!.responseParts.unshift({ + kind: ResponsePartKind.ToolCall, + toolCall: { + toolCallId: 'tc-subagent-done', + toolName: 'task', + displayName: 'Delegating task', + invocationMessage: 'Delegating task', + pastTenseMessage: 'Delegated task', + status: ToolCallStatus.Completed, + confirmed: ToolCallConfirmationReason.NotNeeded, + success: true, + content: [], + }, + }); + agentHostService.sessionStates.set(sessionUri.toString(), sessionState); + // Register the child chat so subagent observation resolves a + // distinct, terminal session instead of recursing into the parent. + agentHostService.sessionStates.set(buildSubagentChatUri(sessionUri.toString(), 'tc-subagent-done'), { + ...createSessionState({ resource: buildSubagentChatUri(sessionUri.toString(), 'tc-subagent-done'), provider: 'copilot', title: 'Delegated task', status: SessionStatus.Idle, createdAt: new Date().toISOString(), modifiedAt: new Date().toISOString() }), + lifecycle: SessionLifecycle.Ready, + turns: [{ + id: 'child-turn-1', + message: { text: 'do the task', origin: { kind: MessageKind.User } }, + state: TurnState.Complete, + responseParts: [], + usage: undefined, + }], + } as SessionState); + + const sessionResource = URI.from({ scheme: 'agent-host-copilot', path: '/reconnect-settled-subagent' }); + const session = await sessionHandler.provideChatSessionContent(sessionResource, CancellationToken.None); + disposables.add(toDisposable(() => session.dispose())); + + agentHostService.fireAction({ + channel: sessionUri.toString(), + action: { type: 'chat/delta', turnId: 'turn-active', partId: 'md-active', content: 'ward destroying history.' } as ChatAction, + serverSeq: 1, + origin: undefined, + }); + await timeout(10); + + assert.deepStrictEqual( + (session.progressObs?.get() ?? []).map(part => part.kind === 'markdownContent' ? `markdown:${part.content.value}` : part.kind), + ['toolInvocationSerialized', 'markdown:that fallback fails to', 'markdown:ward destroying history.'], + ); + })); + test('adopts and updates an active streaming tool call after reconnect', async () => { const { sessionHandler, agentHostService } = createContribution(disposables); const sessionUri = AgentSession.uri('copilot', 'reconnect-streaming-tool'); From 64506369315f901827a490c66c7952eb3a6a08cb Mon Sep 17 00:00:00 2001 From: roblourens Date: Sat, 15 Aug 2026 18:05:04 -0700 Subject: [PATCH 2/3] agentHost: track Copilot client startup outcomes (#331044) * agentHost: track Copilot client startup outcomes Emit one terminal startup event per Copilot client attempt and separate startup failures from established-client operation failures. (Written by Copilot) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * agentHost: address startup telemetry review Make startup failure classification consistent and remove timing-sensitive test cleanup. (Written by Copilot) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * agentHost: structure startup error classification Use a typed VS Code startup error and centralize SDK message classification. (Written by Copilot) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../agentHost/node/copilot/copilotAgent.ts | 99 +++---- .../node/copilot/copilotFailureTelemetry.ts | 122 ++++++--- .../agentHost/test/node/copilotAgent.test.ts | 245 +++++++++++++++--- .../test/node/copilotFailureTelemetry.test.ts | 94 +++++-- 4 files changed, 419 insertions(+), 141 deletions(-) diff --git a/src/vs/platform/agentHost/node/copilot/copilotAgent.ts b/src/vs/platform/agentHost/node/copilot/copilotAgent.ts index 3dfc29ee36905a..5db3c597357070 100644 --- a/src/vs/platform/agentHost/node/copilot/copilotAgent.ts +++ b/src/vs/platform/agentHost/node/copilot/copilotAgent.ts @@ -91,7 +91,7 @@ import { DiscoveredType, SessionCustomizationDiscovery, areDiscoveredDirectories import { COPILOT_INTEGRATION_ID } from '../../../endpoint/common/licenseAgreement.js'; import { getAppNodeModulesPath } from '../appNodeModules.js'; import { CopilotSlashCommandProvider } from './copilotSlashCommandProvider.js'; -import { classifyCopilotClientFailure, createCopilotFailureCorrelation, reportCopilotClientFailure, reportCopilotClientRecovery, reportCopilotClientRecoveryTurn, type CopilotClientFailureKind, type CopilotClientFailureOperation, type ICopilotFailureCorrelation } from './copilotFailureTelemetry.js'; +import { classifyCopilotClientOperationFailure, CopilotClientStartupConfigChangedError, createCopilotFailureCorrelation, isRecognizedCopilotClientStartupFailure, reportCopilotClientOperationFailure, reportCopilotClientRecovery, reportCopilotClientRecoveryTurn, reportCopilotClientStartup, type CopilotClientOperation, type CopilotClientOperationFailureKind, type ICopilotFailureCorrelation } from './copilotFailureTelemetry.js'; interface ICopilotRuntimeManagedSettingsInput { authInfo?: { type: 'token'; host: string; token: string }; @@ -163,7 +163,7 @@ interface ICopilotClosedConnectionRecoveryResult { } function isCopilotConnectionClosedError(error: unknown): boolean { - return classifyCopilotClientFailure(error) === 'connectionClosed'; + return classifyCopilotClientOperationFailure(error) === 'connectionClosed'; } /** @@ -662,6 +662,7 @@ export class CopilotAgent extends Disposable implements IAgent { private _client: CopilotClient | undefined; private _clientStarting: Promise | undefined; private _clientStopping: Promise | undefined; + private _clientStartupAttemptCount = 0; private _resolvedProxy: string | undefined; private _proxyRefresh: Promise | undefined; private _proxyResolutionGeneration = 0; @@ -672,7 +673,6 @@ export class CopilotAgent extends Disposable implements IAgent { */ private readonly _pendingClientRestartReasons = new Set(); private _closedConnectionRecovery: { readonly clientFailureId: string; readonly promise: Promise } | undefined; - private readonly _reportedClientFailures = new WeakSet(); private readonly _authenticationSequencer = new Sequencer(); private _updatingGitHubCredentials = false; private _githubToken: string | undefined; @@ -1047,18 +1047,15 @@ export class CopilotAgent extends Disposable implements IAgent { }); } - private async _recoverFromClosedConnection(error: unknown, operation: CopilotClientFailureOperation, correlation?: ICopilotFailureCorrelation): Promise { - const failureKind = classifyCopilotClientFailure(error); + private async _handleClientOperationFailure(error: unknown, operation: CopilotClientOperation, correlation?: ICopilotFailureCorrelation): Promise { + const failureKind = classifyCopilotClientOperationFailure(error); if (!failureKind) { return undefined; } - if (error instanceof Error && this._reportedClientFailures.has(error)) { - return undefined; - } const clientFailureId = this._closedConnectionRecovery?.clientFailureId ?? generateUuid(); const recoveryStarted = failureKind === 'connectionClosed' && !this._shutdownPromise && this._closedConnectionRecovery === undefined; - reportCopilotClientFailure(this._telemetryService, clientFailureId, failureKind, operation, this._chatsWithActiveTurn(), recoveryStarted, error, correlation); + reportCopilotClientOperationFailure(this._telemetryService, clientFailureId, failureKind, operation, this._chatsWithActiveTurn(), recoveryStarted, error, correlation); if (failureKind !== 'connectionClosed' || this._shutdownPromise) { return undefined; } @@ -1077,7 +1074,7 @@ export class CopilotAgent extends Disposable implements IAgent { return this._closedConnectionRecovery.promise; } - private async _runClosedConnectionRecovery(clientFailureId: string, failureKind: CopilotClientFailureKind): Promise { + private async _runClosedConnectionRecovery(clientFailureId: string, failureKind: CopilotClientOperationFailureKind): Promise { const stopWatch = StopWatch.create(); const result = await this._doRecoverFromClosedConnection(clientFailureId); reportCopilotClientRecovery(this._telemetryService, { @@ -1123,14 +1120,15 @@ export class CopilotAgent extends Disposable implements IAgent { return { failedTurnIds, stopSucceeded }; } - private async _retryAfterClosedConnection(operation: CopilotClientFailureOperation, task: () => Promise, correlation?: ICopilotFailureCorrelation): Promise { + private async _retryAfterClosedConnection(operation: CopilotClientOperation, task: (client: CopilotClient) => Promise, correlation?: ICopilotFailureCorrelation): Promise { + const client = await this._ensureClient(); try { - return await task(); + return await task(client); } catch (error) { - if (!await this._recoverFromClosedConnection(error, operation, correlation)) { + if (!await this._handleClientOperationFailure(error, operation, correlation)) { throw error; } - return task(); + return task(await this._ensureClient()); } } @@ -1630,7 +1628,7 @@ export class CopilotAgent extends Disposable implements IAgent { if (/\b401\b/.test(getErrorMessage(err))) { this._handleCopilotSessionAuthRequired(); } - await this._recoverFromClosedConnection(err, 'modelRefresh'); + await this._handleClientOperationFailure(err, 'modelRefresh'); if (attempt + 1 < this._modelRefreshMaxAttempts) { const delay = this._modelRefreshBackoff(attempt); this._logService.warn(`[Copilot] Failed to refresh models (attempt ${attempt + 1}), retrying in ${delay}ms`, err); @@ -1739,6 +1737,15 @@ export class CopilotAgent extends Disposable implements IAgent { // ---- client lifecycle --------------------------------------------------- + private async _stopClientAfterStartupTermination(client: CopilotClient, terminalError: Error): Promise { + try { + await client.stop(); + } catch (error) { + this._logService.error(error, '[Copilot] Failed to stop client after startup termination'); + } + throw terminalError; + } + private async _ensureClient(): Promise { if (this._shutdownPromise) { throw new CancellationError(); @@ -1766,7 +1773,9 @@ export class CopilotAgent extends Disposable implements IAgent { const copilotSdkLogLevelSettingAtStartup = this._getCopilotSdkLogLevelSetting(); const enterpriseHostAtStartup = this._getEnterpriseHost(); const systemProxyEnabledAtStartup = this._isSystemProxyEnabled(); - const clientStarting = (async () => { + const attemptNumber = ++this._clientStartupAttemptCount; + const startupStopWatch = StopWatch.create(); + const startClient = async () => { this._logService.info('[Copilot] Starting CopilotClient...'); // Build a clean env for the CLI subprocess, stripping Electron/VS Code vars @@ -1875,28 +1884,36 @@ export class CopilotAgent extends Disposable implements IAgent { onGitHubTelemetry: notification => { void this._routeGitHubTelemetry(notification).catch(err => this._logService.trace(`[Copilot] GitHub telemetry routing failed: ${err instanceof Error ? err.message : String(err)}`)); }, }; const client = this._createCopilotClient(clientOptions); - try { - await client.start(); - } catch (error) { - const failureKind = classifyCopilotClientFailure(error); - if (failureKind && error instanceof Error) { - reportCopilotClientFailure(this._telemetryService, generateUuid(), failureKind, 'startClient', this._chatsWithActiveTurn(), false, error); - this._reportedClientFailures.add(error); - } - throw error; - } + await client.start(); if (this._shutdownPromise) { - await client.stop(); - throw new CancellationError(); + return this._stopClientAfterStartupTermination(client, new CancellationError()); } if (this._isSessionSyncEnabled() !== sessionSyncAtStartup || this._isRubberDuckEnabled() !== rubberDuckAtStartup || this._getCopilotSdkLogLevelSetting() !== copilotSdkLogLevelSettingAtStartup || this._getEnterpriseHost() !== enterpriseHostAtStartup || this._isSystemProxyEnabled() !== systemProxyEnabledAtStartup) { - await client.stop(); - throw new Error('Copilot startup config changed while the client was starting'); + return this._stopClientAfterStartupTermination(client, new CopilotClientStartupConfigChangedError()); } this._logService.info('[Copilot] CopilotClient started successfully'); this._client = client; this._clientStarting = undefined; return client; + }; + const clientStarting = (async () => { + let outcome: 'success' | 'failure' | 'cancelled' = 'failure'; + let startupError: unknown; + try { + const client = await startClient(); + outcome = 'success'; + return client; + } catch (error) { + startupError = error; + outcome = error instanceof CancellationError ? 'cancelled' : 'failure'; + throw error; + } finally { + reportCopilotClientStartup(this._telemetryService, { + outcome, + durationMs: startupStopWatch.elapsed(), + attemptNumber, + }, startupError); + } })(); this._clientStarting = clientStarting; void clientStarting.catch(() => { @@ -2161,12 +2178,9 @@ export class CopilotAgent extends Disposable implements IAgent { private async _listSdkSessions(reason: string): Promise> | undefined> { this._logService.info(`[Copilot] Listing ${reason}...`); try { - return await this._retryAfterClosedConnection('listSessions', async () => { - const client = await this._ensureClient(); - return client.listSessions(); - }); + return await this._retryAfterClosedConnection('listSessions', client => client.listSessions()); } catch (err) { - if (err instanceof CancellationError || classifyCopilotClientFailure(err) !== undefined) { + if (err instanceof CancellationError || isRecognizedCopilotClientStartupFailure(err) || classifyCopilotClientOperationFailure(err) !== undefined) { this._logService.info(`[Copilot] Client unavailable while listing ${reason}: ${err instanceof Error ? err.message : String(err)}`); return undefined; } @@ -2182,10 +2196,7 @@ export class CopilotAgent extends Disposable implements IAgent { } const storedMetadata = await this._readStoredSessionMetadata(session); - const sessionMetadata = await this._retryAfterClosedConnection('getSessionMetadata', async () => { - const client = await this._ensureClient(); - return client.getSessionMetadata(sessionId); - }, createCopilotFailureCorrelation(session, chat, undefined, sessionId)); + const sessionMetadata = await this._retryAfterClosedConnection('getSessionMetadata', client => client.getSessionMetadata(sessionId), createCopilotFailureCorrelation(session, chat, undefined, sessionId)); if (!sessionMetadata) { return undefined; } @@ -3090,7 +3101,7 @@ export class CopilotAgent extends Disposable implements IAgent { try { await this._sendMessageOnce(chat, prompt, attachments, turnId, senderClientId, clientType, workingDirectories, operationContext, clientTelemetryContext); } catch (error) { - const recovery = await this._recoverFromClosedConnection(error, 'sendMessage', this._clientFailureCorrelation(chat, turnId, operationContext)); + const recovery = await this._handleClientOperationFailure(error, 'sendMessage', this._clientFailureCorrelation(chat, turnId, operationContext)); if (turnId && recovery?.failedTurnIds.has(turnId)) { return; } @@ -3297,11 +3308,11 @@ export class CopilotAgent extends Disposable implements IAgent { } catch (error) { const correlation = this._clientFailureCorrelation(chat, undefined, operationContext); if (!isCopilotConnectionClosedError(error)) { - await this._recoverFromClosedConnection(error, 'abort', correlation); + await this._handleClientOperationFailure(error, 'abort', correlation); throw error; } this._resolveChatContext(chat, operationContext).target?.discardActiveTurn(); - if (!await this._recoverFromClosedConnection(error, 'abort', correlation)) { + if (!await this._handleClientOperationFailure(error, 'abort', correlation)) { throw error; } } @@ -3907,7 +3918,7 @@ export class CopilotAgent extends Disposable implements IAgent { try { await this._changeModelOnce(chat, model, operationContext); } catch (error) { - if (!await this._recoverFromClosedConnection(error, 'changeModel', this._clientFailureCorrelation(chat, undefined, operationContext))) { + if (!await this._handleClientOperationFailure(error, 'changeModel', this._clientFailureCorrelation(chat, undefined, operationContext))) { throw error; } await this._changeModelOnce(chat, model, operationContext); @@ -3947,7 +3958,7 @@ export class CopilotAgent extends Disposable implements IAgent { try { await this._changeAgentOnce(chat, agent, operationContext); } catch (error) { - if (!await this._recoverFromClosedConnection(error, 'changeAgent', this._clientFailureCorrelation(chat, undefined, operationContext))) { + if (!await this._handleClientOperationFailure(error, 'changeAgent', this._clientFailureCorrelation(chat, undefined, operationContext))) { throw error; } await this._changeAgentOnce(chat, agent, operationContext); diff --git a/src/vs/platform/agentHost/node/copilot/copilotFailureTelemetry.ts b/src/vs/platform/agentHost/node/copilot/copilotFailureTelemetry.ts index f4a4af5c8a4523..3595bf3b2d5ac5 100644 --- a/src/vs/platform/agentHost/node/copilot/copilotFailureTelemetry.ts +++ b/src/vs/platform/agentHost/node/copilot/copilotFailureTelemetry.ts @@ -13,11 +13,19 @@ import type { IAgentHostClientTelemetryContext } from '../../common/agentHostTel import { getTelemetryChatSessionId } from '../../common/agentTelemetryCorrelation.js'; import { toInitiatorTelemetry, type IAgentHostInitiatorClassification, type IAgentHostInitiatorTelemetry } from '../agentHostTelemetryReporter.js'; -export type CopilotClientFailureOperation = 'abort' | 'changeAgent' | 'changeModel' | 'getSessionMetadata' | 'listSessions' | 'modelRefresh' | 'sendMessage' | 'startClient'; -export type CopilotClientFailureKind = 'clientNotConnected' | 'connectionClosed' | 'connectionDisposed' | 'runtimeConnectionClosed' | 'startupFailed'; -type CopilotStartupFailureCause = 'nativeModuleProcedureNotFound' | 'nativeModuleInitializationFailed' | 'nativeModuleNotFound' | 'permissionDenied' | 'timeout' | 'spawnFailed' | 'processExitedUnexpectedly' | 'processExited'; +export type CopilotClientOperation = 'abort' | 'changeAgent' | 'changeModel' | 'getSessionMetadata' | 'listSessions' | 'modelRefresh' | 'sendMessage'; +export type CopilotClientOperationFailureKind = 'clientNotConnected' | 'connectionClosed' | 'connectionDisposed' | 'runtimeConnectionClosed'; +type CopilotClientStartupOutcome = 'success' | 'failure' | 'cancelled'; +type CopilotStartupFailureCause = 'nativeModuleProcedureNotFound' | 'nativeModuleInitializationFailed' | 'nativeModuleNotFound' | 'permissionDenied' | 'timeout' | 'spawnFailed' | 'processExitedUnexpectedly' | 'processExited' | 'configurationChanged' | 'other'; type CopilotStartupFailureResource = 'runtime' | 'cliNative' | 'conpty' | 'sandbox' | 'other'; +export class CopilotClientStartupConfigChangedError extends Error { + constructor() { + super('Copilot startup config changed while the client was starting'); + this.name = 'CopilotClientStartupConfigChangedError'; + } +} + export interface ICopilotFailureCorrelation extends IAgentHostInitiatorTelemetry { readonly agentSessionId?: string; readonly chatSessionId?: string; @@ -74,7 +82,7 @@ export function createCopilotFailureCorrelation(sessionUri: URI, chatUri: URI, t }; } -export function classifyCopilotClientFailure(error: unknown): CopilotClientFailureKind | undefined { +export function classifyCopilotClientOperationFailure(error: unknown): CopilotClientOperationFailureKind | undefined { if (!(error instanceof Error)) { return undefined; } @@ -88,21 +96,17 @@ export function classifyCopilotClientFailure(error: unknown): CopilotClientFailu case 'The in-process runtime connection is closed.': return 'runtimeConnectionClosed'; } - return error.message.startsWith('Failed to start CLI server:') - || error.message.startsWith('CLI server exited with code ') - || error.message.startsWith('CLI server exited unexpectedly with code ') - || error.message === 'Timeout waiting for CLI server to start' - ? 'startupFailed' - : undefined; + return undefined; +} + +export function isRecognizedCopilotClientStartupFailure(error: unknown): boolean { + return error instanceof Error && getCopilotStartupFailureCause(error) !== undefined; } -type CopilotClientFailureEvent = ICopilotFailureCorrelation & { +type CopilotClientOperationFailureEvent = ICopilotFailureCorrelation & { clientFailureId: string; - failureKind: CopilotClientFailureKind; - operation: CopilotClientFailureOperation; - startupFailureCause?: CopilotStartupFailureCause; - startupFailureResource?: CopilotStartupFailureResource; - startupExitCode?: number; + failureKind: CopilotClientOperationFailureKind; + operation: CopilotClientOperation; activeTurnCount: number; recoveryStarted: boolean; errorName: string | undefined; @@ -111,13 +115,10 @@ type CopilotClientFailureEvent = ICopilotFailureCorrelation & { callstack: string | undefined; }; -type CopilotClientFailureClassification = IAgentHostInitiatorClassification & { +type CopilotClientOperationFailureClassification = IAgentHostInitiatorClassification & { clientFailureId: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Identifier shared by detections and recovery telemetry for one Copilot client failure episode.' }; failureKind: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The bounded category of Copilot client failure that was detected.' }; operation: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The Copilot provider operation that detected the client failure.' }; - startupFailureCause?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The bounded cause extracted from a Copilot client startup failure.' }; - startupFailureResource?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The bounded Copilot CLI resource involved in a startup failure.' }; - startupExitCode?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'The Copilot CLI process exit code reported for a startup failure.' }; agentSessionId?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The Agent Host session identifier, when the failing operation targeted a session.' }; chatSessionId?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The Agent Host chat identifier, when the failing operation targeted a chat.' }; turnId?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The Agent Host turn identifier, when available.' }; @@ -129,10 +130,35 @@ type CopilotClientFailureClassification = IAgentHostInitiatorClassification & { msg: { classification: 'CallstackOrException'; purpose: 'PerformanceAndHealth'; comment: 'The client failure message. VS Code telemetry scrubs file paths and likely secrets before transmission.' }; callstack: { classification: 'CallstackOrException'; purpose: 'PerformanceAndHealth'; comment: 'The client failure stack. VS Code telemetry scrubs file paths and likely secrets before transmission.' }; owner: 'roblourens'; - comment: 'Tracks detected Copilot client failures and whether recovery was started.'; + comment: 'Tracks failures detected while operating an established Copilot client and whether recovery was started.'; +}; + +type CopilotClientStartupEvent = { + outcome: CopilotClientStartupOutcome; + durationMs: number; + attemptNumber: number; + startupFailureCause?: CopilotStartupFailureCause; + startupFailureResource?: CopilotStartupFailureResource; + startupExitCode?: number; +}; + +type CopilotClientStartupClassification = { + outcome: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Whether the startup attempt succeeded, failed, or was cancelled during shutdown.' }; + durationMs: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Wall-clock duration of the Copilot client startup attempt in milliseconds.' }; + attemptNumber: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'One-based Copilot client startup attempt number within this Agent Host process.' }; + startupFailureCause?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The bounded cause of a failed Copilot client startup attempt.' }; + startupFailureResource?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The bounded Copilot CLI resource involved in a failed startup attempt.' }; + startupExitCode?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'The Copilot CLI process exit code reported for a failed startup attempt.' }; + owner: 'roblourens'; + comment: 'Tracks one terminal outcome for every Copilot client startup attempt.'; }; -function getCopilotStartupFailureCause(message: string): CopilotStartupFailureCause { +function getCopilotStartupFailureCause(error: Error): CopilotStartupFailureCause | undefined { + if (error instanceof CopilotClientStartupConfigChangedError) { + return 'configurationChanged'; + } + + const message = error.message; const normalizedMessage = message.toLowerCase(); if (normalizedMessage.includes('specified procedure could not be found')) { return 'nativeModuleProcedureNotFound'; @@ -152,9 +178,13 @@ function getCopilotStartupFailureCause(message: string): CopilotStartupFailureCa if (message.startsWith('Failed to start CLI server:')) { return 'spawnFailed'; } - return message.startsWith('CLI server exited unexpectedly with code ') - ? 'processExitedUnexpectedly' - : 'processExited'; + if (message.startsWith('CLI server exited unexpectedly with code ')) { + return 'processExitedUnexpectedly'; + } + if (message.startsWith('CLI server exited with code ')) { + return 'processExited'; + } + return undefined; } function getCopilotStartupFailureResource(message: string): CopilotStartupFailureResource { @@ -177,8 +207,12 @@ function getCopilotStartupFailureResource(message: string): CopilotStartupFailur return 'other'; } -function getCopilotStartupFailureDetails(error: unknown): Pick { - if (!(error instanceof Error) || classifyCopilotClientFailure(error) !== 'startupFailed') { +function getCopilotStartupFailureDetails(error: unknown): Pick { + if (!(error instanceof Error)) { + return {}; + } + const startupFailureCause = getCopilotStartupFailureCause(error); + if (!startupFailureCause) { return {}; } @@ -187,28 +221,48 @@ function getCopilotStartupFailureDetails(error: unknown): Pick, + error?: unknown, +): void { + let failureDetails: Pick = {}; + if (data.outcome === 'failure') { + failureDetails = getCopilotStartupFailureDetails(error); + if (!failureDetails.startupFailureCause) { + failureDetails = { + startupFailureCause: 'other', + startupFailureResource: 'other', + }; + } + } + telemetryService.publicLog2('agentHost.copilotClientStartup', { + ...data, + ...failureDetails, + }); +} + +export function reportCopilotClientOperationFailure( telemetryService: ITelemetryService, clientFailureId: string, - failureKind: CopilotClientFailureKind, - operation: CopilotClientFailureOperation, + failureKind: CopilotClientOperationFailureKind, + operation: CopilotClientOperation, activeTurnCount: number, recoveryStarted: boolean, error: unknown, correlation?: ICopilotFailureCorrelation, ): void { const packed = packErrorForTelemetry(error); - telemetryService.publicLogError2('agentHost.copilotClientFailure', { + telemetryService.publicLogError2('agentHost.copilotClientFailure', { clientFailureId, failureKind, operation, - ...getCopilotStartupFailureDetails(error), ...correlation, activeTurnCount, recoveryStarted, @@ -221,7 +275,7 @@ export function reportCopilotClientFailure( type CopilotClientRecoveryEvent = { clientFailureId: string; - failureKind: CopilotClientFailureKind; + failureKind: CopilotClientOperationFailureKind; durationMs: number; failedTurnCount: number; stopSucceeded: boolean; diff --git a/src/vs/platform/agentHost/test/node/copilotAgent.test.ts b/src/vs/platform/agentHost/test/node/copilotAgent.test.ts index 98b028bdccff20..532fde1ea8e410 100644 --- a/src/vs/platform/agentHost/test/node/copilotAgent.test.ts +++ b/src/vs/platform/agentHost/test/node/copilotAgent.test.ts @@ -479,6 +479,7 @@ class TestCopilotClient implements ITestCopilotClient { }; startCallCount = 0; stopCallCount = 0; + readonly startCalled = new DeferredPromise(); startGate: Promise | undefined; startError: Error | undefined; listSessionCallCount = 0; @@ -499,6 +500,7 @@ class TestCopilotClient implements ITestCopilotClient { async start(): Promise { this.startCallCount++; + this.startCalled.complete(); await this.startGate; if (this.startError) { throw this.startError; @@ -1002,8 +1004,16 @@ suite('CopilotAgent', () => { restricted: false, ...(assignmentContext ? { 'abexp.assignmentcontext': assignmentContext } : {}), }); - assert.deepStrictEqual({ events: telemetryService.events, experimentProperties: telemetryService.experimentProperties }, { + const events = telemetryService.events.map(event => { + if (event.eventName !== 'agentHost.copilotClientStartup') { + return event; + } + const data = event.data as Record; + return { ...event, data: { ...data, durationMs: typeof data.durationMs } }; + }); + assert.deepStrictEqual({ events, experimentProperties: telemetryService.experimentProperties }, { events: [ + { eventName: 'agentHost.copilotClientStartup', data: { outcome: 'success', durationMs: 'number', attemptNumber: 1 } }, { eventName: 'copilotSdk/response.success', data: expectedData('set', 'experiment:1') }, { eventName: 'copilotSdk/response.success', data: expectedData('wiped-sticky', 'experiment:1') }, { eventName: 'copilotSdk/response.success', data: expectedData('cleared') }, @@ -1056,15 +1066,18 @@ suite('CopilotAgent', () => { await forward(notification('idle-session', 'runtime-idle')); await forward(notification('unknown-session', 'runtime-unknown')); - assert.deepStrictEqual(telemetryService.events.map(event => ({ - sessionId: (event.data as Record).sdk_session_id, - turnId: (event.data as Record).turnId, - })), [ - { sessionId: 'active-session', turnId: 'turn-1' }, - { sessionId: 'second-active-session', turnId: 'turn-2' }, - { sessionId: 'active-session', turnId: 'turn-1' }, - { sessionId: 'idle-session', turnId: undefined }, - { sessionId: 'unknown-session', turnId: undefined }, + assert.deepStrictEqual(telemetryService.events.map(event => { + const data = event.data as Record; + return event.eventName === 'agentHost.copilotClientStartup' + ? { eventName: event.eventName, outcome: data.outcome, durationMs: typeof data.durationMs, attemptNumber: data.attemptNumber } + : { eventName: event.eventName, sessionId: data.sdk_session_id, turnId: data.turnId }; + }), [ + { eventName: 'agentHost.copilotClientStartup', outcome: 'success', durationMs: 'number', attemptNumber: 1 }, + { eventName: 'copilotSdk/response.success', sessionId: 'active-session', turnId: 'turn-1' }, + { eventName: 'copilotSdk/response.success', sessionId: 'second-active-session', turnId: 'turn-2' }, + { eventName: 'copilotSdk/response.success', sessionId: 'active-session', turnId: 'turn-1' }, + { eventName: 'copilotSdk/response.success', sessionId: 'idle-session', turnId: undefined }, + { eventName: 'copilotSdk/response.success', sessionId: 'unknown-session', turnId: undefined }, ]); } finally { await disposeAgent(agent); @@ -2118,7 +2131,7 @@ suite('CopilotAgent', () => { await agent.authenticate('https://api.github.com', 'token'); const models = await waitForState(agent.models, m => m.length > 0); const failure = telemetryService.errorEvents[0].data as Record; - const recovery = telemetryService.events[0].data as Record; + const recovery = telemetryService.events.find(event => event.eventName === 'agentHost.copilotClientRecovery')?.data as Record; assert.deepStrictEqual({ modelNames: models.map(model => model.name), @@ -2167,8 +2180,95 @@ suite('CopilotAgent', () => { } }); + test('reports one successful Copilot client startup outcome for concurrent callers', async () => { + const client = new TestCopilotClient([]); + const startGate = new DeferredPromise(); + client.startGate = startGate.p; + const telemetryService = new RecordingTelemetryService(); + const agent = createTestAgent(disposables, { copilotClient: client, telemetryService }); + const first = agent.listChatsToMigrate(); + const second = agent.listChatsToMigrate(); + try { + await client.startCalled.p; + startGate.complete(); + await Promise.all([first, second]); + const startupEvents = telemetryService.events + .filter(event => event.eventName === 'agentHost.copilotClientStartup') + .map(event => { + const data = event.data as Record; + return { ...data, durationMs: typeof data.durationMs }; + }); + + assert.deepStrictEqual({ + startCallCount: client.startCallCount, + listSessionCallCount: client.listSessionCallCount, + startupEvents, + }, { + startCallCount: 1, + listSessionCallCount: 2, + startupEvents: [{ + outcome: 'success', + durationMs: 'number', + attemptNumber: 1, + }], + }); + } finally { + startGate.complete(); + await Promise.allSettled([first, second]); + await disposeAgent(agent); + } + }); + + test('reports one startup failure and no operation failure when concurrent callers share a failed start', async () => { + const client = new TestCopilotClient([]); + const startGate = new DeferredPromise(); + client.startGate = startGate.p; + client.startError = new Error('Connection is closed.'); + const telemetryService = new RecordingTelemetryService(); + const agent = createTestAgent(disposables, { copilotClient: client, telemetryService }); + const first = agent.listChatsToMigrate(); + const second = agent.listChatsToMigrate(); + try { + await client.startCalled.p; + startGate.complete(); + const results = await Promise.all([first, second]); + const startupEvents = telemetryService.events.map(event => { + const data = event.data as Record; + return { eventName: event.eventName, ...data, durationMs: typeof data.durationMs }; + }); + + assert.deepStrictEqual({ + results, + startCallCount: client.startCallCount, + stopCallCount: client.stopCallCount, + listSessionCallCount: client.listSessionCallCount, + startupEvents, + errorEvents: telemetryService.errorEvents, + }, { + results: [undefined, undefined], + startCallCount: 1, + stopCallCount: 0, + listSessionCallCount: 0, + startupEvents: [{ + eventName: 'agentHost.copilotClientStartup', + outcome: 'failure', + durationMs: 'number', + attemptNumber: 1, + startupFailureCause: 'other', + startupFailureResource: 'other', + }], + errorEvents: [], + }); + } finally { + startGate.complete(); + await Promise.allSettled([first, second]); + client.startError = undefined; + await disposeAgent(agent); + } + }); + test('surfaces undefined (not a rejection) for a classified Copilot client startup failure', async () => { - // A `startupFailed`-classified error means the CLI client is transiently + // A recognized startup error means the CLI client is transiently // unavailable, not that this provider authoritatively has no legacy // chats: `listChatsToMigrate` must resolve to `undefined` (still reporting // the failure via telemetry) rather than reject or return `[]`. @@ -2178,25 +2278,23 @@ suite('CopilotAgent', () => { const agent = createTestAgent(disposables, { copilotClient: client, telemetryService }); try { assert.strictEqual(await agent.listChatsToMigrate(), undefined); - assert.strictEqual(telemetryService.errorEvents.length, 1); - const failure = telemetryService.errorEvents[0].data as Record; + const startup = telemetryService.events.find(event => event.eventName === 'agentHost.copilotClientStartup')?.data as Record; assert.deepStrictEqual({ - ...failure, - clientFailureId: typeof failure.clientFailureId, - callstack: typeof failure.callstack, + operationFailureEvents: telemetryService.errorEvents.filter(event => event.eventName === 'agentHost.copilotClientFailure').length, + startup: { + ...startup, + durationMs: typeof startup.durationMs, + }, }, { - clientFailureId: 'string', - failureKind: 'startupFailed', - operation: 'startClient', - startupFailureCause: 'spawnFailed', - startupFailureResource: 'other', - startupExitCode: undefined, - activeTurnCount: 0, - recoveryStarted: false, - errorName: 'Error', - errorCode: undefined, - msg: 'Failed to start CLI server: spawn failed', - callstack: 'string', + operationFailureEvents: 0, + startup: { + outcome: 'failure', + durationMs: 'number', + attemptNumber: 1, + startupFailureCause: 'spawnFailed', + startupFailureResource: 'other', + startupExitCode: undefined, + }, }); } finally { client.startError = undefined; @@ -2249,21 +2347,29 @@ suite('CopilotAgent', () => { try { for (const testCase of cases) { client.startError = new Error(testCase.message); - // All of these are `startupFailed`-classified: the client is + // All of these are recognized startup failures: the client is // transiently unavailable, so `listChatsToMigrate` resolves to // `undefined` (still reporting telemetry below) rather than // rejecting. assert.strictEqual(await agent.listChatsToMigrate(), undefined); } - assert.deepStrictEqual(telemetryService.errorEvents.map(event => { + assert.deepStrictEqual(telemetryService.events.filter(event => event.eventName === 'agentHost.copilotClientStartup').map(event => { const data = event.data as Record; return { + outcome: data.outcome, + durationMs: typeof data.durationMs, + attemptNumber: data.attemptNumber, startupFailureCause: data.startupFailureCause, startupFailureResource: data.startupFailureResource, startupExitCode: data.startupExitCode, }; - }), cases.map(testCase => testCase.expected)); + }), cases.map((testCase, index) => ({ + outcome: 'failure', + durationMs: 'number', + attemptNumber: index + 1, + ...testCase.expected, + }))); } finally { client.startError = undefined; await disposeAgent(agent); @@ -2272,7 +2378,7 @@ suite('CopilotAgent', () => { test('coalesces closed connection recovery and preserves an already-cancelled turn', async () => { type RecoveryInternals = { - _recoverFromClosedConnection(error: unknown, operation: 'modelRefresh'): Promise<{ failedTurnIds: ReadonlySet } | undefined>; + _handleClientOperationFailure(error: unknown, operation: 'modelRefresh'): Promise<{ failedTurnIds: ReadonlySet } | undefined>; }; class GatedStopClient extends TestCopilotClient { readonly stopGate = new DeferredPromise(); @@ -2333,7 +2439,7 @@ suite('CopilotAgent', () => { await timeout(0); } const internals = agent as unknown as RecoveryInternals; - const second = internals._recoverFromClosedConnection(new Error('Connection is closed.'), 'modelRefresh'); + const second = internals._handleClientOperationFailure(new Error('Connection is closed.'), 'modelRefresh'); client.stopGate.complete(); const [, secondResult] = await Promise.all([abort, second]); const failures = telemetryService.errorEvents @@ -2342,7 +2448,7 @@ suite('CopilotAgent', () => { const recoveryTurns = telemetryService.errorEvents .filter(event => event.eventName === 'agentHost.copilotClientRecoveryTurnFailed') .map(event => event.data as Record); - const recovery = telemetryService.events[0].data as Record; + const recovery = telemetryService.events.find(event => event.eventName === 'agentHost.copilotClientRecovery')?.data as Record; assert.deepStrictEqual({ calls, @@ -2638,14 +2744,21 @@ suite('CopilotAgent', () => { } }); - test('stops a client that finishes starting after shutdown begins', async () => { - const client = new TestCopilotClient([]); + test('preserves startup cancellation when stopping the started client fails', async () => { + class FailingStopClient extends TestCopilotClient { + override async stop(): ReturnType { + await super.stop(); + throw new Error('stop failed'); + } + } + const client = new FailingStopClient([]); const startGate = new DeferredPromise(); client.startGate = startGate.p; - const agent = createTestAgent(disposables, { copilotClient: client }); + const telemetryService = new RecordingTelemetryService(); + const agent = createTestAgent(disposables, { copilotClient: client, telemetryService }); try { const listPromise = agent.listChatsToMigrate(); - await Promise.resolve(); + await client.startCalled.p; const shutdownPromise = agent.shutdown(); startGate.complete(); @@ -2659,9 +2772,20 @@ suite('CopilotAgent', () => { assert.deepStrictEqual({ starts: client.startCallCount, stops: client.stopCallCount, + startup: telemetryService.events + .filter(event => event.eventName === 'agentHost.copilotClientStartup') + .map(event => { + const data = event.data as Record; + return { ...data, durationMs: typeof data.durationMs }; + }), }, { starts: 1, stops: 1, + startup: [{ + outcome: 'cancelled', + durationMs: 'number', + attemptNumber: 1, + }], }); } finally { await disposeAgent(agent); @@ -2835,6 +2959,49 @@ suite('CopilotAgent', () => { } } + test('preserves configuration-changed outcome when stopping the started client fails', async () => { + const client = new StopCountingClient([]); + const startGate = new DeferredPromise(); + client.startGate = startGate.p; + client.stopError = new Error('stop failed'); + const telemetryService = new RecordingTelemetryService(); + const { agent, configurationService } = createTestAgentContext(disposables, { copilotClient: client, telemetryService }); + const startup = agent.listChatsToMigrate(); + try { + await client.startCalled.p; + configurationService.updateRootConfig({ [CopilotCliConfigKey.RubberDuck]: false }); + startGate.complete(); + + assert.strictEqual(await startup, undefined); + const startupEvents = telemetryService.events.map(event => { + const data = event.data as Record; + return { eventName: event.eventName, ...data, durationMs: typeof data.durationMs }; + }); + assert.deepStrictEqual({ + startCallCount: client.startCallCount, + stopCount: client.stopCount, + startupEvents, + }, { + startCallCount: 1, + stopCount: 1, + startupEvents: [{ + eventName: 'agentHost.copilotClientStartup', + outcome: 'failure', + durationMs: 'number', + attemptNumber: 1, + startupFailureCause: 'configurationChanged', + startupFailureResource: 'other', + startupExitCode: undefined, + }], + }); + } finally { + client.stopError = undefined; + startGate.complete(); + await startup; + await disposeAgent(agent); + } + }); + test('resolves the system proxy by default and bypasses it when disabled', async () => { const proxyResolver = new TestProxyResolver(); proxyResolver.resolvedProxy = 'http://system-proxy.example:8080'; diff --git a/src/vs/platform/agentHost/test/node/copilotFailureTelemetry.test.ts b/src/vs/platform/agentHost/test/node/copilotFailureTelemetry.test.ts index eff51699a4b222..cd87242804a795 100644 --- a/src/vs/platform/agentHost/test/node/copilotFailureTelemetry.test.ts +++ b/src/vs/platform/agentHost/test/node/copilotFailureTelemetry.test.ts @@ -14,7 +14,7 @@ import { AgentHostClientType } from '../../common/agentHostClientInfo.js'; import { AgentHostClientConnectionKind, AgentHostLaunchKind, AgentHostTransportKind } from '../../common/agentHostTelemetry.js'; import { readAgentErrorTelemetryMeta } from '../../common/meta/agentErrorMeta.js'; import { buildChatUri, buildSubagentSessionUri } from '../../common/state/sessionState.js'; -import { classifyCopilotClientFailure, createCopilotFailureCorrelation, normalizeCopilotApiEndpoint, reportCopilotModelCallFailure } from '../../node/copilot/copilotFailureTelemetry.js'; +import { classifyCopilotClientOperationFailure, CopilotClientStartupConfigChangedError, createCopilotFailureCorrelation, isRecognizedCopilotClientStartupFailure, normalizeCopilotApiEndpoint, reportCopilotClientStartup, reportCopilotModelCallFailure } from '../../node/copilot/copilotFailureTelemetry.js'; class CapturingTelemetryService implements ITelemetryService { declare readonly _serviceBrand: undefined; @@ -28,7 +28,9 @@ class CapturingTelemetryService implements ITelemetryService { readonly events: { eventName: string; data: Record | undefined }[] = []; publicLog(): void { } - publicLog2(): void { } + publicLog2(eventName: string, data?: Record): void { + this.events.push({ eventName, data }); + } publicLogError(): void { } publicLogError2(eventName: string, data?: Record): void { this.events.push({ eventName, data }); @@ -40,28 +42,72 @@ class CapturingTelemetryService implements ITelemetryService { suite('CopilotFailureTelemetry', () => { ensureNoDisposablesAreLeakedInTestSuite(); - test('classifies only known client lifecycle failures', () => { - assert.deepStrictEqual([ - classifyCopilotClientFailure(new Error('Connection is closed.')), - classifyCopilotClientFailure(new Error('Connection is disposed.')), - classifyCopilotClientFailure(new Error('Client not connected')), - classifyCopilotClientFailure(new Error('The in-process runtime connection is closed.')), - classifyCopilotClientFailure(new Error('Failed to start CLI server: spawn failed')), - classifyCopilotClientFailure(new Error('CLI server exited with code 1')), - classifyCopilotClientFailure(new Error('CLI server exited unexpectedly with code 1')), - classifyCopilotClientFailure(new Error('Timeout waiting for CLI server to start')), - classifyCopilotClientFailure(new Error('429 too many requests')), - ], [ - 'connectionClosed', - 'connectionDisposed', - 'clientNotConnected', - 'runtimeConnectionClosed', - 'startupFailed', - 'startupFailed', - 'startupFailed', - 'startupFailed', - undefined, - ]); + test('separates startup failures from established-client operation failures', () => { + const errors = [ + new Error('Connection is closed.'), + new Error('Connection is disposed.'), + new Error('Client not connected'), + new Error('The in-process runtime connection is closed.'), + new Error('Failed to start CLI server: spawn failed'), + new Error('CLI server exited with code 1'), + new Error('CLI server exited unexpectedly with code 1'), + new Error('Timeout waiting for CLI server to start'), + new CopilotClientStartupConfigChangedError(), + new Error('429 too many requests'), + ]; + assert.deepStrictEqual({ + operationFailures: errors.map(classifyCopilotClientOperationFailure), + startupFailures: errors.map(isRecognizedCopilotClientStartupFailure), + }, { + operationFailures: [ + 'connectionClosed', + 'connectionDisposed', + 'clientNotConnected', + 'runtimeConnectionClosed', + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + ], + startupFailures: [false, false, false, false, true, true, true, true, true, false], + }); + }); + + test('reports bounded causes for configuration changes and unknown startup failures', () => { + const telemetryService = new CapturingTelemetryService(); + reportCopilotClientStartup(telemetryService, { + outcome: 'failure', + durationMs: 10, + attemptNumber: 1, + }, new CopilotClientStartupConfigChangedError()); + reportCopilotClientStartup(telemetryService, { + outcome: 'failure', + durationMs: 20, + attemptNumber: 2, + }, new Error('Unexpected startup failure')); + + assert.deepStrictEqual(telemetryService.events, [{ + eventName: 'agentHost.copilotClientStartup', + data: { + outcome: 'failure', + durationMs: 10, + attemptNumber: 1, + startupFailureCause: 'configurationChanged', + startupFailureResource: 'other', + startupExitCode: undefined, + }, + }, { + eventName: 'agentHost.copilotClientStartup', + data: { + outcome: 'failure', + durationMs: 20, + attemptNumber: 2, + startupFailureCause: 'other', + startupFailureResource: 'other', + }, + }]); }); test('builds the Agent Host and SDK correlation tuple', () => { From 6f50f21b9601a33027360c19e05e115bfb7d64fe Mon Sep 17 00:00:00 2001 From: roblourens Date: Sat, 15 Aug 2026 18:06:29 -0700 Subject: [PATCH 3/3] agentHost: Respect telemetry disablement from process launch (#330929) * agentHost: Respect telemetry disablement during initialization Send each client's effective telemetry level with initialize and reconnect so the host applies consent before connection telemetry or queued actions. Keep the host disabled until a client level arrives, and propagate process-level restrictions to generic remote telemetry.\n\n(Written by Copilot)\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * agentHost: Preserve telemetry wrapper defaults for direct callers Keep fail-closed startup explicit to the production factory while preserving the established constructor behavior used by isolated Agent Host components and tests.\n\n(Written by Copilot)\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * agentHost: Gate telemetry on client consent Keep seeded root configuration from enabling telemetry before initialize or reconnect provides a client telemetry level. Preserve existing direct-construction defaults for isolated callers and tests.\n\n(Written by Copilot)\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * agentHost: Propagate telemetry level at process launch Start controlled Agent Host processes with the launcher's effective telemetry level so opted-in clients retain early diagnostics while opted-out clients disable telemetry before startup. Keep initialize and reconnect updates as a monotonic multi-client clamp across local, remote-server, SSH, WSL, and CLI-supervised hosts.\n\n(Written by Copilot)\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Update generated extension point cache Include the link presentation provider extension point generated by hygiene after merging origin/main.\n\n(Written by Copilot)\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * agentHost: Derive SSH telemetry in shared process Use the shared process telemetry service when launching SSH and WSL Agent Hosts instead of threading the telemetry level through renderer IPC contracts.\n\n(Written by Copilot)\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/src/commands/agent_host.rs | 6 + cli/src/tunnels/agent_host.rs | 8 +- .../browser/remoteAgentHostProtocolClient.ts | 31 +-- .../agentHost/common/agentHostSchema.ts | 12 +- .../agentHost/common/agentHostTelemetry.ts | 39 +++- .../agentHost/common/agentHostTelemetryEnv.ts | 1 + .../electron-main/electronAgentHostStarter.ts | 8 +- .../node/agentHostTelemetryService.ts | 19 +- .../agentHost/node/nodeAgentHostStarter.ts | 8 +- .../agentHost/node/protocolServerHandler.ts | 16 +- .../node/sshRemoteAgentHostHelpers.ts | 21 +- .../node/sshRemoteAgentHostService.ts | 23 ++- .../node/wslRemoteAgentHostHelpers.ts | 9 +- .../node/wslRemoteAgentHostService.ts | 4 + .../remoteAgentHostProtocolClient.test.ts | 52 ++++- .../node/agentHostTelemetryService.test.ts | 68 ++++++- .../test/node/protocolServerHandler.test.ts | 183 +++++++++++++++++- .../test/node/sshHostKeyVerification.test.ts | 3 +- .../node/sshRemoteAgentHostHelpers.test.ts | 40 ++-- .../node/sshRemoteAgentHostService.test.ts | 11 +- .../node/wslRemoteAgentHostHelpers.test.ts | 32 +++ src/vs/platform/environment/common/argv.ts | 1 + src/vs/platform/environment/node/argv.ts | 1 + .../common/serverTelemetryService.ts | 6 +- .../common/serverTelemetryService.test.ts | 39 ++++ .../electron-browser/remote.contribution.ts | 9 +- 26 files changed, 561 insertions(+), 89 deletions(-) create mode 100644 src/vs/platform/telemetry/test/common/serverTelemetryService.test.ts diff --git a/cli/src/commands/agent_host.rs b/cli/src/commands/agent_host.rs index 8377336272b80b..e8722a98869b8a 100644 --- a/cli/src/commands/agent_host.rs +++ b/cli/src/commands/agent_host.rs @@ -15,6 +15,7 @@ use tokio::io::{AsyncBufReadExt, BufReader}; use crate::auth::Auth; use crate::constants::{self, AGENT_HOST_PORT}; use crate::log; +use crate::options::TelemetryLevel; use crate::state::LauncherPaths; use crate::tunnels::agent_host::{ classify_agent_host, serve_agent_host_tunnel_connection, AgentHostConfig, AgentHostManager, @@ -288,6 +289,11 @@ async fn run_supervisor(mut ctx: CommandContext, mut args: AgentHostArgs) -> Res Arc::new(ReqwestSimpleHttp::with_client(ctx.http.clone())), AgentHostConfig { server_data_dir: args.server_data_dir.clone(), + telemetry_level: if ctx.args.global_options.disable_telemetry { + Some(TelemetryLevel::Off) + } else { + ctx.args.global_options.telemetry_level + }, // The AH backend runs on an internal-only unix socket / named // pipe between this supervisor and its child, so we // deliberately disable the backend's token check; this diff --git a/cli/src/tunnels/agent_host.rs b/cli/src/tunnels/agent_host.rs index 74e8b970a9f152..dbd54c1c363203 100644 --- a/cli/src/tunnels/agent_host.rs +++ b/cli/src/tunnels/agent_host.rs @@ -31,7 +31,7 @@ use crate::async_pipe::{ use crate::constants::VSCODE_CLI_QUALITY; use crate::download_cache::DownloadCache; use crate::log; -use crate::options::Quality; +use crate::options::{Quality, TelemetryLevel}; use crate::state::LauncherPaths; use crate::update_service::{ unzip_downloaded_release, Platform, Release, TargetKind, UpdateService, @@ -98,6 +98,7 @@ const UPGRADE_KILL_DELAY: Duration = Duration::from_secs(3); #[derive(Clone, Debug)] pub struct AgentHostConfig { pub server_data_dir: Option, + pub telemetry_level: Option, pub without_connection_token: bool, pub connection_token: Option, pub connection_token_file: Option, @@ -262,6 +263,10 @@ impl AgentHostManager { cmd.arg("--server-data-dir"); cmd.arg(a); } + if let Some(level) = self.config.telemetry_level { + cmd.arg("--telemetry-level"); + cmd.arg(level.to_string()); + } if self.config.without_connection_token { cmd.arg("--without-connection-token"); } @@ -2255,6 +2260,7 @@ mod tests { Arc::new(ReqwestSimpleHttp::new()), AgentHostConfig { server_data_dir: None, + telemetry_level: None, without_connection_token: true, connection_token: None, connection_token_file: None, diff --git a/src/vs/platform/agentHost/browser/remoteAgentHostProtocolClient.ts b/src/vs/platform/agentHost/browser/remoteAgentHostProtocolClient.ts index 43762892b7d4fa..bd841eb310eb30 100644 --- a/src/vs/platform/agentHost/browser/remoteAgentHostProtocolClient.ts +++ b/src/vs/platform/agentHost/browser/remoteAgentHostProtocolClient.ts @@ -37,12 +37,12 @@ import { ChatSourceKind, ContentEncoding, ResourceRequestParams, type Completion import type { InvokeChangesetOperationParams, InvokeChangesetOperationResult } from '../common/state/protocol/channels-changeset/commands.js'; import { encodeBase64 } from '../../../base/common/buffer.js'; import { ILoadEstimator, LoadEstimator } from '../../../base/parts/ipc/common/ipc.net.js'; -import { ITelemetryService, TELEMETRY_CRASH_REPORTER_SETTING_ID, TELEMETRY_OLD_SETTING_ID, TELEMETRY_SETTING_ID, TelemetryLevel, telemetryLevelEnabled } from '../../telemetry/common/telemetry.js'; +import { ITelemetryService, TelemetryLevel, TELEMETRY_CRASH_REPORTER_SETTING_ID, TELEMETRY_OLD_SETTING_ID, TELEMETRY_SETTING_ID } from '../../telemetry/common/telemetry.js'; import { getTelemetryLevel } from '../../telemetry/common/telemetryUtils.js'; import { AgentHostTelemetryLevelConfigKey, AgentHostTerminalAutoApproveEnabledConfigKey, AgentHostTerminalAutoApproveRulesConfigKey, AgentHostDisableRepoInfoTelemetryConfigKey, getAgentHostTerminalAutoApproveRulesConfig, TERMINAL_AUTO_APPROVE_ENABLED_SETTING_ID, TERMINAL_AUTO_APPROVE_SETTING_ID, TERMINAL_IGNORE_DEFAULT_AUTO_APPROVE_RULES_SETTING_ID, DISABLE_REPO_INFO_TELEMETRY_SETTING_ID, telemetryLevelToAgentHostConfigValue } from '../common/agentHostSchema.js'; import { getAgentHostConfigurationSyncEntries, resolveAgentHostConfigurationSyncPatch, resolveAgentHostConfigurationSyncValue } from '../common/agentHostConfigurationSync.js'; import { managedPermissionsConfigurationIds, resolveManagedSettingsPermissions, type IAgentHostManagedSettingsPermissions } from '../common/agentHostManagedSettings.js'; -import { AgentHostClientConnectionKind, toClientTelemetryMeta } from '../common/agentHostTelemetry.js'; +import { AgentHostClientConnectionKind, toAgentHostClientMeta } from '../common/agentHostTelemetry.js'; import type { OtlpExportLogsParams } from '../common/state/protocol/channels-otlp/notifications.js'; import type { TelemetryCapabilities } from '../common/state/protocol/channels-otlp/state.js'; import type { Implementation, InitializeResult } from '../common/state/protocol/common/commands.js'; @@ -455,7 +455,7 @@ export class RemoteAgentHostProtocolClient extends Disposable implements IAgentC protocolVersions: [...SUPPORTED_PROTOCOL_VERSIONS], clientId: this._clientId, clientInfo: this._clientInfo, - ...this._clientConnectionTelemetryMeta(), + _meta: this._clientMeta(), initialSubscriptions: [ROOT_STATE_URI], }, { bypassInitializeQueue: true }); this._applyInitializeResult(result); @@ -705,7 +705,7 @@ export class RemoteAgentHostProtocolClient extends Disposable implements IAgentC clientId: this._clientId, lastSeenServerSeq, subscriptions, - ...this._clientConnectionTelemetryMeta(), + _meta: this._clientMeta(), }, { bypassReconnectGate: true }); return { result, freshInitialize: false }; } catch (error) { @@ -720,7 +720,7 @@ export class RemoteAgentHostProtocolClient extends Disposable implements IAgentC protocolVersions: [...SUPPORTED_PROTOCOL_VERSIONS], clientId: this._clientId, clientInfo: this._clientInfo, - ...this._clientConnectionTelemetryMeta(), + _meta: this._clientMeta(), initialSubscriptions: subscriptions, }, { bypassReconnectGate: true }); this._applyInitializeResult(initializeResult, false); @@ -774,12 +774,15 @@ export class RemoteAgentHostProtocolClient extends Disposable implements IAgentC }, { bypassReconnectGate: true }))); } - private _clientConnectionTelemetryMeta(): { _meta: Record } | Record { - const sendIdentity = telemetryLevelEnabled(this._telemetryService, TelemetryLevel.USAGE); - const machineId = sendIdentity ? this._telemetryService.machineId : undefined; - const devDeviceId = sendIdentity ? this._telemetryService.devDeviceId : undefined; - const meta = toClientTelemetryMeta(this._transport.clientConnectionKind, machineId, devDeviceId); - return meta ? { _meta: meta } : {}; + private _clientMeta(): Record { + const telemetryLevel = this._effectiveTelemetryLevel(); + const sendIdentity = telemetryLevel >= TelemetryLevel.USAGE; + return toAgentHostClientMeta( + this._transport.clientConnectionKind, + telemetryLevel, + sendIdentity ? this._telemetryService.machineId : undefined, + sendIdentity ? this._telemetryService.devDeviceId : undefined, + ); } private _applyInitializeResult(result: CommandMap['initialize']['result'], forwardClientConfig = true): void { @@ -1616,7 +1619,11 @@ export class RemoteAgentHostProtocolClient extends Disposable implements IAgentC } private _updateTelemetryLevel(): void { - this._dispatchRootConfig({ [AgentHostTelemetryLevelConfigKey]: telemetryLevelToAgentHostConfigValue(getTelemetryLevel(this._configurationService)) }); + this._dispatchRootConfig({ [AgentHostTelemetryLevelConfigKey]: telemetryLevelToAgentHostConfigValue(this._effectiveTelemetryLevel()) }); + } + + private _effectiveTelemetryLevel(): TelemetryLevel { + return Math.min(getTelemetryLevel(this._configurationService), this._telemetryService.telemetryLevel); } /** Merge a patch into the agent host's root configuration. */ diff --git a/src/vs/platform/agentHost/common/agentHostSchema.ts b/src/vs/platform/agentHost/common/agentHostSchema.ts index ff6de4a49ac628..f849be88090cfb 100644 --- a/src/vs/platform/agentHost/common/agentHostSchema.ts +++ b/src/vs/platform/agentHost/common/agentHostSchema.ts @@ -9,6 +9,7 @@ import { ConfigurationTarget, type IConfigurationService, type IConfigurationVal import { DEFAULT_EDIT_AUTO_APPROVE_PATTERNS, type ChatEditAutoApprovePatterns } from '../../chat/common/chatSettings.js'; import type { IMcpServerConfiguration } from '../../mcp/common/mcpPlatformTypes.js'; import { TelemetryConfiguration, TelemetryLevel } from '../../telemetry/common/telemetry.js'; +import { telemetryLevelToAgentHostValue } from './agentHostTelemetry.js'; import { SessionConfigKey } from './sessionConfigKeys.js'; import type { SessionConfigPropertySchema, SessionConfigSchema } from './state/protocol/commands.js'; import { JsonRpcErrorCodes, ProtocolError } from './state/sessionProtocol.js'; @@ -593,16 +594,7 @@ export const AgentHostMcpServersConfigKey = 'mcpServers'; export type AgentHostMcpServers = Record; export function telemetryLevelToAgentHostConfigValue(telemetryLevel: TelemetryLevel): TelemetryConfiguration { - switch (telemetryLevel) { - case TelemetryLevel.NONE: - return TelemetryConfiguration.OFF; - case TelemetryLevel.CRASH: - return TelemetryConfiguration.CRASH; - case TelemetryLevel.ERROR: - return TelemetryConfiguration.ERROR; - case TelemetryLevel.USAGE: - return TelemetryConfiguration.ON; - } + return telemetryLevelToAgentHostValue(telemetryLevel); } export function agentHostConfigValueToTelemetryLevel(value: unknown): TelemetryLevel | undefined { diff --git a/src/vs/platform/agentHost/common/agentHostTelemetry.ts b/src/vs/platform/agentHost/common/agentHostTelemetry.ts index 828a1561a12f52..220568eacb5ed3 100644 --- a/src/vs/platform/agentHost/common/agentHostTelemetry.ts +++ b/src/vs/platform/agentHost/common/agentHostTelemetry.ts @@ -3,6 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { TelemetryConfiguration, TelemetryLevel } from '../../telemetry/common/telemetry.js'; import type { AgentHostClientType } from './agentHostClientInfo.js'; export const enum AgentHostLaunchKind { @@ -49,11 +50,14 @@ export function createUnknownAgentHostClientTelemetryContext(clientType: AgentHo } const CLIENT_CONNECTION_KIND_META_KEY = 'vscode.clientConnectionKind'; +const CLIENT_TELEMETRY_LEVEL_META_KEY = 'vscode.telemetryLevel'; const CLIENT_MACHINE_ID_META_KEY = 'vscode.clientMachineId'; const CLIENT_DEV_DEVICE_ID_META_KEY = 'vscode.clientDevDeviceId'; -export function toClientTelemetryMeta(connectionKind: AgentHostClientConnectionKind | undefined, machineId: string | undefined, devDeviceId: string | undefined): Record | undefined { - const meta: Record = {}; +export function toAgentHostClientMeta(connectionKind: AgentHostClientConnectionKind | undefined, telemetryLevel: TelemetryLevel, machineId: string | undefined, devDeviceId: string | undefined): Record { + const meta: Record = { + [CLIENT_TELEMETRY_LEVEL_META_KEY]: telemetryLevelToAgentHostValue(telemetryLevel), + }; if (connectionKind !== undefined && connectionKind !== AgentHostClientConnectionKind.Unknown) { meta[CLIENT_CONNECTION_KIND_META_KEY] = connectionKind; } @@ -63,7 +67,7 @@ export function toClientTelemetryMeta(connectionKind: AgentHostClientConnectionK if (devDeviceId) { meta[CLIENT_DEV_DEVICE_ID_META_KEY] = devDeviceId; } - return Object.keys(meta).length > 0 ? meta : undefined; + return meta; } export function readClientConnectionKind(meta: Record | undefined): AgentHostClientConnectionKind { @@ -82,6 +86,35 @@ export function readClientConnectionKind(meta: Record | undefin } } +export function readClientTelemetryLevel(meta: Record | undefined): TelemetryLevel | undefined { + const value = meta?.[CLIENT_TELEMETRY_LEVEL_META_KEY]; + switch (value) { + case TelemetryConfiguration.OFF: + return TelemetryLevel.NONE; + case TelemetryConfiguration.CRASH: + return TelemetryLevel.CRASH; + case TelemetryConfiguration.ERROR: + return TelemetryLevel.ERROR; + case TelemetryConfiguration.ON: + return TelemetryLevel.USAGE; + default: + return value === undefined ? undefined : TelemetryLevel.NONE; + } +} + +export function telemetryLevelToAgentHostValue(telemetryLevel: TelemetryLevel): TelemetryConfiguration { + switch (telemetryLevel) { + case TelemetryLevel.NONE: + return TelemetryConfiguration.OFF; + case TelemetryLevel.CRASH: + return TelemetryConfiguration.CRASH; + case TelemetryLevel.ERROR: + return TelemetryConfiguration.ERROR; + case TelemetryLevel.USAGE: + return TelemetryConfiguration.ON; + } +} + export function readClientMachineId(meta: Record | undefined): string | undefined { return readClientTelemetryIdentity(meta, CLIENT_MACHINE_ID_META_KEY); } diff --git a/src/vs/platform/agentHost/common/agentHostTelemetryEnv.ts b/src/vs/platform/agentHost/common/agentHostTelemetryEnv.ts index d55d5e33af613c..705f17038336c8 100644 --- a/src/vs/platform/agentHost/common/agentHostTelemetryEnv.ts +++ b/src/vs/platform/agentHost/common/agentHostTelemetryEnv.ts @@ -21,6 +21,7 @@ export const AgentHostMachineIdEnvKey = 'VSCODE_AGENT_HOST_MACHINE_ID'; export const AgentHostSqmIdEnvKey = 'VSCODE_AGENT_HOST_SQM_ID'; export const AgentHostDevDeviceIdEnvKey = 'VSCODE_AGENT_HOST_DEV_DEVICE_ID'; +export const AgentHostTelemetryLevelEnvKey = 'VSCODE_AGENT_HOST_TELEMETRY_LEVEL'; export interface IAgentHostForwardedTelemetryIds { readonly machineId: string; diff --git a/src/vs/platform/agentHost/electron-main/electronAgentHostStarter.ts b/src/vs/platform/agentHost/electron-main/electronAgentHostStarter.ts index 65196556c5f6ed..0b6f6dece3e0b5 100644 --- a/src/vs/platform/agentHost/electron-main/electronAgentHostStarter.ts +++ b/src/vs/platform/agentHost/electron-main/electronAgentHostStarter.ts @@ -17,11 +17,12 @@ import { ILifecycleMainService } from '../../lifecycle/electron-main/lifecycleMa import { ILogService } from '../../log/common/log.js'; import { Schemas } from '../../../base/common/network.js'; import { getResolvedShellEnv } from '../../shell/node/shellEnv.js'; +import { ITelemetryService } from '../../telemetry/common/telemetry.js'; import { NullTelemetryService } from '../../telemetry/common/telemetryUtils.js'; import { UtilityProcess } from '../../utilityProcess/electron-main/utilityProcess.js'; import { AgentHostStartError, IAgentHostConnection, IAgentHostShutdownRequest, IAgentHostStarter, IAgentHostStartRequest } from '../common/agent.js'; import { buildAgentHostTelemetryIdEnv, IAgentHostForwardedTelemetryIds } from '../common/agentHostTelemetryEnv.js'; -import { AgentHostLaunchKind, AgentHostLaunchKindEnvVar } from '../common/agentHostTelemetry.js'; +import { AgentHostLaunchKind, AgentHostLaunchKindEnvVar, telemetryLevelToAgentHostValue } from '../common/agentHostTelemetry.js'; import { AgentHostByokModelsEnabledSettingId, AgentHostClaudeAgentEnabledSettingId, AgentHostCodexAgentBinaryArgsSettingId, AgentHostCodexAgentEnabledSettingId, AgentHostCodexAgentSdkRootSettingId, AgentHostCodexAgentCodexHomeSettingId, AgentHostIpcChannels, AgentHostOTelCaptureContentSettingId, AgentHostOTelDbSpanExporterEnabledSettingId, AgentHostOTelEnabledSettingId, AgentHostOTelExporterTypeSettingId, AgentHostOTelOtlpEndpointSettingId, AgentHostOTelOtlpProtocolSettingId, AgentHostOTelOutfileSettingId, AgentHostOTelResourceAttributesSettingId, AgentHostOTelServiceNameSettingId, AgentHostOTelPolicyIpcChannel, AgentHostRestartIpcChannel, AgentHostWillRestartIpcChannel, buildAgentHostOTelEnv, buildAgentSdkEnv, IAgentHostManagementService, IAgentHostOTelSettings, sanitizeAgentHostOTelPolicySettings } from '../common/agentService.js'; import { deepClone } from '../../../base/common/objects.js'; import '../common/agentHostStarter.config.contribution.js'; @@ -54,6 +55,7 @@ export class ElectronAgentHostStarter extends Disposable implements IAgentHostSt @IEnvironmentMainService private readonly _environmentMainService: IEnvironmentMainService, @ILifecycleMainService private readonly _lifecycleMainService: ILifecycleMainService, @ILogService private readonly _logService: ILogService, + @ITelemetryService private readonly _telemetryService: ITelemetryService, ) { super(); @@ -158,10 +160,8 @@ export class ElectronAgentHostStarter extends Disposable implements IAgentHostSt const args = [ '--logsPath', this._environmentMainService.logsHome.with({ scheme: Schemas.file }).fsPath, '--user-data-dir', this._environmentMainService.userDataPath, + '--telemetry-level', telemetryLevelToAgentHostValue(this._telemetryService.telemetryLevel), ]; - if (this._environmentMainService.disableTelemetry) { - args.push('--disable-telemetry'); - } // Forward the host's resolved telemetry identifiers so the agent host // reuses the same persisted machineId/sqmId/devDeviceId instead of diff --git a/src/vs/platform/agentHost/node/agentHostTelemetryService.ts b/src/vs/platform/agentHost/node/agentHostTelemetryService.ts index 73d5af4f44902a..0614f8686f9776 100644 --- a/src/vs/platform/agentHost/node/agentHostTelemetryService.ts +++ b/src/vs/platform/agentHost/node/agentHostTelemetryService.ts @@ -23,7 +23,7 @@ import { TelemetryLogAppender } from '../../telemetry/common/telemetryLogAppende import { TelemetryService } from '../../telemetry/common/telemetryService.js'; import { getPiiPathsFromEnvironment, isInternalTelemetry, isLoggingOnly, NullTelemetryService, supportsTelemetry, type ITelemetryAppender } from '../../telemetry/common/telemetryUtils.js'; import { AgentHostTelemetryLevelConfigKey, agentHostConfigValueToTelemetryLevel } from '../common/agentHostSchema.js'; -import { AgentHostDevDeviceIdEnvKey, AgentHostMachineIdEnvKey, AgentHostSqmIdEnvKey } from '../common/agentHostTelemetryEnv.js'; +import { AgentHostDevDeviceIdEnvKey, AgentHostMachineIdEnvKey, AgentHostSqmIdEnvKey, AgentHostTelemetryLevelEnvKey } from '../common/agentHostTelemetryEnv.js'; import { AgentHostRestrictedTelemetrySender, IAgentHostRestrictedTelemetry, IAgentHostInternalTelemetryContext, IAgentHostRestrictedTelemetryContext, TelemetryMeasurements, TelemetryProps } from './agentHostRestrictedTelemetry.js'; import { AgentHostInternalTelemetrySender } from './agentHostMicrosoftTelemetry.js'; @@ -37,6 +37,7 @@ export interface IAgentHostTelemetryServiceOptions { readonly disableTelemetry?: boolean; readonly fetchFn?: typeof globalThis.fetch; readonly requestService?: IRequestService; + readonly readTelemetryLevelEnvironment?: () => string | undefined; } export interface IAgentHostTelemetryService extends ITelemetryService, IAgentHostRestrictedTelemetry { @@ -46,7 +47,7 @@ export interface IAgentHostTelemetryService extends ITelemetryService, IAgentHos export class AgentHostTelemetryService extends Disposable implements IAgentHostTelemetryService { declare readonly _serviceBrand: undefined; - private _telemetryLevel = TelemetryLevel.USAGE; + private _telemetryLevel: TelemetryLevel; /** * Whether the current Copilot token opts into enhanced/restricted telemetry (`rt=1`). Defaults @@ -61,8 +62,10 @@ export class AgentHostTelemetryService extends Disposable implements IAgentHostT private readonly _restricted?: IAgentHostRestrictedTelemetry, copilotSdkVersion?: string, copilotRuntimeVersion?: string, + initialTelemetryLevel: TelemetryLevel = TelemetryLevel.USAGE, ) { super(); + this._telemetryLevel = initialTelemetryLevel; if (isDisposable(_delegate)) { this._register(_delegate); } @@ -274,5 +277,15 @@ export async function createAgentHostTelemetryService(options: IAgentHostTelemet const internalSender = loggingOnly ? undefined : disposables.add(new AgentHostInternalTelemetrySender({ requestService: options.requestService, commonProperties, extensionVersion })); const restricted = loggingOnly ? undefined : new AgentHostRestrictedTelemetrySender(commonProperties, logService, undefined, internalSender, options.fetchFn); - return disposables.add(new AgentHostTelemetryService(telemetryService, restricted, productService.copilotVersions?.sdk, productService.copilotVersions?.runtime)); + const initialTelemetryLevel = Math.min( + parseLaunchTelemetryLevel(environmentService.args?.['telemetry-level']), + parseLaunchTelemetryLevel((options.readTelemetryLevelEnvironment ?? (() => process.env[AgentHostTelemetryLevelEnvKey]))()), + ); + return disposables.add(new AgentHostTelemetryService(telemetryService, restricted, productService.copilotVersions?.sdk, productService.copilotVersions?.runtime, initialTelemetryLevel)); +} + +function parseLaunchTelemetryLevel(value: string | undefined): TelemetryLevel { + return value === undefined + ? TelemetryLevel.USAGE + : agentHostConfigValueToTelemetryLevel(value) ?? TelemetryLevel.NONE; } diff --git a/src/vs/platform/agentHost/node/nodeAgentHostStarter.ts b/src/vs/platform/agentHost/node/nodeAgentHostStarter.ts index e0a97d122af7b2..e1db01135344aa 100644 --- a/src/vs/platform/agentHost/node/nodeAgentHostStarter.ts +++ b/src/vs/platform/agentHost/node/nodeAgentHostStarter.ts @@ -14,8 +14,9 @@ import { IEnvironmentService, INativeEnvironmentService } from '../../environmen import { parseAgentHostDebugPort } from '../../environment/node/environmentService.js'; import { ILogService } from '../../log/common/log.js'; import { getResolvedShellEnv } from '../../shell/node/shellEnv.js'; +import { ITelemetryService } from '../../telemetry/common/telemetry.js'; import { IAgentHostConnection, IAgentHostStarter } from '../common/agent.js'; -import { AgentHostLaunchKind, AgentHostLaunchKindEnvVar } from '../common/agentHostTelemetry.js'; +import { AgentHostLaunchKind, AgentHostLaunchKindEnvVar, telemetryLevelToAgentHostValue } from '../common/agentHostTelemetry.js'; import { AgentHostByokModelsEnabledSettingId, AgentHostClaudeAgentEnabledSettingId, AgentHostCodexAgentBinaryArgsSettingId, AgentHostCodexAgentEnabledSettingId, AgentHostCodexAgentSdkRootSettingId, AgentHostCodexAgentCodexHomeSettingId, AgentHostIpcChannels, AgentHostOTelCaptureContentSettingId, AgentHostOTelDbSpanExporterEnabledSettingId, AgentHostOTelEnabledSettingId, AgentHostOTelExporterTypeSettingId, AgentHostOTelOtlpEndpointSettingId, AgentHostOTelOtlpProtocolSettingId, AgentHostOTelOutfileSettingId, AgentHostOTelResourceAttributesSettingId, AgentHostOTelServiceNameSettingId, buildAgentHostOTelEnv, buildAgentSdkEnv, IAgentHostManagementService } from '../common/agentService.js'; import '../common/agentHostStarter.config.contribution.js'; @@ -46,6 +47,7 @@ export class NodeAgentHostStarter extends Disposable implements IAgentHostStarte @IConfigurationService private readonly _configurationService: IConfigurationService, @IEnvironmentService private readonly _environmentService: INativeEnvironmentService, @ILogService private readonly _logService: ILogService, + @ITelemetryService private readonly _telemetryService: ITelemetryService, ) { super(); } @@ -132,10 +134,8 @@ export class NodeAgentHostStarter extends Disposable implements IAgentHostStarte '--type=agentHost', '--logsPath', this._environmentService.logsHome.with({ scheme: Schemas.file }).fsPath, '--user-data-dir', this._environmentService.userDataPath, + '--telemetry-level', telemetryLevelToAgentHostValue(this._telemetryService.telemetryLevel), ]; - if (this._environmentService.disableTelemetry) { - args.push('--disable-telemetry'); - } const opts: IIPCOptions = { serverName: 'Agent Host', diff --git a/src/vs/platform/agentHost/node/protocolServerHandler.ts b/src/vs/platform/agentHost/node/protocolServerHandler.ts index 79c8593079af0c..baecb6a34881fa 100644 --- a/src/vs/platform/agentHost/node/protocolServerHandler.ts +++ b/src/vs/platform/agentHost/node/protocolServerHandler.ts @@ -15,7 +15,7 @@ import { ILogService } from '../../log/common/log.js'; import { ITelemetryService } from '../../telemetry/common/telemetry.js'; import { AHPFileSystemProvider } from '../common/agentHostFileSystemProvider.js'; import { getAgentHostClientType } from '../common/agentHostClientInfo.js'; -import { AgentHostClientConnectionKind, AgentHostLaunchKind, AgentHostTransportKind, readClientConnectionKind, readClientDevDeviceId, readClientMachineId, type IAgentHostClientTelemetryContext } from '../common/agentHostTelemetry.js'; +import { AgentHostClientConnectionKind, AgentHostLaunchKind, AgentHostTransportKind, readClientConnectionKind, readClientDevDeviceId, readClientMachineId, readClientTelemetryLevel, type IAgentHostClientTelemetryContext } from '../common/agentHostTelemetry.js'; import { AgentSession, type IAgentCreateChatOptions, type IMcpNotification } from '../common/agent.js'; import { isManagedSettingsPermissions } from '../common/agentHostManagedSettings.js'; import { type IAgentService } from '../common/agentService.js'; @@ -66,6 +66,7 @@ import { isFileResourceRead } from '../common/resourceReadLogging.js'; import type { Implementation } from '../common/state/protocol/common/commands.js'; import { AGENT_HOST_CLIENT_CONNECTION_HISTORY_RETENTION, AgentHostClientConnectionTelemetryTracker } from './agentHostClientConnectionTelemetry.js'; import { AgentHostTelemetryReporter } from './agentHostTelemetryReporter.js'; +import { isAgentHostTelemetryService } from './agentHostTelemetryService.js'; /** Default capacity of the server-side action replay buffer. */ const REPLAY_BUFFER_CAPACITY = 1000; @@ -371,11 +372,11 @@ export class ProtocolServerHandler extends Disposable { private readonly _config: IProtocolServerConfig, private readonly _clientFileSystemProvider: AHPFileSystemProvider, @ILogService private readonly _logService: ILogService, - @ITelemetryService telemetryService: ITelemetryService, + @ITelemetryService private readonly _telemetryService: ITelemetryService, @IAgentHostManagedSettingsService private readonly _managedSettingsService: IAgentHostManagedSettingsService, ) { super(); - this._telemetryReporter = new AgentHostTelemetryReporter(telemetryService); + this._telemetryReporter = new AgentHostTelemetryReporter(this._telemetryService); this._connectionTelemetryTracker = this._config.connectionTelemetryTracker ?? this._register(new AgentHostClientConnectionTelemetryTracker()); this._register(this._server.onConnection(transport => { @@ -596,6 +597,7 @@ export class ProtocolServerHandler extends Disposable { } const previousRecord = this._clients.get(params.clientId); + this._applyClientTelemetryLevel(params._meta); const telemetryTransportToken = {}; const initializationDisposables = disposables.add(new DisposableStore()); const telemetryContext = this._createClientTelemetryContext(params.clientInfo, params._meta, transport); @@ -736,6 +738,7 @@ export class ProtocolServerHandler extends Disposable { if (!existingRecord) { throw new ProtocolError(AhpErrorCodes.NotFound, `Reconnect client not found: ${params.clientId}`); } + this._applyClientTelemetryLevel(params._meta); // Synchronously install the client so messages arriving on this transport // while we restore subscriptions can find a valid client object. The @@ -1194,6 +1197,13 @@ export class ProtocolServerHandler extends Disposable { }; } + private _applyClientTelemetryLevel(meta: Record | undefined): void { + const telemetryLevel = readClientTelemetryLevel(meta); + if (telemetryLevel !== undefined && isAgentHostTelemetryService(this._telemetryService)) { + this._telemetryService.updateTelemetryLevel(telemetryLevel); + } + } + private _reportClientDisconnected(client: IConnectedClient, subscriptionCount: number): void { if (!client.telemetryConnectionActive) { return; diff --git a/src/vs/platform/agentHost/node/sshRemoteAgentHostHelpers.ts b/src/vs/platform/agentHost/node/sshRemoteAgentHostHelpers.ts index e4d814c25ecfd8..0cb0a6b1dd3911 100644 --- a/src/vs/platform/agentHost/node/sshRemoteAgentHostHelpers.ts +++ b/src/vs/platform/agentHost/node/sshRemoteAgentHostHelpers.ts @@ -6,6 +6,7 @@ import { timeout } from '../../../base/common/async.js'; import { CancellationToken } from '../../../base/common/cancellation.js'; import { vArray, vObj, vString, vUnknown } from '../../../base/common/validation.js'; +import { TelemetryConfiguration } from '../../telemetry/common/telemetry.js'; import { getAgentHostEndpointIdentityKey, IAgentHostEndpointMetadata, parseAgentHostEndpointRegistry } from '../common/agentHostEndpointRegistry.js'; /** @@ -21,6 +22,18 @@ export function validateShellToken(value: string, label: string): string { return value; } +export function validateAgentHostTelemetryLevel(value: unknown): TelemetryConfiguration { + switch (value) { + case TelemetryConfiguration.OFF: + case TelemetryConfiguration.CRASH: + case TelemetryConfiguration.ERROR: + case TelemetryConfiguration.ON: + return value; + default: + throw new Error(`Unsafe telemetry level for shell interpolation: ${JSON.stringify(value)}`); + } +} + /** * Validate and normalize a commit SHA. Returns the lowercase form. * @@ -127,8 +140,8 @@ export function shellEscape(s: string): string { * build them via {@link getRemoteCLIBin} / {@link getRemoteCLIDataDir} * which validate their components. */ -export function buildAgentHostBaseCommand(cliBin: string, cliDataDir: string): string { - return `${cliBin} --cli-data-dir ${cliDataDir} agent host --port 0`; +export function buildAgentHostBaseCommand(cliBin: string, cliDataDir: string, telemetryLevel: TelemetryConfiguration): string { + return `${cliBin} --cli-data-dir ${cliDataDir} --telemetry-level ${validateAgentHostTelemetryLevel(telemetryLevel)} agent host --port 0`; } export function resolveRemotePlatform(unameS: string, unameM: string): { os: string; arch: string } | undefined { @@ -345,11 +358,11 @@ export function buildAgentEndpointsCommand(cliBin: string, cliDataDir: string, u * genuinely new process/registry entry every time this command runs, * leaving all existing standalone/editor entries untouched. */ -export function buildAgentHostSpawnCommand(cliBin: string, cliDataDir: string, userDataPath: string, idleTimeoutSec = 300): string { +export function buildAgentHostSpawnCommand(cliBin: string, cliDataDir: string, userDataPath: string, telemetryLevel: TelemetryConfiguration, idleTimeoutSec = 300): string { if (!Number.isSafeInteger(idleTimeoutSec) || idleTimeoutSec <= 0) { throw new Error(`Unsafe idle timeout value for shell interpolation: ${JSON.stringify(idleTimeoutSec)}`); } - return `${buildAgentHostBaseCommand(cliBin, cliDataDir)} --new-instance --user-data-dir ${shellEscape(userDataPath)} --idle-timeout ${idleTimeoutSec}`; + return `${buildAgentHostBaseCommand(cliBin, cliDataDir, telemetryLevel)} --new-instance --user-data-dir ${shellEscape(userDataPath)} --idle-timeout ${idleTimeoutSec}`; } /** diff --git a/src/vs/platform/agentHost/node/sshRemoteAgentHostService.ts b/src/vs/platform/agentHost/node/sshRemoteAgentHostService.ts index 6b85586cb3eb24..6442994a2e6e6e 100644 --- a/src/vs/platform/agentHost/node/sshRemoteAgentHostService.ts +++ b/src/vs/platform/agentHost/node/sshRemoteAgentHostService.ts @@ -17,6 +17,7 @@ import { URI } from '../../../base/common/uri.js'; import { localize } from '../../../nls.js'; import { ILogService } from '../../log/common/log.js'; import { IProductService } from '../../product/common/productService.js'; +import { ITelemetryService, TelemetryConfiguration } from '../../telemetry/common/telemetry.js'; import { ISSHRemoteAgentHostMainService, SSHAuthMethod, @@ -46,6 +47,8 @@ import { } from './sshKnownHosts.js'; import type { RemoteAgentHostLocationPreference } from '../common/remoteAgentHostLocationPreference.js'; import type { IRelayMessage } from '../common/relayTransport.js'; +import { AgentHostTelemetryLevelEnvKey } from '../common/agentHostTelemetryEnv.js'; +import { telemetryLevelToAgentHostValue } from '../common/agentHostTelemetry.js'; import { type AgentHostEndpointAddress, type AgentHostServerType, @@ -69,6 +72,7 @@ import { resolveRemotePlatform, runAgentEndpoints, shellEscape, + validateAgentHostTelemetryLevel, waitForNewStandaloneEndpoint, } from './sshRemoteAgentHostHelpers.js'; import { parseSSHConfigHostEntries, parseSSHGOutput, stripSSHComment } from '../common/sshConfigParsing.js'; @@ -372,18 +376,20 @@ function startRemoteAgentHost( cliBin: string | undefined, cliDataDir: string | undefined, commandOverride?: string, + telemetryLevel = TelemetryConfiguration.OFF, ): Promise<{ port: number; connectionToken: string | undefined; pid: number | undefined; stream: SSHChannel }> { return new Promise((resolve, reject) => { if (!commandOverride && (!cliBin || !cliDataDir)) { reject(new Error(`${LOG_PREFIX} startRemoteAgentHost requires either a cliBin+cliDataDir pair or a commandOverride`)); return; } - const baseCmd = commandOverride ?? buildAgentHostBaseCommand(cliBin!, cliDataDir!); + const validatedTelemetryLevel = validateAgentHostTelemetryLevel(telemetryLevel); + const baseCmd = commandOverride ?? buildAgentHostBaseCommand(cliBin!, cliDataDir!, validatedTelemetryLevel); // Wrap in a login shell so the agent host process inherits the // user's PATH and environment from ~/.bash_profile / ~/.bashrc // (ssh2 exec runs a non-interactive non-login shell by default). // Echo the PID so we can record it for process reuse detection. - const cmd = `bash -l -c ${shellEscape(`echo VSCODE_PID=$$ && exec ${baseCmd}`)}`; + const cmd = `bash -l -c ${shellEscape(`echo VSCODE_PID=$$ && export ${AgentHostTelemetryLevelEnvKey}=${validatedTelemetryLevel} && exec ${baseCmd}`)}`; logService.info(`${LOG_PREFIX} Starting remote agent host: ${cmd}`); client.exec(cmd, (err: Error | undefined, stream: SSHChannel) => { @@ -768,6 +774,7 @@ export class SSHRemoteAgentHostMainService extends Disposable implements ISSHRem constructor( @ILogService private readonly _logService: ILogService, @IProductService private readonly _productService: IProductService, + @ITelemetryService private readonly _telemetryService: ITelemetryService, ) { super(); } @@ -917,7 +924,7 @@ export class SSHRemoteAgentHostMainService extends Disposable implements ISSHRem // picker over. Always start a fresh process (requirement 6). this._logService.info(`${LOG_PREFIX} Using custom agent host command: ${config.remoteAgentHostCommand}; skipping endpoint discovery/selection`); reportProgress(localize('sshProgressStartingAgent', "Starting remote agent host...")); - const result = await this._startRemoteAgentHost(sshClient, undefined, undefined, config.remoteAgentHostCommand); + const result = await this._startRemoteAgentHost(sshClient, undefined, undefined, config.remoteAgentHostCommand, this._effectiveTelemetryLevel); endpoint = { type: 'tcp', host: '127.0.0.1', port: result.port }; connectionToken = result.connectionToken; agentStream = result.stream; @@ -948,7 +955,7 @@ export class SSHRemoteAgentHostMainService extends Disposable implements ISSHRem const standalones = live.filter(e => e.type === 'standalone'); const spawnDedicated = async (): Promise => { - const spawnCommand = buildAgentHostSpawnCommand(cliBin, cliDataDir, userDataPath); + const spawnCommand = buildAgentHostSpawnCommand(cliBin, cliDataDir, userDataPath, this._effectiveTelemetryLevel); reportProgress(localize('sshProgressStartingAgent', "Starting remote agent host...")); this._logService.info(`${LOG_PREFIX} Spawning dedicated standalone agent host: ${spawnCommand}`); // Fire-and-forget: the spawned process is self-managed via @@ -2032,10 +2039,14 @@ export class SSHRemoteAgentHostMainService extends Disposable implements ISSHRem return this._productService.commit; } + private get _effectiveTelemetryLevel(): TelemetryConfiguration { + return telemetryLevelToAgentHostValue(this._telemetryService.telemetryLevel); + } + protected _startRemoteAgentHost( - client: SSHClient, cliBin: string | undefined, cliDataDir: string | undefined, commandOverride?: string, + client: SSHClient, cliBin: string | undefined, cliDataDir: string | undefined, commandOverride?: string, telemetryLevel?: TelemetryConfiguration, ): Promise<{ port: number; connectionToken: string | undefined; pid: number | undefined; stream: SSHChannel }> { - return startRemoteAgentHost(client, this._logService, cliBin, cliDataDir, commandOverride); + return startRemoteAgentHost(client, this._logService, cliBin, cliDataDir, commandOverride, telemetryLevel); } protected async _createWebSocketRelay( diff --git a/src/vs/platform/agentHost/node/wslRemoteAgentHostHelpers.ts b/src/vs/platform/agentHost/node/wslRemoteAgentHostHelpers.ts index 861e70b89558dc..c63d24300fdf7c 100644 --- a/src/vs/platform/agentHost/node/wslRemoteAgentHostHelpers.ts +++ b/src/vs/platform/agentHost/node/wslRemoteAgentHostHelpers.ts @@ -5,6 +5,8 @@ import * as cp from 'child_process'; import { join } from '../../../base/common/path.js'; +import { TelemetryConfiguration } from '../../telemetry/common/telemetry.js'; +import { AgentHostTelemetryLevelEnvKey } from '../common/agentHostTelemetryEnv.js'; import type { IWSLDistro } from '../common/wslRemoteAgentHost.js'; import { buildAgentHostBaseCommand, @@ -15,6 +17,7 @@ import { getRemoteCLIDataDir, getRemoteCLIInstallRoot, shellEscape, + validateAgentHostTelemetryLevel, validateShellToken, } from './sshRemoteAgentHostHelpers.js'; @@ -250,6 +253,7 @@ export interface IComposeAgentHostBootstrapScriptArgs { readonly commit: string | undefined; readonly os: string; readonly arch: string; + readonly telemetryLevel?: TelemetryConfiguration; /** Dev override; when set, returned verbatim and all CLI bootstrap is skipped. */ readonly remoteAgentHostCommand?: string; } @@ -270,14 +274,15 @@ export interface IComposeAgentHostBootstrapScriptArgs { * lives in the helper functions above, not in the composition itself. */ export function composeAgentHostBootstrapScript(args: IComposeAgentHostBootstrapScriptArgs): string { + const telemetryLevel = validateAgentHostTelemetryLevel(args.telemetryLevel ?? TelemetryConfiguration.OFF); if (args.remoteAgentHostCommand) { - return args.remoteAgentHostCommand; + return `export ${AgentHostTelemetryLevelEnvKey}=${telemetryLevel} && ${args.remoteAgentHostCommand}`; } const installRoot = getRemoteCLIInstallRoot(args.serverDataFolderName); const cliBin = getRemoteCLIBin(args.serverDataFolderName, args.quality, args.commit); const cliDataDir = getRemoteCLIDataDir(args.serverDataFolderName); const url = buildCLIDownloadUrl(args.os, args.arch, args.quality, args.commit); - const launch = `exec ${buildAgentHostBaseCommand(cliBin, cliDataDir)}`; + const launch = `exec ${buildAgentHostBaseCommand(cliBin, cliDataDir, telemetryLevel)}`; if (args.commit) { // Pinned-install path. Mirrors SSH's _ensureCLIInstalledPinned: stage diff --git a/src/vs/platform/agentHost/node/wslRemoteAgentHostService.ts b/src/vs/platform/agentHost/node/wslRemoteAgentHostService.ts index 32f01db58075a9..0f6d052bd2d3b5 100644 --- a/src/vs/platform/agentHost/node/wslRemoteAgentHostService.ts +++ b/src/vs/platform/agentHost/node/wslRemoteAgentHostService.ts @@ -12,6 +12,8 @@ import { generateUuid } from '../../../base/common/uuid.js'; import { localize } from '../../../nls.js'; import { ILogService } from '../../log/common/log.js'; import { IProductService } from '../../product/common/productService.js'; +import { ITelemetryService } from '../../telemetry/common/telemetry.js'; +import { telemetryLevelToAgentHostValue } from '../common/agentHostTelemetry.js'; import type { IRelayMessage } from '../common/relayTransport.js'; import { IWSLRemoteAgentHostMainService, @@ -80,6 +82,7 @@ export class WSLRemoteAgentHostMainService extends Disposable implements IWSLRem constructor( @ILogService private readonly _logService: ILogService, @IProductService private readonly _productService: IProductService, + @ITelemetryService private readonly _telemetryService: ITelemetryService, ) { super(); this._register(toDisposable(() => { @@ -193,6 +196,7 @@ export class WSLRemoteAgentHostMainService extends Disposable implements IWSLRem commit: this._commit, os: targetOs, arch: targetArch, + telemetryLevel: telemetryLevelToAgentHostValue(this._telemetryService.telemetryLevel), remoteAgentHostCommand: config.remoteAgentHostCommand, }); diff --git a/src/vs/platform/agentHost/test/electron-browser/remoteAgentHostProtocolClient.test.ts b/src/vs/platform/agentHost/test/electron-browser/remoteAgentHostProtocolClient.test.ts index 5d5b1de81870c4..faf4bc7dbfd81b 100644 --- a/src/vs/platform/agentHost/test/electron-browser/remoteAgentHostProtocolClient.test.ts +++ b/src/vs/platform/agentHost/test/electron-browser/remoteAgentHostProtocolClient.test.ts @@ -63,7 +63,7 @@ const syncTestConfigurationNode = { }, }; import type { Implementation } from '../../common/state/protocol/common/commands.js'; -import { agentsWindowAgentHostClientInfo } from '../../common/agentHostClientInfo.js'; +import { agentsWindowAgentHostClientInfo, editorWindowAgentHostClientInfo } from '../../common/agentHostClientInfo.js'; import { AgentHostClientConnectionKind } from '../../common/agentHostTelemetry.js'; type ProtocolTransportMessage = ProtocolMessage | AhpServerNotification | JsonRpcNotification | JsonRpcResponse | JsonRpcRequest; @@ -318,6 +318,7 @@ suite('RemoteAgentHostProtocolClient', () => { assert.deepStrictEqual((initialize.params as { _meta?: Record })._meta, { 'vscode.clientConnectionKind': AgentHostClientConnectionKind.RemoteExtensionHost, + 'vscode.telemetryLevel': 'all', 'vscode.clientMachineId': 'client-machine-id', 'vscode.clientDevDeviceId': 'client-dev-device-id', }); @@ -333,7 +334,9 @@ suite('RemoteAgentHostProtocolClient', () => { const noTelemetryClient = createClient(noTelemetryTransport).client; const noTelemetryConnectPromise = noTelemetryClient.connect(); const noTelemetryInitialize = noTelemetryTransport.sentMessages[0] as JsonRpcRequest; - assert.strictEqual((noTelemetryInitialize.params as { _meta?: Record })._meta, undefined); + assert.deepStrictEqual((noTelemetryInitialize.params as { _meta?: Record })._meta, { + 'vscode.telemetryLevel': 'off', + }); noTelemetryTransport.fireMessage({ jsonrpc: '2.0', id: noTelemetryInitialize.id, @@ -966,7 +969,7 @@ suite('RemoteAgentHostProtocolClient', () => { test('initialize handshake includes protocol version and client info', async () => { const transport = disposables.add(new TestClientProtocolTransport(AgentHostClientConnectionKind.DevTunnel)); const clientInfo = agentsWindowAgentHostClientInfo; - const { client } = createClient(transport, undefined, undefined, undefined, undefined, 'renderer-client-id', clientInfo); + const { client } = createClientForIdentity('test.example:1234', transport, createPermissionService(), undefined, new NullLogService(), new TestConfigurationService(), 'renderer-client-id', clientInfo, new TestClientIdentityTelemetryService()); const connectPromise = client.connect(); transport.connectDeferred.complete(); @@ -990,7 +993,12 @@ suite('RemoteAgentHostProtocolClient', () => { protocolVersions: [...SUPPORTED_PROTOCOL_VERSIONS], clientId: 'renderer-client-id', clientInfo, - _meta: { 'vscode.clientConnectionKind': 'dev_tunnel' }, + _meta: { + 'vscode.clientConnectionKind': 'dev_tunnel', + 'vscode.telemetryLevel': 'all', + 'vscode.clientMachineId': 'client-machine-id', + 'vscode.clientDevDeviceId': 'client-dev-device-id', + }, }); assert.strictEqual(params.protocolVersions[0], PROTOCOL_VERSION); @@ -1030,6 +1038,40 @@ suite('RemoteAgentHostProtocolClient', () => { }); }); + test('forwards the actual telemetry service restriction during initialization and config sync', async () => { + const transport = disposables.add(new TestProtocolTransport(AgentHostClientConnectionKind.RemoteExtensionHost)); + const configurationService = new TestConfigurationService(); + const client = disposables.add(new RemoteAgentHostProtocolClient( + 'test.example:1234', + transport, + undefined, + 'telemetry-disabled-client', + editorWindowAgentHostClientInfo, + new NullLogService(), + createPermissionService(), + configurationService, + NullTelemetryService, + )); + + const connectPromise = client.connect(); + const initialize = transport.sentMessages[0] as JsonRpcRequest; + assert.deepStrictEqual((initialize.params as { _meta?: Record })._meta, { + 'vscode.clientConnectionKind': AgentHostClientConnectionKind.RemoteExtensionHost, + 'vscode.telemetryLevel': 'off', + }); + transport.fireMessage({ + jsonrpc: '2.0', + id: initialize.id, + result: { protocolVersion: PROTOCOL_VERSION, serverSeq: 0, snapshots: [] }, + }); + await connectPromise; + + assert.strictEqual( + findRootConfigValue(transport.sentMessages, AgentHostTelemetryLevelConfigKey), + 'off', + ); + }); + test('forwards every setting declaring `agentHost` on connect and when one changes', async () => { const configurationService = new TestConfigurationService({ [SYNC_SETTING_A]: true, @@ -2004,6 +2046,7 @@ suite('RemoteAgentHostProtocolClient', () => { reconnectTransport.connectDeferred.complete(); const reconnect = await waitForRequest(reconnectTransport, 'reconnect'); assert.deepStrictEqual((reconnect.params as { _meta?: Record })._meta, { + 'vscode.telemetryLevel': 'all', 'vscode.clientMachineId': 'client-machine-id', 'vscode.clientDevDeviceId': 'client-dev-device-id', }); @@ -2020,6 +2063,7 @@ suite('RemoteAgentHostProtocolClient', () => { }, { clientInfo: agentsWindowAgentHostClientInfo, meta: { + 'vscode.telemetryLevel': 'all', 'vscode.clientMachineId': 'client-machine-id', 'vscode.clientDevDeviceId': 'client-dev-device-id', }, diff --git a/src/vs/platform/agentHost/test/node/agentHostTelemetryService.test.ts b/src/vs/platform/agentHost/test/node/agentHostTelemetryService.test.ts index eb4e65928b7a32..38870271b5b65a 100644 --- a/src/vs/platform/agentHost/test/node/agentHostTelemetryService.test.ts +++ b/src/vs/platform/agentHost/test/node/agentHostTelemetryService.test.ts @@ -17,7 +17,7 @@ import type { IProductService } from '../../../product/common/productService.js' import { ITelemetryData, ITelemetryService, TelemetryLevel } from '../../../telemetry/common/telemetry.js'; import { AgentHostTelemetryLevelConfigKey, telemetryLevelToAgentHostConfigValue } from '../../common/agentHostSchema.js'; import { AgentHostRestrictedTelemetrySender, IAgentHostRestrictedTelemetry, IAgentHostInternalTelemetryContext, IAgentHostRestrictedTelemetryContext, TelemetryProps } from '../../node/agentHostRestrictedTelemetry.js'; -import { AgentHostTelemetryService, createAgentHostTelemetryService, updateAgentHostTelemetryLevelFromConfig } from '../../node/agentHostTelemetryService.js'; +import { AgentHostTelemetryService, createAgentHostTelemetryService, type IAgentHostTelemetryService, updateAgentHostTelemetryLevelFromConfig } from '../../node/agentHostTelemetryService.js'; import { AgentHostInternalTelemetrySender } from '../../node/agentHostMicrosoftTelemetry.js'; class TestTelemetryService implements ITelemetryService { @@ -101,6 +101,54 @@ class TestRestrictedSink implements IAgentHostRestrictedTelemetry { suite('AgentHostTelemetryService', () => { const disposables = ensureNoDisposablesAreLeakedInTestSuite(); + async function createFactoryService(telemetryLevelArg: string | undefined, telemetryLevelEnvironment: string | undefined): Promise { + const localDisposables = disposables.add(new DisposableStore()); + const logService = new NullLogService(); + const fileService = localDisposables.add(new FileService(logService)); + localDisposables.add(fileService.registerProvider(Schemas.file, localDisposables.add(new InMemoryFileSystemProvider()))); + return createAgentHostTelemetryService({ + environmentService: { + args: telemetryLevelArg === undefined ? {} : { 'telemetry-level': telemetryLevelArg }, + isBuilt: true, + disableTelemetry: false, + appRoot: '/app', + extensionsPath: '/extensions', + userHome: URI.file('/home'), + tmpDir: URI.file('/tmp'), + userDataPath: '/user-data', + appSettingsHome: URI.file('/User'), + } as INativeEnvironmentService, + productService: { + _serviceBrand: undefined, + version: '1.130.0', + enableTelemetry: true, + } as IProductService, + fileService, + loggerService: localDisposables.add(new NullLoggerService()), + logService, + disposables: localDisposables, + readTelemetryLevelEnvironment: () => telemetryLevelEnvironment, + }); + } + + test('uses the most restrictive valid launch source and fails closed for malformed sources', async () => { + const services = await Promise.all([ + createFactoryService('all', 'off'), + createFactoryService('off', 'all'), + createFactoryService('invalid', 'all'), + createFactoryService('all', 'invalid'), + createFactoryService(undefined, undefined), + ]); + + assert.deepStrictEqual(services.map(service => service.telemetryLevel), [ + TelemetryLevel.NONE, + TelemetryLevel.NONE, + TelemetryLevel.NONE, + TelemetryLevel.NONE, + TelemetryLevel.USAGE, + ]); + }); + test('logging-only builds do not create restricted network senders', async () => { const localDisposables = disposables.add(new DisposableStore()); const logService = new NullLogService(); @@ -165,17 +213,18 @@ suite('AgentHostTelemetryService', () => { assert.strictEqual((internalSender as unknown as { _options: { extensionVersion: string | undefined } })._options.extensionVersion, '0.58.0'); }); - test('permanently disables usage and error telemetry after TelemetryLevel.NONE', async () => { + test('uses the launch telemetry level before a client connects and only becomes more restrictive', () => { const delegate = new TestTelemetryService(); - const service = disposables.add(new AgentHostTelemetryService(delegate)); + const service = disposables.add(new AgentHostTelemetryService(delegate, undefined, undefined, undefined, TelemetryLevel.USAGE)); - service.publicLog('beforeDisable', { count: 1 }); + service.publicLog('beforeClientLevel', { count: 1 }); + service.updateTelemetryLevel(TelemetryLevel.ERROR); + service.publicLog('afterClientLevel', { count: 2 }); + service.publicLogError('afterClientLevelError', { count: 3 }); service.updateTelemetryLevel(TelemetryLevel.NONE); service.updateTelemetryLevel(TelemetryLevel.USAGE); service.publicLog2('afterDisable'); service.publicLogError2('afterDisableError'); - service.publicLog('afterDisableAsync', { count: 4 }); - service.publicLogError('afterDisableErrorAsync', { count: 5 }); assert.deepStrictEqual({ telemetryLevel: service.telemetryLevel, @@ -185,8 +234,8 @@ suite('AgentHostTelemetryService', () => { }, { telemetryLevel: TelemetryLevel.NONE, sendErrorTelemetry: false, - events: [{ eventName: 'beforeDisable', data: { count: 1 } }], - errorEvents: [], + events: [{ eventName: 'beforeClientLevel', data: { count: 1 } }], + errorEvents: [{ eventName: 'afterClientLevelError', data: { count: 3 } }], }); }); @@ -235,6 +284,7 @@ suite('AgentHostTelemetryService', () => { test('enhanced GH telemetry is gated on the restricted (rt) opt-in; standard GH telemetry is not', () => { const restricted = new TestRestrictedSink(); const service = disposables.add(new AgentHostTelemetryService(new TestTelemetryService(), restricted)); + service.updateTelemetryLevel(TelemetryLevel.USAGE); service.sendEnhancedGHTelemetryEvent('request.options.tools'); // dropped: rt disabled by default service.sendGHTelemetryEvent('completion'); // sent: standard GH telemetry is not rt-gated @@ -264,6 +314,7 @@ suite('AgentHostTelemetryService', () => { delegate.telemetryLevel = TelemetryLevel.ERROR; // user opted below USAGE const restricted = new TestRestrictedSink(); const service = disposables.add(new AgentHostTelemetryService(delegate, restricted)); + service.updateTelemetryLevel(TelemetryLevel.USAGE); service.setRestrictedTelemetryEnabled(true); // rt=1 service.sendEnhancedGHTelemetryEvent('request.options.tools'); @@ -276,6 +327,7 @@ suite('AgentHostTelemetryService', () => { test('internal telemetry is independently gated and identity is cleared on account changes', () => { const restricted = new TestRestrictedSink(); const service = disposables.add(new AgentHostTelemetryService(new TestTelemetryService(), restricted)); + service.updateTelemetryLevel(TelemetryLevel.USAGE); const internalContext = { isInternal: true, trackingId: 'tid-1', userName: 'octocat', isVscodeTeamMember: true }; service.sendInternalMSFTTelemetryEvent('beforeIdentity'); diff --git a/src/vs/platform/agentHost/test/node/protocolServerHandler.test.ts b/src/vs/platform/agentHost/test/node/protocolServerHandler.test.ts index f03d8b67a7ac5a..2d2bc1ee1840e7 100644 --- a/src/vs/platform/agentHost/test/node/protocolServerHandler.test.ts +++ b/src/vs/platform/agentHost/test/node/protocolServerHandler.test.ts @@ -7,12 +7,14 @@ import assert from 'assert'; import { DeferredPromise } from '../../../../base/common/async.js'; import { Emitter, Event } from '../../../../base/common/event.js'; import { DisposableStore } from '../../../../base/common/lifecycle.js'; +import { hasKey } from '../../../../base/common/types.js'; import { URI } from '../../../../base/common/uri.js'; import { runWithFakedTimers } from '../../../../base/test/common/timeTravelScheduler.js'; import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js'; import { NullLogService } from '../../../log/common/log.js'; import { FileType } from '../../../files/common/files.js'; -import { NullTelemetryService, NullTelemetryServiceShape } from '../../../telemetry/common/telemetryUtils.js'; +import { NullTelemetryService } from '../../../telemetry/common/telemetryUtils.js'; +import { ITelemetryService, TelemetryLevel } from '../../../telemetry/common/telemetry.js'; import { type IAgentCreateChatOptions, type IAgentCreateSessionConfig, type IAgentResolveSessionConfigParams, type IAgentSessionConfigCompletionsParams, type IAgentSessionMetadata, type AuthenticateParams, type AuthenticateResult } from '../../common/agent.js'; import { type IAgentHostManagedSettingsDiagnostics, type IAgentHostNetworkDiagnosticsInfo, type IAgentHostNetworkFetchResult, type IAgentService } from '../../common/agentService.js'; import { ChatSourceKind, CompletionsParams, CompletionsResult, ContentEncoding, ListSessionsResult, ResourceReadResult, ResolveSessionConfigResult, SessionConfigCompletionsResult, ResourceMkdirParams, ResourceMkdirResult, ResourceResolveParams, ResourceResolveResult, ResourceCopyParams, ResourceCopyResult } from '../../common/state/protocol/commands.js'; @@ -33,6 +35,7 @@ import { iterateOtlpLogRecords, OtlpLogEmitter } from '../../common/otlp/otlpLog import { MessagePortProtocolServer } from '../../node/messagePortProtocolServer.js'; import { AgentHostClientConnectionTelemetryTracker } from '../../node/agentHostClientConnectionTelemetry.js'; import { AgentHostManagedSettingsService } from '../../node/agentHostManagedSettingsService.js'; +import { AgentHostTelemetryService } from '../../node/agentHostTelemetryService.js'; // ---- Mock helpers ----------------------------------------------------------- @@ -108,14 +111,27 @@ class FailingReconnectAgentHostFileSystemProvider extends AgentHostFileSystemPro } } -class TestTelemetryService extends NullTelemetryServiceShape { +class TestTelemetryService implements ITelemetryService { + declare readonly _serviceBrand: undefined; + readonly telemetryLevel = TelemetryLevel.USAGE; + readonly sendErrorTelemetry = true; + readonly sessionId = 'session'; + readonly machineId = 'machine'; + readonly sqmId = 'sqm'; + readonly devDeviceId = 'device'; + readonly firstSessionDate = 'first-session'; readonly events: { eventName: string; data: unknown }[] = []; - override publicLog2(eventName?: string, data?: unknown): void { + publicLog(): void { } + publicLog2(eventName?: string, data?: unknown): void { if (eventName) { this.events.push({ eventName, data }); } } + publicLogError(): void { } + publicLogError2(): void { } + setExperimentProperty(): void { } + setCommonProperty(): void { } } class MockAgentService implements IAgentService { @@ -291,6 +307,7 @@ suite('ProtocolServerHandler', () => { let fileSystemProvider: AgentHostFileSystemProvider; let logService: CountingLogService; let telemetryService: TestTelemetryService; + let agentHostTelemetryService: AgentHostTelemetryService; const sessionUri = URI.from({ scheme: 'copilot', path: '/test-session' }).toString(); const defaultChatUri = buildDefaultChatUri(sessionUri); @@ -314,7 +331,10 @@ suite('ProtocolServerHandler', () => { protocolVersions: [PROTOCOL_VERSION], clientId, clientInfo, - _meta: meta, + _meta: { + 'vscode.telemetryLevel': 'all', + ...meta, + }, initialSubscriptions, })); return transport; @@ -329,6 +349,7 @@ suite('ProtocolServerHandler', () => { managedSettingsService = disposables.add(new AgentHostManagedSettingsService()); logService = new CountingLogService(); telemetryService = new TestTelemetryService(); + agentHostTelemetryService = disposables.add(new AgentHostTelemetryService(telemetryService)); disposables.add(agentService); disposables.add(handler = new ProtocolServerHandler( agentService, @@ -337,7 +358,7 @@ suite('ProtocolServerHandler', () => { { hostLaunchKind: AgentHostLaunchKind.VSCodeMainProcess, defaultDirectory: URI.file('/home/testuser').toString() }, disposables.add(fileSystemProvider = new AgentHostFileSystemProvider()), logService, - telemetryService, + agentHostTelemetryService, managedSettingsService, )); }); @@ -352,12 +373,79 @@ suite('ProtocolServerHandler', () => { const transport = connectClient('client-1'); const resp = findResponse(transport.sent, 1); - assert.ok(resp, 'should have sent initialize response'); - const result = (resp as { result: InitializeResult }).result; + if (!resp || !hasKey(resp, { result: true })) { + assert.fail('should have sent initialize response'); + } + const result = resp.result as InitializeResult; assert.strictEqual(result.protocolVersion, PROTOCOL_VERSION); assert.strictEqual(result.serverSeq, stateManager.serverSeq); }); + test('applies telemetry disablement before reporting the client connection', () => { + const transport = new MockProtocolTransport(AgentHostTransportKind.WebSocket); + server.simulateConnection(transport); + transport.simulateMessage(request(1, 'initialize', { + protocolVersions: [PROTOCOL_VERSION], + clientId: 'telemetry-disabled-client', + clientInfo: editorWindowAgentHostClientInfo, + _meta: { + 'vscode.clientConnectionKind': AgentHostClientConnectionKind.RemoteExtensionHost, + 'vscode.telemetryLevel': 'off', + }, + })); + + assert.deepStrictEqual({ + telemetryLevel: agentHostTelemetryService.telemetryLevel, + events: telemetryService.events, + }, { + telemetryLevel: TelemetryLevel.NONE, + events: [], + }); + transport.simulateClose(); + transport.dispose(); + }); + + test('uses the launch telemetry level when a legacy client omits telemetry metadata', () => { + const transport = new MockProtocolTransport(AgentHostTransportKind.WebSocket); + server.simulateConnection(transport); + transport.simulateMessage(request(1, 'initialize', { + protocolVersions: [PROTOCOL_VERSION], + clientId: 'legacy-client', + clientInfo: editorWindowAgentHostClientInfo, + })); + + assert.deepStrictEqual({ + telemetryLevel: agentHostTelemetryService.telemetryLevel, + eventNames: telemetryService.events.map(event => event.eventName), + }, { + telemetryLevel: TelemetryLevel.USAGE, + eventNames: ['agentHost.clientConnection'], + }); + transport.simulateClose(); + transport.dispose(); + }); + + test('fails closed before reporting the client connection for malformed telemetry metadata', () => { + const transport = new MockProtocolTransport(AgentHostTransportKind.WebSocket); + server.simulateConnection(transport); + transport.simulateMessage(request(1, 'initialize', { + protocolVersions: [PROTOCOL_VERSION], + clientId: 'malformed-telemetry-client', + clientInfo: editorWindowAgentHostClientInfo, + _meta: { 'vscode.telemetryLevel': 'invalid' }, + })); + + assert.deepStrictEqual({ + telemetryLevel: agentHostTelemetryService.telemetryLevel, + events: telemetryService.events, + }, { + telemetryLevel: TelemetryLevel.NONE, + events: [], + }); + transport.simulateClose(); + transport.dispose(); + }); + test('handshake rejects unsupported protocol versions', () => { const transport = new MockProtocolTransport(); server.simulateConnection(transport); @@ -1292,6 +1380,86 @@ suite('ProtocolServerHandler', () => { }); }); + test('applies telemetry disablement before reporting a reconnected client', async () => { + const transport1 = connectClient('telemetry-reconnect-client'); + transport1.simulateClose(); + const eventsBeforeReconnect = [...telemetryService.events]; + + const transport2 = new MockProtocolTransport(); + server.simulateConnection(transport2); + const reconnectResponse = waitForResponse(transport2, 2); + transport2.simulateMessage(request(2, 'reconnect', { + clientId: 'telemetry-reconnect-client', + lastSeenServerSeq: stateManager.serverSeq, + subscriptions: [], + _meta: { 'vscode.telemetryLevel': 'off' }, + })); + await reconnectResponse; + + assert.deepStrictEqual({ + telemetryLevel: agentHostTelemetryService.telemetryLevel, + events: telemetryService.events, + }, { + telemetryLevel: TelemetryLevel.NONE, + events: eventsBeforeReconnect, + }); + transport2.simulateClose(); + transport2.dispose(); + }); + + test('fails closed before reporting a reconnected client for malformed telemetry metadata', async () => { + const transport1 = connectClient('malformed-telemetry-reconnect-client'); + transport1.simulateClose(); + const eventsBeforeReconnect = [...telemetryService.events]; + + const transport2 = new MockProtocolTransport(); + server.simulateConnection(transport2); + const reconnectResponse = waitForResponse(transport2, 2); + transport2.simulateMessage(request(2, 'reconnect', { + clientId: 'malformed-telemetry-reconnect-client', + lastSeenServerSeq: stateManager.serverSeq, + subscriptions: [], + _meta: { 'vscode.telemetryLevel': 'invalid' }, + })); + await reconnectResponse; + + assert.deepStrictEqual({ + telemetryLevel: agentHostTelemetryService.telemetryLevel, + events: telemetryService.events, + }, { + telemetryLevel: TelemetryLevel.NONE, + events: eventsBeforeReconnect, + }); + transport2.simulateClose(); + transport2.dispose(); + }); + + test('uses the launch telemetry level when a legacy reconnect omits telemetry metadata', async () => { + const transport1 = connectClient('legacy-reconnect-client'); + transport1.simulateClose(); + const eventCountBeforeReconnect = telemetryService.events.length; + + const transport2 = new MockProtocolTransport(); + server.simulateConnection(transport2); + const reconnectResponse = waitForResponse(transport2, 2); + transport2.simulateMessage(request(2, 'reconnect', { + clientId: 'legacy-reconnect-client', + lastSeenServerSeq: stateManager.serverSeq, + subscriptions: [], + })); + await reconnectResponse; + + assert.deepStrictEqual({ + telemetryLevel: agentHostTelemetryService.telemetryLevel, + newEventNames: telemetryService.events.slice(eventCountBeforeReconnect).map(event => event.eventName), + }, { + telemetryLevel: TelemetryLevel.USAGE, + newEventNames: ['agentHost.clientConnection'], + }); + transport2.simulateClose(); + transport2.dispose(); + }); + test('does not retain client telemetry identity when reconnect omits it', async () => { const transport1 = connectClient('client-consent', undefined, agentsWindowAgentHostClientInfo, { 'vscode.clientMachineId': 'client-machine-id', @@ -1366,6 +1534,7 @@ suite('ProtocolServerHandler', () => { clientInfo: { name: 'vscode-agents-window', version: '1.2.3', title: 'VS Code Agents Window' }, _meta: { 'vscode.clientConnectionKind': AgentHostClientConnectionKind.DevTunnel, + 'vscode.telemetryLevel': 'all', 'vscode.clientMachineId': 'client-machine-id', 'vscode.clientDevDeviceId': 'client-dev-device-id', }, diff --git a/src/vs/platform/agentHost/test/node/sshHostKeyVerification.test.ts b/src/vs/platform/agentHost/test/node/sshHostKeyVerification.test.ts index a1404cce6f604e..379c3d8d0f4b30 100644 --- a/src/vs/platform/agentHost/test/node/sshHostKeyVerification.test.ts +++ b/src/vs/platform/agentHost/test/node/sshHostKeyVerification.test.ts @@ -8,6 +8,7 @@ import type { ConnectConfig } from 'ssh2'; import { DisposableStore } from '../../../../base/common/lifecycle.js'; import { NullLogService } from '../../../log/common/log.js'; import { IProductService } from '../../../product/common/productService.js'; +import { NullTelemetryService } from '../../../telemetry/common/telemetryUtils.js'; import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js'; import { isSSHHostKeyDeniedError, SSHAuthMethod, type ISSHAgentHostConfig, type ISSHHostKeyVerificationRequest } from '../../common/sshRemoteAgentHost.js'; import { SSHRemoteAgentHostMainService, type SSHAuthAttempt } from '../../node/sshRemoteAgentHostService.js'; @@ -184,7 +185,7 @@ suite('SSHRemoteAgentHostMainService - host key verification', () => { quality: 'stable', dataFolderName: '.vscode-oss', }; - return disposables.add(new HostKeyTestService(new NullLogService(), productService as IProductService)); + return disposables.add(new HostKeyTestService(new NullLogService(), productService as IProductService, NullTelemetryService)); } /** Run a connect attempt, answering the verification request with `trusted`. */ diff --git a/src/vs/platform/agentHost/test/node/sshRemoteAgentHostHelpers.test.ts b/src/vs/platform/agentHost/test/node/sshRemoteAgentHostHelpers.test.ts index b7e94a0167fcef..a2cdddfe4d2b10 100644 --- a/src/vs/platform/agentHost/test/node/sshRemoteAgentHostHelpers.test.ts +++ b/src/vs/platform/agentHost/test/node/sshRemoteAgentHostHelpers.test.ts @@ -5,6 +5,7 @@ import assert from 'assert'; import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js'; +import { TelemetryConfiguration } from '../../../telemetry/common/telemetry.js'; import { AGENT_HOST_ENDPOINT_REGISTRY_SCHEMA_VERSION, type IAgentHostEndpointMetadata } from '../../common/agentHostEndpointRegistry.js'; import { buildAgentEndpointsCommand, @@ -26,6 +27,7 @@ import { resolveRemotePlatform, runAgentEndpoints, shellEscape, + validateAgentHostTelemetryLevel, validateCommit, validateShellToken, waitForNewStandaloneEndpoint, @@ -157,9 +159,18 @@ suite('SSH Remote Agent Host Helpers', () => { }); suite('buildAgentHostBaseCommand', () => { - test('includes --cli-data-dir before the agent host subcommand', () => { - const cmd = buildAgentHostBaseCommand('~/.vscode-server/code-insiders-abc', '~/.vscode-server/cli'); - assert.strictEqual(cmd, '~/.vscode-server/code-insiders-abc --cli-data-dir ~/.vscode-server/cli agent host --port 0'); + test('includes --cli-data-dir and the default telemetry level before the agent host subcommand', () => { + const cmd = buildAgentHostBaseCommand('~/.vscode-server/code-insiders-abc', '~/.vscode-server/cli', TelemetryConfiguration.ON); + assert.strictEqual(cmd, '~/.vscode-server/code-insiders-abc --cli-data-dir ~/.vscode-server/cli --telemetry-level all agent host --port 0'); + }); + + test('includes telemetry disablement before the agent host subcommand', () => { + const cmd = buildAgentHostBaseCommand('~/.vscode-server/code-insiders-abc', '~/.vscode-server/cli', TelemetryConfiguration.OFF); + assert.strictEqual(cmd, '~/.vscode-server/code-insiders-abc --cli-data-dir ~/.vscode-server/cli --telemetry-level off agent host --port 0'); + }); + + test('rejects unsafe telemetry levels', () => { + assert.throws(() => validateAgentHostTelemetryLevel('off; touch /tmp/unsafe'), /Unsafe telemetry level/); }); }); @@ -456,26 +467,33 @@ suite('SSH Remote Agent Host Helpers', () => { suite('buildAgentHostSpawnCommand', () => { test('includes --new-instance, --user-data-dir and default --idle-timeout', () => { assert.strictEqual( - buildAgentHostSpawnCommand('~/.vscode-server/code', '~/.vscode-server/cli', '/home/user/.vscode-remote'), - '~/.vscode-server/code --cli-data-dir ~/.vscode-server/cli agent host --port 0 --new-instance --user-data-dir \'/home/user/.vscode-remote\' --idle-timeout 300', + buildAgentHostSpawnCommand('~/.vscode-server/code', '~/.vscode-server/cli', '/home/user/.vscode-remote', TelemetryConfiguration.ON), + '~/.vscode-server/code --cli-data-dir ~/.vscode-server/cli --telemetry-level all agent host --port 0 --new-instance --user-data-dir \'/home/user/.vscode-remote\' --idle-timeout 300', ); }); test('honors a custom idle timeout', () => { assert.strictEqual( - buildAgentHostSpawnCommand('~/.vscode-server/code', '~/.vscode-server/cli', '/home/user/.vscode-remote', 60), - '~/.vscode-server/code --cli-data-dir ~/.vscode-server/cli agent host --port 0 --new-instance --user-data-dir \'/home/user/.vscode-remote\' --idle-timeout 60', + buildAgentHostSpawnCommand('~/.vscode-server/code', '~/.vscode-server/cli', '/home/user/.vscode-remote', TelemetryConfiguration.ON, 60), + '~/.vscode-server/code --cli-data-dir ~/.vscode-server/cli --telemetry-level all agent host --port 0 --new-instance --user-data-dir \'/home/user/.vscode-remote\' --idle-timeout 60', + ); + }); + + test('propagates telemetry disablement to a new dedicated agent host', () => { + assert.strictEqual( + buildAgentHostSpawnCommand('~/.vscode-server/code', '~/.vscode-server/cli', '/home/user/.vscode-remote', TelemetryConfiguration.OFF), + '~/.vscode-server/code --cli-data-dir ~/.vscode-server/cli --telemetry-level off agent host --port 0 --new-instance --user-data-dir \'/home/user/.vscode-remote\' --idle-timeout 300', ); }); test('rejects unsafe idle timeout values', () => { - assert.throws(() => buildAgentHostSpawnCommand('~/.vscode-server/code', '~/.vscode-server/cli', '/x', 0), /Unsafe idle timeout/); - assert.throws(() => buildAgentHostSpawnCommand('~/.vscode-server/code', '~/.vscode-server/cli', '/x', -1), /Unsafe idle timeout/); - assert.throws(() => buildAgentHostSpawnCommand('~/.vscode-server/code', '~/.vscode-server/cli', '/x', 1.5), /Unsafe idle timeout/); + assert.throws(() => buildAgentHostSpawnCommand('~/.vscode-server/code', '~/.vscode-server/cli', '/x', TelemetryConfiguration.ON, 0), /Unsafe idle timeout/); + assert.throws(() => buildAgentHostSpawnCommand('~/.vscode-server/code', '~/.vscode-server/cli', '/x', TelemetryConfiguration.ON, -1), /Unsafe idle timeout/); + assert.throws(() => buildAgentHostSpawnCommand('~/.vscode-server/code', '~/.vscode-server/cli', '/x', TelemetryConfiguration.ON, 1.5), /Unsafe idle timeout/); }); test('always includes --new-instance so an existing standalone is never silently reused', () => { - const cmd = buildAgentHostSpawnCommand('~/.vscode-server/code', '~/.vscode-server/cli', '/x'); + const cmd = buildAgentHostSpawnCommand('~/.vscode-server/code', '~/.vscode-server/cli', '/x', TelemetryConfiguration.ON); assert.ok(cmd.includes(' --new-instance '), 'spawn command must request a genuinely new instance, not reuse an existing standalone'); }); }); diff --git a/src/vs/platform/agentHost/test/node/sshRemoteAgentHostService.test.ts b/src/vs/platform/agentHost/test/node/sshRemoteAgentHostService.test.ts index ed7da6dc3fba62..9143912f2b1c93 100644 --- a/src/vs/platform/agentHost/test/node/sshRemoteAgentHostService.test.ts +++ b/src/vs/platform/agentHost/test/node/sshRemoteAgentHostService.test.ts @@ -12,6 +12,8 @@ import { DisposableStore } from '../../../../base/common/lifecycle.js'; import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js'; import { NullLogService } from '../../../log/common/log.js'; import { IProductService } from '../../../product/common/productService.js'; +import { TelemetryConfiguration } from '../../../telemetry/common/telemetry.js'; +import { NullTelemetryService } from '../../../telemetry/common/telemetryUtils.js'; import { AGENT_HOST_ENDPOINT_REGISTRY_SCHEMA_VERSION, type AgentHostEndpointAddress, type IAgentHostEndpointMetadata } from '../../common/agentHostEndpointRegistry.js'; import { SSHAuthMethod, type ISSHAgentHostConfig, type ISSHConnectProgress, type ISSHEndpointSelection, type ISSHEndpointSelectionRequest, type ISSHKeyboardInteractivePrompt, type ISSHKeyboardInteractiveRequest } from '../../common/sshRemoteAgentHost.js'; import { SSHRemoteAgentHostMainService, makeAuthHandler, type SSHAuthAttempt } from '../../node/sshRemoteAgentHostService.js'; @@ -301,7 +303,7 @@ class TestableSSHRemoteAgentHostMainService extends SSHRemoteAgentHostMainServic } protected override async _startRemoteAgentHost( - _client: unknown, _cliBin: string | undefined, _cliDataDir: string | undefined, _commandOverride?: string, + _client: unknown, _cliBin: string | undefined, _cliDataDir: string | undefined, _commandOverride?: string, _telemetryLevel?: TelemetryConfiguration, ) { this.startCalled++; return { ...this.startResult, stream: new MockSSHChannel() as never }; @@ -463,6 +465,7 @@ suite('SSHRemoteAgentHostMainService - connect flow', () => { service = new TestableSSHRemoteAgentHostMainService( logService, productService as IProductService, + NullTelemetryService, ); disposables.add(service); }); @@ -588,6 +591,7 @@ suite('SSHRemoteAgentHostMainService - connect flow', () => { const execCalls = service.mockClients[0].execCalls; assert.ok(execCalls.some(c => c.includes('--idle-timeout 300')), `should spawn with idle timeout; saw: ${JSON.stringify(execCalls)}`); assert.ok(execCalls.some(c => c.includes('--new-instance')), `spawn must request a genuinely new instance; saw: ${JSON.stringify(execCalls)}`); + assert.ok(execCalls.some(c => c.includes('--telemetry-level off')), `spawn must apply telemetry disablement; saw: ${JSON.stringify(execCalls)}`); }); test('reuses the single live standalone deterministically without a picker', async () => { @@ -1188,6 +1192,7 @@ suite('SSHRemoteAgentHostMainService - connect flow', () => { quality, dataFolderName, } as IProductService, + NullTelemetryService, )); const request = new DeferredPromise(); disposables.add(kbiService.onDidRequestKeyboardInteractive(kbiRequest => request.complete(kbiRequest))); @@ -1282,6 +1287,7 @@ suite('SSHRemoteAgentHostMainService - connect flow', () => { const loggingService = disposables.add(new TestableSSHRemoteAgentHostMainService( logService, productService as IProductService, + NullTelemetryService, )); loggingService.execResponses = [ { stdout: 'Linux\n', code: 0 }, @@ -1309,6 +1315,7 @@ suite('SSHRemoteAgentHostMainService - connect flow', () => { const loggingService = disposables.add(new TestableSSHRemoteAgentHostMainService( logService, productService as IProductService, + NullTelemetryService, )); loggingService.execResponses = [ { stdout: 'Linux\n', code: 0 }, @@ -1343,6 +1350,7 @@ suite('SSHRemoteAgentHostMainService - connect flow', () => { pinnedService = new TestableSSHRemoteAgentHostMainService( logService, productService as IProductService, + NullTelemetryService, ); disposables.add(pinnedService); }); @@ -1632,6 +1640,7 @@ suite('SSHRemoteAgentHostMainService - _buildAuthAttempts', () => { service = new AuthAttemptsTestService( logService, productService as IProductService, + NullTelemetryService, ); disposables.add(service); }); diff --git a/src/vs/platform/agentHost/test/node/wslRemoteAgentHostHelpers.test.ts b/src/vs/platform/agentHost/test/node/wslRemoteAgentHostHelpers.test.ts index 46804334868a9c..600eee006a8a47 100644 --- a/src/vs/platform/agentHost/test/node/wslRemoteAgentHostHelpers.test.ts +++ b/src/vs/platform/agentHost/test/node/wslRemoteAgentHostHelpers.test.ts @@ -5,7 +5,9 @@ import assert from 'assert'; import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js'; +import { TelemetryConfiguration } from '../../../telemetry/common/telemetry.js'; import { + composeAgentHostBootstrapScript, decodeWslOutput, parseRunningDistros, parseWslListVerbose, @@ -99,4 +101,34 @@ suite('WSL Remote Agent Host Helpers', () => { assert.strictEqual(decodeWslOutput(Buffer.alloc(0)), ''); }); }); + + suite('composeAgentHostBootstrapScript', () => { + test('propagates telemetry disablement to the WSL agent host', () => { + const commit = 'a'.repeat(40); + const script = composeAgentHostBootstrapScript({ + serverDataFolderName: '.vscode-server', + quality: 'stable', + commit, + os: 'linux', + arch: 'x64', + telemetryLevel: TelemetryConfiguration.OFF, + }); + + assert.ok(script.endsWith(`exec ~/.vscode-server/code-${commit} --cli-data-dir ~/.vscode-server/cli --telemetry-level off agent host --port 0`)); + }); + + test('exports telemetry disablement for a custom command', () => { + const script = composeAgentHostBootstrapScript({ + serverDataFolderName: '.vscode-server', + quality: 'stable', + commit: undefined, + os: 'linux', + arch: 'x64', + telemetryLevel: TelemetryConfiguration.OFF, + remoteAgentHostCommand: './start-agent-host', + }); + + assert.strictEqual(script, 'export VSCODE_AGENT_HOST_TELEMETRY_LEVEL=off && ./start-agent-host'); + }); + }); }); diff --git a/src/vs/platform/environment/common/argv.ts b/src/vs/platform/environment/common/argv.ts index 18d653bac95609..00e12e78007805 100644 --- a/src/vs/platform/environment/common/argv.ts +++ b/src/vs/platform/environment/common/argv.ts @@ -110,6 +110,7 @@ export interface NativeParsedArgs { 'skip-release-notes'?: boolean; 'skip-welcome'?: boolean; 'disable-telemetry'?: boolean; + 'telemetry-level'?: string; 'export-default-configuration'?: string; 'export-policy-data'?: string; 'export-default-keybindings'?: string; diff --git a/src/vs/platform/environment/node/argv.ts b/src/vs/platform/environment/node/argv.ts index 5933c9c71d9473..8a36a25aea5fe4 100644 --- a/src/vs/platform/environment/node/argv.ts +++ b/src/vs/platform/environment/node/argv.ts @@ -185,6 +185,7 @@ export const OPTIONS: OptionDescriptions> = { 'skip-release-notes': { type: 'boolean' }, 'skip-welcome': { type: 'boolean' }, 'disable-telemetry': { type: 'boolean' }, + 'telemetry-level': { type: 'string' }, 'disable-updates': { type: 'boolean' }, 'share-secrets-with-agents-app': { type: 'boolean' }, 'transient': { type: 'boolean', cat: 't', description: localize('transient', "Run with temporary data and extension directories, as if launched for the first time.") }, diff --git a/src/vs/platform/telemetry/common/serverTelemetryService.ts b/src/vs/platform/telemetry/common/serverTelemetryService.ts index f6fc225ab842fe..7dd6ad7560c00d 100644 --- a/src/vs/platform/telemetry/common/serverTelemetryService.ts +++ b/src/vs/platform/telemetry/common/serverTelemetryService.ts @@ -30,6 +30,10 @@ export class ServerTelemetryService extends TelemetryService implements IServerT this._injectedTelemetryLevel = injectedTelemetryLevel; } + override get telemetryLevel(): TelemetryLevel { + return Math.min(super.telemetryLevel, this._injectedTelemetryLevel); + } + override publicLog(eventName: string, data?: ITelemetryData) { if (this._injectedTelemetryLevel < TelemetryLevel.USAGE) { return; @@ -58,7 +62,7 @@ export class ServerTelemetryService extends TelemetryService implements IServerT throw new Error('Telemetry level cannot be undefined. This will cause infinite looping!'); } // We always take the most restrictive level because we don't want multiple clients to connect and send data when one client does not consent - this._injectedTelemetryLevel = this._injectedTelemetryLevel ? Math.min(this._injectedTelemetryLevel, telemetryLevel) : telemetryLevel; + this._injectedTelemetryLevel = Math.min(this._injectedTelemetryLevel, telemetryLevel); if (this._injectedTelemetryLevel === TelemetryLevel.NONE) { this.dispose(); } diff --git a/src/vs/platform/telemetry/test/common/serverTelemetryService.test.ts b/src/vs/platform/telemetry/test/common/serverTelemetryService.test.ts new file mode 100644 index 00000000000000..327449d4eea358 --- /dev/null +++ b/src/vs/platform/telemetry/test/common/serverTelemetryService.test.ts @@ -0,0 +1,39 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js'; +import { TestConfigurationService } from '../../../configuration/test/common/testConfigurationService.js'; +import product from '../../../product/common/product.js'; +import { IProductService } from '../../../product/common/productService.js'; +import { TelemetryLevel } from '../../common/telemetry.js'; +import { ServerTelemetryService } from '../../common/serverTelemetryService.js'; +import { NullAppender } from '../../common/telemetryUtils.js'; + +suite('ServerTelemetryService', () => { + const disposables = ensureNoDisposablesAreLeakedInTestSuite(); + const productService: IProductService = { _serviceBrand: undefined, ...product }; + + test('exposes and preserves the most restrictive injected telemetry level', async () => { + const service = disposables.add(new ServerTelemetryService( + { appenders: [NullAppender] }, + TelemetryLevel.ERROR, + new TestConfigurationService(), + productService, + )); + + const initialLevel = service.telemetryLevel; + await service.updateInjectedTelemetryLevel(TelemetryLevel.NONE); + await service.updateInjectedTelemetryLevel(TelemetryLevel.USAGE); + + assert.deepStrictEqual({ + initialLevel, + finalLevel: service.telemetryLevel, + }, { + initialLevel: TelemetryLevel.ERROR, + finalLevel: TelemetryLevel.NONE, + }); + }); +}); diff --git a/src/vs/workbench/contrib/remote/electron-browser/remote.contribution.ts b/src/vs/workbench/contrib/remote/electron-browser/remote.contribution.ts index 2feba38fb4f864..f6e4636aea002b 100644 --- a/src/vs/workbench/contrib/remote/electron-browser/remote.contribution.ts +++ b/src/vs/workbench/contrib/remote/electron-browser/remote.contribution.ts @@ -25,7 +25,7 @@ import { IConfigurationRegistry, Extensions as ConfigurationExtensions } from '. import { IRemoteAuthorityResolverService } from '../../../../platform/remote/common/remoteAuthorityResolver.js'; import { OpenLocalFileFolderCommand, OpenLocalFileCommand, OpenLocalFolderCommand, SaveLocalFileCommand, RemoteFileDialogContext } from '../../../services/dialogs/browser/simpleFileDialog.js'; import { IWorkspaceContextService, WorkbenchState } from '../../../../platform/workspace/common/workspace.js'; -import { TELEMETRY_SETTING_ID } from '../../../../platform/telemetry/common/telemetry.js'; +import { ITelemetryService, TELEMETRY_CRASH_REPORTER_SETTING_ID, TELEMETRY_OLD_SETTING_ID, TELEMETRY_SETTING_ID } from '../../../../platform/telemetry/common/telemetry.js'; import { getTelemetryLevel } from '../../../../platform/telemetry/common/telemetryUtils.js'; import { IContextKeyService, RawContextKey, ContextKeyExpr } from '../../../../platform/contextkey/common/contextkey.js'; import { INativeHostService } from '../../../../platform/native/common/native.js'; @@ -100,21 +100,22 @@ class RemoteTelemetryEnablementUpdater extends Disposable implements IWorkbenchC constructor( @IRemoteAgentService private readonly remoteAgentService: IRemoteAgentService, - @IConfigurationService private readonly configurationService: IConfigurationService + @IConfigurationService private readonly configurationService: IConfigurationService, + @ITelemetryService private readonly telemetryService: ITelemetryService, ) { super(); this.updateRemoteTelemetryEnablement(); this._register(configurationService.onDidChangeConfiguration(e => { - if (e.affectsConfiguration(TELEMETRY_SETTING_ID)) { + if (e.affectsConfiguration(TELEMETRY_SETTING_ID) || e.affectsConfiguration(TELEMETRY_OLD_SETTING_ID) || e.affectsConfiguration(TELEMETRY_CRASH_REPORTER_SETTING_ID)) { this.updateRemoteTelemetryEnablement(); } })); } private updateRemoteTelemetryEnablement(): Promise { - return this.remoteAgentService.updateTelemetryLevel(getTelemetryLevel(this.configurationService)); + return this.remoteAgentService.updateTelemetryLevel(Math.min(getTelemetryLevel(this.configurationService), this.telemetryService.telemetryLevel)); } }