diff --git a/src/client/chat/baseTool.ts b/src/client/chat/baseTool.ts index 2eedbbe226e3..d8e2e1d60d42 100644 --- a/src/client/chat/baseTool.ts +++ b/src/client/chat/baseTool.ts @@ -16,8 +16,10 @@ import { IResourceReference, isCancellationError, resolveFilePath } from './util import { ErrorWithTelemetrySafeReason } from '../common/errors/errorUtils'; import { sendTelemetryEvent } from '../telemetry'; import { EventName } from '../telemetry/constants'; +import { StopWatch } from '../common/utils/stopWatch'; export abstract class BaseTool implements LanguageModelTool { + protected extraTelemetryProperties: Record = {}; constructor(private readonly toolName: string) {} async invoke( @@ -29,8 +31,10 @@ export abstract class BaseTool implements Language new LanguageModelTextPart('Cannot use this tool in an untrusted workspace.'), ]); } + this.extraTelemetryProperties = {}; let error: Error | undefined; const resource = resolveFilePath(options.input.resourcePath); + const stopWatch = new StopWatch(); try { return await this.invokeImpl(options, resource, token); } catch (ex) { @@ -46,10 +50,11 @@ export abstract class BaseTool implements Language ? error.telemetrySafeReason : 'error' : undefined; - sendTelemetryEvent(EventName.INVOKE_TOOL, undefined, { + sendTelemetryEvent(EventName.INVOKE_TOOL, stopWatch.elapsedTime, { toolName: this.toolName, failed, failureCategory, + ...this.extraTelemetryProperties, }); } } diff --git a/src/client/chat/configurePythonEnvTool.ts b/src/client/chat/configurePythonEnvTool.ts index 0634b9c9ac34..914a92f81c52 100644 --- a/src/client/chat/configurePythonEnvTool.ts +++ b/src/client/chat/configurePythonEnvTool.ts @@ -18,6 +18,7 @@ import { ICodeExecutionService } from '../terminals/types'; import { TerminalCodeExecutionProvider } from '../terminals/codeExecution/terminalCodeExecution'; import { getEnvDetailsForResponse, + getEnvTypeForTelemetry, getToolResponseIfNotebook, IResourceReference, isCancellationError, @@ -58,6 +59,7 @@ export class ConfigurePythonEnvTool extends BaseTool ): Promise { const notebookResponse = getToolResponseIfNotebook(resource); if (notebookResponse) { + this.extraTelemetryProperties.resolveOutcome = 'notebook'; return notebookResponse; } @@ -67,6 +69,8 @@ export class ConfigurePythonEnvTool extends BaseTool ); if (workspaceSpecificEnv) { + this.extraTelemetryProperties.resolveOutcome = 'existingWorkspaceEnv'; + this.extraTelemetryProperties.envType = getEnvTypeForTelemetry(workspaceSpecificEnv); return getEnvDetailsForResponse( workspaceSpecificEnv, this.api, @@ -79,7 +83,9 @@ export class ConfigurePythonEnvTool extends BaseTool if (await this.createEnvTool.shouldCreateNewVirtualEnv(resource, token)) { try { - return await lm.invokeTool(CreateVirtualEnvTool.toolName, options, token); + const result = await lm.invokeTool(CreateVirtualEnvTool.toolName, options, token); + this.extraTelemetryProperties.resolveOutcome = 'createdVirtualEnv'; + return result; } catch (ex) { if (isCancellationError(ex)) { const input: ISelectPythonEnvToolArguments = { @@ -87,6 +93,7 @@ export class ConfigurePythonEnvTool extends BaseTool reason: 'cancelled', }; // If the user cancelled the tool, then we should invoke the select env tool. + this.extraTelemetryProperties.resolveOutcome = 'selectedEnvAfterCancelledCreate'; return lm.invokeTool(SelectPythonEnvTool.toolName, { ...options, input }, token); } throw ex; @@ -95,6 +102,7 @@ export class ConfigurePythonEnvTool extends BaseTool const input: ISelectPythonEnvToolArguments = { ...options.input, }; + this.extraTelemetryProperties.resolveOutcome = 'selectedEnv'; return lm.invokeTool(SelectPythonEnvTool.toolName, { ...options, input }, token); } } diff --git a/src/client/chat/getExecutableTool.ts b/src/client/chat/getExecutableTool.ts index 746a540d14f8..38dabce644a7 100644 --- a/src/client/chat/getExecutableTool.ts +++ b/src/client/chat/getExecutableTool.ts @@ -19,6 +19,7 @@ import { TerminalCodeExecutionProvider } from '../terminals/codeExecution/termin import { getEnvDisplayName, getEnvironmentDetails, + getEnvTypeForTelemetry, getToolResponseIfNotebook, IResourceReference, raceCancellationError, @@ -53,6 +54,12 @@ export class GetExecutableTool extends BaseTool implements L return notebookResponse; } + const envPath = this.api.getActiveEnvironmentPath(resourcePath); + const environment = await raceCancellationError(this.api.resolveEnvironment(envPath), token); + if (environment) { + this.extraTelemetryProperties.envType = getEnvTypeForTelemetry(environment); + } + const message = await getEnvironmentDetails( resourcePath, this.api, diff --git a/src/client/chat/getPythonEnvTool.ts b/src/client/chat/getPythonEnvTool.ts index ed1dd0374424..d25d72baeba8 100644 --- a/src/client/chat/getPythonEnvTool.ts +++ b/src/client/chat/getPythonEnvTool.ts @@ -17,7 +17,13 @@ import { IServiceContainer } from '../ioc/types'; import { ICodeExecutionService } from '../terminals/types'; import { TerminalCodeExecutionProvider } from '../terminals/codeExecution/terminalCodeExecution'; import { IProcessServiceFactory, IPythonExecutionFactory } from '../common/process/types'; -import { getEnvironmentDetails, getToolResponseIfNotebook, IResourceReference, raceCancellationError } from './utils'; +import { + getEnvironmentDetails, + getEnvTypeForTelemetry, + getToolResponseIfNotebook, + IResourceReference, + raceCancellationError, +} from './utils'; import { getPythonPackagesResponse } from './listPackagesTool'; import { ITerminalHelper } from '../common/terminal/types'; import { getEnvExtApi, useEnvExtension } from '../envExt/api.internal'; @@ -64,13 +70,16 @@ export class GetEnvironmentInfoTool extends BaseTool 'noEnvFound', ); } + this.extraTelemetryProperties.envType = getEnvTypeForTelemetry(environment); let packages = ''; + let responsePackageCount = 0; if (useEnvExtension()) { const api = await getEnvExtApi(); const env = await api.getEnvironment(resourcePath); const pkgs = env ? await api.getPackages(env) : []; if (pkgs && pkgs.length > 0) { + responsePackageCount = pkgs.length; // Installed Python packages, each in the format or (). The version may be omitted if unknown. Returns an empty array if no packages are installed. const response = [ 'Below is a list of the Python packages, each in the format or (). The version may be omitted if unknown: ', @@ -90,7 +99,10 @@ export class GetEnvironmentInfoTool extends BaseTool resourcePath, token, ); + // Count lines starting with '- ' to get the number of packages + responsePackageCount = (packages.match(/^- /gm) || []).length; } + this.extraTelemetryProperties.responsePackageCount = String(responsePackageCount); const message = await getEnvironmentDetails( resourcePath, this.api, diff --git a/src/client/chat/installPackagesTool.ts b/src/client/chat/installPackagesTool.ts index f7795620cf13..5d3d456361f9 100644 --- a/src/client/chat/installPackagesTool.ts +++ b/src/client/chat/installPackagesTool.ts @@ -16,6 +16,7 @@ import { PythonExtension } from '../api/types'; import { IServiceContainer } from '../ioc/types'; import { getEnvDisplayName, + getEnvTypeForTelemetry, getToolResponseIfNotebook, IResourceReference, isCancellationError, @@ -51,6 +52,7 @@ export class InstallPackagesTool extends BaseTool ): Promise { const packageCount = options.input.packageList.length; const packagePlurality = packageCount === 1 ? 'package' : 'packages'; + this.extraTelemetryProperties.packageCount = String(packageCount); const notebookResponse = getToolResponseIfNotebook(resourcePath); if (notebookResponse) { return notebookResponse; @@ -84,9 +86,11 @@ export class InstallPackagesTool extends BaseTool 'noEnvFound', ); } + this.extraTelemetryProperties.envType = getEnvTypeForTelemetry(environment); const isConda = isCondaEnv(environment); const installers = this.serviceContainer.getAll(IModuleInstaller); const installerType = isConda ? ModuleInstallerType.Conda : ModuleInstallerType.Pip; + this.extraTelemetryProperties.installerType = isConda ? 'conda' : 'pip'; const installer = installers.find((i) => i.type === installerType); if (!installer) { throw new ErrorWithTelemetrySafeReason( diff --git a/src/client/chat/utils.ts b/src/client/chat/utils.ts index 84df2901341b..2309316bcbdd 100644 --- a/src/client/chat/utils.ts +++ b/src/client/chat/utils.ts @@ -76,6 +76,10 @@ export function isCondaEnv(env: ResolvedEnvironment) { return (env.environment?.type || '').toLowerCase() === 'conda'; } +export function getEnvTypeForTelemetry(env: ResolvedEnvironment): string { + return (env.environment?.type || 'unknown').toLowerCase(); +} + export async function getEnvironmentDetails( resourcePath: Uri | undefined, api: PythonExtension['environments'], diff --git a/src/client/telemetry/index.ts b/src/client/telemetry/index.ts index 738c5f8a2776..c5faec51d01d 100644 --- a/src/client/telemetry/index.ts +++ b/src/client/telemetry/index.ts @@ -1992,7 +1992,12 @@ export interface IEventNamePropertyMapping { "duration" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "owner": "donjayamanne" }, "toolName" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "donjayamanne" }, "failed": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Whether there was a failure. Common to most of the events.", "owner": "donjayamanne" }, - "failureCategory": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"A reason that we generate (e.g. kerneldied, noipykernel, etc), more like a category of the error. Common to most of the events.", "owner": "donjayamanne" } + "failureCategory": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"A reason that we generate (e.g. kerneldied, noipykernel, etc), more like a category of the error. Common to most of the events.", "owner": "donjayamanne" }, + "resolveOutcome": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Which code path resolved the environment in configure_python_environment.", "owner": "donjayamanne" }, + "envType": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"The type of Python environment (e.g. venv, conda, system).", "owner": "donjayamanne" }, + "packageCount": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Number of packages requested for installation (install_python_packages only).", "owner": "donjayamanne" }, + "installerType": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Which installer was used: pip or conda (install_python_packages only).", "owner": "donjayamanne" }, + "responsePackageCount": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Number of packages in the environment response (get_python_environment_details only).", "owner": "donjayamanne" } } */ [EventName.INVOKE_TOOL]: { @@ -2009,6 +2014,26 @@ export interface IEventNamePropertyMapping { * A reason the error was thrown. */ failureCategory?: string; + /** + * Which code path resolved the environment (configure_python_environment only). + */ + resolveOutcome?: string; + /** + * The type of Python environment (e.g. venv, conda, system). + */ + envType?: string; + /** + * Number of packages requested for installation (install_python_packages only). + */ + packageCount?: string; + /** + * Which installer was used: pip or conda (install_python_packages only). + */ + installerType?: string; + /** + * Number of packages in the environment response (get_python_environment_details only). + */ + responsePackageCount?: string; }; /** * Telemetry event sent if and when user configure tests command. This command can be trigerred from multiple places in the extension. (Command palette, prompt etc.)