qwen-code/packages/core/src/telemetry/loggers.ts

302 lines
8.2 KiB
TypeScript
Raw Normal View History

/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { logs, LogRecord, LogAttributes } from '@opentelemetry/api-logs';
import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
import { Config } from '../config/config.js';
import {
EVENT_API_ERROR,
EVENT_API_REQUEST,
EVENT_API_RESPONSE,
EVENT_CLI_CONFIG,
EVENT_TOOL_CALL,
EVENT_USER_PROMPT,
SERVICE_NAME,
} from './constants.js';
import {
ApiErrorEvent,
ApiRequestEvent,
ApiResponseEvent,
ToolCallEvent,
UserPromptEvent,
} from './types.js';
import {
recordApiErrorMetrics,
2025-06-11 06:56:53 +00:00
recordTokenUsageMetrics,
recordApiResponseMetrics,
recordToolCallMetrics,
} from './metrics.js';
import { isTelemetrySdkInitialized } from './sdk.js';
import { ToolConfirmationOutcome } from '../index.js';
import {
GenerateContentResponse,
GenerateContentResponseUsageMetadata,
} from '@google/genai';
import { AuthType } from '../core/contentGenerator.js';
const shouldLogUserPrompts = (config: Config): boolean =>
Add telemetry command and refactor telemetry settings (#1060) #750 ### Telemetry Settings Refactors telemetry configuration to use a nested `telemetry` object in `settings.json`, for example: ```json { "telemetry": { "enabled": true, "target": "gcp" "log-prompts": "true" }, "sandbox": false } ``` The above includes - Centralized telemetry settings under a `telemetry` object in `settings.json`. - CLI flags for the `gemini` command to override all telemetry sub-settings: - `--telemetry` / `--no-telemetry` - `--telemetry-target <local|gcp>` - `--telemetry-otlp-endpoint <URL>` - `--telemetry-log-prompts` / `--no-telemetry-log-prompts` - Updates `packages/cli/src/config/config.ts` and `packages/core/src/config/config.ts` to read from the new settings structure and respect the new CLI flags. - Modifies `scripts/handle-telemetry.js`, `scripts/local_telemetry.js`, and `scripts/telemetry_utils.js` to align with the new settings structure. - Updates `docs/core/telemetry.md` to reflect the new settings structure, CLI flags, and order of precedence. - Renames `logUserPromptsEnabled` to `logPrompts` for brevity. ### `npm run telemetry` Add a new `npm run telemetry` command that uses `scripts/telemetry.js`, automates the entire process of setting up a local and GCP telemetry pipelines, including configuring the necessary settings in the `.gemini/settings.json` workspace file and installing required binaries (e.g. `otelcol-contrib`). --- ```shell $ npm run telemetry -- --target=gcp > gemini-cli@0.1.0 telemetry > node scripts/telemetry.js --target=gcp ⚙️ Using command-line target: gcp 🚀 Running telemetry script for target: gcp. ✨ Starting Local Telemetry Exporter for Google Cloud ✨ ⚙️ Enabled telemetry in workspace settings. 🔧 Set telemetry OTLP endpoint to http://localhost:4317. 🎯 Set telemetry target to gcp. ✅ Workspace settings updated. ✅ Using Google Cloud Project ID: foo-bar 🔑 Please ensure you are authenticated with Google Cloud: - Run `gcloud auth application-default login` OR ensure `GOOGLE_APPLICATION_CREDENTIALS` environment variable points to a valid service account key. - The account needs "Cloud Trace Agent", "Monitoring Metric Writer", and "Logs Writer" roles. ✅ otelcol-contrib already exists at /Users/jerop/github/gemini-cli/.gemini/otel/bin/otelcol-contrib 🧹 Cleaning up old processes and logs... ✅ Deleted old GCP collector log. 📄 Wrote OTEL collector config to /Users/jerop/github/gemini-cli/.gemini/otel/collector-gcp.yaml 🚀 Starting OTEL collector for GCP... Logs: /Users/jerop/github/gemini-cli/.gemini/otel/collector-gcp.log ⏳ Waiting for OTEL collector to start (PID: 17013)... ✅ OTEL collector started successfully on port 4317. ✨ Local OTEL collector for GCP is running. 🚀 To send telemetry, run the Gemini CLI in a separate terminal window. 📄 Collector logs are being written to: /Users/jerop/github/gemini-cli/.gemini/otel/collector-gcp.log 📊 View your telemetry data in Google Cloud Console: - Logs: https://console.cloud.google.com/logs/query;query=logName%3D%22projects%2Ffoo-bar%2Flogs%2Fgemini_cli%22?project=foo-bar - Metrics: https://console.cloud.google.com/monitoring/metrics-explorer?project=foo-bar - Traces: https://console.cloud.google.com/traces/list?project=foo-bar Press Ctrl+C to exit. ^C 👋 Shutting down... ⚙️ Disabled telemetry in workspace settings. 🔧 Cleared telemetry OTLP endpoint. 🎯 Cleared telemetry target. ✅ Workspace settings updated. 🛑 Stopping otelcol-contrib (PID: 17013)... ✅ otelcol-contrib stopped. ```
2025-06-15 00:47:32 -04:00
config.getTelemetryLogPromptsEnabled() ?? false;
function getCommonAttributes(config: Config): LogAttributes {
return {
'session.id': config.getSessionId(),
};
}
export enum ToolCallDecision {
ACCEPT = 'accept',
REJECT = 'reject',
MODIFY = 'modify',
}
export function getDecisionFromOutcome(
outcome: ToolConfirmationOutcome,
): ToolCallDecision {
switch (outcome) {
case ToolConfirmationOutcome.ProceedOnce:
case ToolConfirmationOutcome.ProceedAlways:
case ToolConfirmationOutcome.ProceedAlwaysServer:
case ToolConfirmationOutcome.ProceedAlwaysTool:
return ToolCallDecision.ACCEPT;
case ToolConfirmationOutcome.ModifyWithEditor:
return ToolCallDecision.MODIFY;
case ToolConfirmationOutcome.Cancel:
default:
return ToolCallDecision.REJECT;
}
}
export function logCliConfiguration(config: Config): void {
if (!isTelemetrySdkInitialized()) return;
const generatorConfig = config.getContentGeneratorConfig();
let useGemini = false;
let useVertex = false;
if (generatorConfig && generatorConfig.authType) {
useGemini = generatorConfig.authType === AuthType.USE_GEMINI;
useVertex = generatorConfig.authType === AuthType.USE_VERTEX_AI;
}
const mcpServers = config.getMcpServers();
const attributes: LogAttributes = {
...getCommonAttributes(config),
'event.name': EVENT_CLI_CONFIG,
'event.timestamp': new Date().toISOString(),
model: config.getModel(),
embedding_model: config.getEmbeddingModel(),
sandbox_enabled: !!config.getSandbox(),
core_tools_enabled: (config.getCoreTools() ?? []).join(','),
approval_mode: config.getApprovalMode(),
api_key_enabled: useGemini || useVertex,
vertex_ai_enabled: useVertex,
Add telemetry command and refactor telemetry settings (#1060) #750 ### Telemetry Settings Refactors telemetry configuration to use a nested `telemetry` object in `settings.json`, for example: ```json { "telemetry": { "enabled": true, "target": "gcp" "log-prompts": "true" }, "sandbox": false } ``` The above includes - Centralized telemetry settings under a `telemetry` object in `settings.json`. - CLI flags for the `gemini` command to override all telemetry sub-settings: - `--telemetry` / `--no-telemetry` - `--telemetry-target <local|gcp>` - `--telemetry-otlp-endpoint <URL>` - `--telemetry-log-prompts` / `--no-telemetry-log-prompts` - Updates `packages/cli/src/config/config.ts` and `packages/core/src/config/config.ts` to read from the new settings structure and respect the new CLI flags. - Modifies `scripts/handle-telemetry.js`, `scripts/local_telemetry.js`, and `scripts/telemetry_utils.js` to align with the new settings structure. - Updates `docs/core/telemetry.md` to reflect the new settings structure, CLI flags, and order of precedence. - Renames `logUserPromptsEnabled` to `logPrompts` for brevity. ### `npm run telemetry` Add a new `npm run telemetry` command that uses `scripts/telemetry.js`, automates the entire process of setting up a local and GCP telemetry pipelines, including configuring the necessary settings in the `.gemini/settings.json` workspace file and installing required binaries (e.g. `otelcol-contrib`). --- ```shell $ npm run telemetry -- --target=gcp > gemini-cli@0.1.0 telemetry > node scripts/telemetry.js --target=gcp ⚙️ Using command-line target: gcp 🚀 Running telemetry script for target: gcp. ✨ Starting Local Telemetry Exporter for Google Cloud ✨ ⚙️ Enabled telemetry in workspace settings. 🔧 Set telemetry OTLP endpoint to http://localhost:4317. 🎯 Set telemetry target to gcp. ✅ Workspace settings updated. ✅ Using Google Cloud Project ID: foo-bar 🔑 Please ensure you are authenticated with Google Cloud: - Run `gcloud auth application-default login` OR ensure `GOOGLE_APPLICATION_CREDENTIALS` environment variable points to a valid service account key. - The account needs "Cloud Trace Agent", "Monitoring Metric Writer", and "Logs Writer" roles. ✅ otelcol-contrib already exists at /Users/jerop/github/gemini-cli/.gemini/otel/bin/otelcol-contrib 🧹 Cleaning up old processes and logs... ✅ Deleted old GCP collector log. 📄 Wrote OTEL collector config to /Users/jerop/github/gemini-cli/.gemini/otel/collector-gcp.yaml 🚀 Starting OTEL collector for GCP... Logs: /Users/jerop/github/gemini-cli/.gemini/otel/collector-gcp.log ⏳ Waiting for OTEL collector to start (PID: 17013)... ✅ OTEL collector started successfully on port 4317. ✨ Local OTEL collector for GCP is running. 🚀 To send telemetry, run the Gemini CLI in a separate terminal window. 📄 Collector logs are being written to: /Users/jerop/github/gemini-cli/.gemini/otel/collector-gcp.log 📊 View your telemetry data in Google Cloud Console: - Logs: https://console.cloud.google.com/logs/query;query=logName%3D%22projects%2Ffoo-bar%2Flogs%2Fgemini_cli%22?project=foo-bar - Metrics: https://console.cloud.google.com/monitoring/metrics-explorer?project=foo-bar - Traces: https://console.cloud.google.com/traces/list?project=foo-bar Press Ctrl+C to exit. ^C 👋 Shutting down... ⚙️ Disabled telemetry in workspace settings. 🔧 Cleared telemetry OTLP endpoint. 🎯 Cleared telemetry target. ✅ Workspace settings updated. 🛑 Stopping otelcol-contrib (PID: 17013)... ✅ otelcol-contrib stopped. ```
2025-06-15 00:47:32 -04:00
log_user_prompts_enabled: config.getTelemetryLogPromptsEnabled(),
file_filtering_respect_git_ignore:
config.getFileFilteringRespectGitIgnore(),
debug_mode: config.getDebugMode(),
mcp_servers: mcpServers ? Object.keys(mcpServers).join(',') : '',
};
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: 'CLI configuration loaded.',
attributes,
};
logger.emit(logRecord);
}
export function logUserPrompt(
config: Config,
event: Omit<UserPromptEvent, 'event.name' | 'event.timestamp' | 'prompt'> & {
prompt: string;
},
): void {
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
'event.name': EVENT_USER_PROMPT,
'event.timestamp': new Date().toISOString(),
prompt_length: event.prompt_length,
};
if (shouldLogUserPrompts(config)) {
attributes.prompt = event.prompt;
}
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `User prompt. Length: ${event.prompt_length}.`,
attributes,
};
logger.emit(logRecord);
}
export function logToolCall(
config: Config,
event: Omit<ToolCallEvent, 'event.name' | 'event.timestamp' | 'decision'>,
outcome?: ToolConfirmationOutcome,
): void {
if (!isTelemetrySdkInitialized()) return;
const decision = outcome ? getDecisionFromOutcome(outcome) : undefined;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_TOOL_CALL,
'event.timestamp': new Date().toISOString(),
function_args: JSON.stringify(event.function_args, null, 2),
decision,
};
if (event.error) {
attributes['error.message'] = event.error;
if (event.error_type) {
attributes['error.type'] = event.error_type;
}
}
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `Tool call: ${event.function_name}${decision ? `. Decision: ${decision}` : ''}. Success: ${event.success}. Duration: ${event.duration_ms}ms.`,
attributes,
};
logger.emit(logRecord);
recordToolCallMetrics(
config,
event.function_name,
event.duration_ms,
event.success,
decision,
);
}
export function logApiRequest(
config: Config,
event: Omit<ApiRequestEvent, 'event.name' | 'event.timestamp'>,
): void {
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_API_REQUEST,
'event.timestamp': new Date().toISOString(),
};
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `API request to ${event.model}.`,
attributes,
};
logger.emit(logRecord);
}
export function logApiError(
config: Config,
event: Omit<ApiErrorEvent, 'event.name' | 'event.timestamp'>,
): void {
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_API_ERROR,
'event.timestamp': new Date().toISOString(),
['error.message']: event.error,
};
if (event.error_type) {
attributes['error.type'] = event.error_type;
}
if (typeof event.status_code === 'number') {
attributes[SemanticAttributes.HTTP_STATUS_CODE] = event.status_code;
}
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `API error for ${event.model}. Error: ${event.error}. Duration: ${event.duration_ms}ms.`,
attributes,
};
logger.emit(logRecord);
recordApiErrorMetrics(
config,
event.model,
event.duration_ms,
event.status_code,
event.error_type,
);
}
export function logApiResponse(
config: Config,
event: Omit<ApiResponseEvent, 'event.name' | 'event.timestamp'>,
): void {
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_API_RESPONSE,
'event.timestamp': new Date().toISOString(),
};
if (event.response_text) {
attributes.response_text = event.response_text;
}
if (event.error) {
attributes['error.message'] = event.error;
} else if (event.status_code) {
if (typeof event.status_code === 'number') {
attributes[SemanticAttributes.HTTP_STATUS_CODE] = event.status_code;
}
}
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `API response from ${event.model}. Status: ${event.status_code || 'N/A'}. Duration: ${event.duration_ms}ms.`,
attributes,
};
logger.emit(logRecord);
recordApiResponseMetrics(
config,
event.model,
event.duration_ms,
event.status_code,
event.error,
);
recordTokenUsageMetrics(
config,
event.model,
event.input_token_count,
'input',
);
2025-06-11 06:56:53 +00:00
recordTokenUsageMetrics(
config,
event.model,
event.output_token_count,
'output',
);
recordTokenUsageMetrics(
config,
2025-06-11 06:56:53 +00:00
event.model,
event.cached_content_token_count,
'cache',
);
recordTokenUsageMetrics(
config,
event.model,
event.thoughts_token_count,
'thought',
);
recordTokenUsageMetrics(config, event.model, event.tool_token_count, 'tool');
}
export function getFinalUsageMetadata(
chunks: GenerateContentResponse[],
): GenerateContentResponseUsageMetadata | undefined {
// Only the last streamed item has the final token count.
const lastChunkWithMetadata = chunks
.slice()
.reverse()
.find((chunk) => chunk.usageMetadata);
return lastChunkWithMetadata?.usageMetadata;
}