diff --git a/cli/azd/.vscode/cspell.yaml b/cli/azd/.vscode/cspell.yaml index f0a25445ce5..3a53d765055 100644 --- a/cli/azd/.vscode/cspell.yaml +++ b/cli/azd/.vscode/cspell.yaml @@ -177,6 +177,9 @@ dictionaryDefinitions: dictionaries: - azdProjectDictionary overrides: + - filename: "**/internal/agent/types.go" + words: + - billionized - filename: internal/appdetect/aspire_polyglot.go words: - pylock diff --git a/cli/azd/cmd/telemetry_test.go b/cli/azd/cmd/telemetry_test.go index cea6bf3b082..60c6cea35ff 100644 --- a/cli/azd/cmd/telemetry_test.go +++ b/cli/azd/cmd/telemetry_test.go @@ -132,6 +132,7 @@ func TestTelemetryFieldConstants(t *testing.T) { measurementFields := []fields.AttributeKey{ fields.AgentFixAttempts, + fields.CopilotMessageAICredits, fields.ExeGraphDeployConcurrencyKey, fields.ExeGraphMaxConcurrencyKey, fields.ExeGraphPackageConcurrencyKey, diff --git a/cli/azd/docs/extensions/extension-framework.md b/cli/azd/docs/extensions/extension-framework.md index c25b4736b78..d6102632b8e 100644 --- a/cli/azd/docs/extensions/extension-framework.md +++ b/cli/azd/docs/extensions/extension-framework.md @@ -3128,8 +3128,9 @@ Returns cumulative usage metrics cached for a session. - `input_tokens` (double): Total input tokens consumed - `output_tokens` (double): Total output tokens consumed - `total_tokens` (double): Sum of input + output tokens - - `billing_rate` (double): Per-request cost multiplier (e.g., 1.0x, 2.0x) - - `premium_requests` (double): Number of premium requests used + - `billing_rate` (double, deprecated): Legacy per-request cost multiplier; use `ai_credits` instead + - `premium_requests` (double, deprecated): Legacy premium request count; use `ai_credits` instead + - `ai_credits` (double): Total AI credits consumed - `duration_ms` (double): Total API duration in milliseconds #### GetFileChanges @@ -3223,8 +3224,8 @@ metricsResp, err := copilot.GetUsageMetrics(ctx, &v1beta.GetCopilotUsageMetricsR if err != nil { return fmt.Errorf("failed to get metrics: %w", err) } -fmt.Printf("Total tokens: %.0f, Premium requests: %.0f\n", - metricsResp.Usage.TotalTokens, metricsResp.Usage.PremiumRequests) +fmt.Printf("Total tokens: %.0f, AI credits: %.2f AIC\n", + metricsResp.Usage.TotalTokens, metricsResp.Usage.AiCredits) // Retrieve file changes changesResp, err := copilot.GetFileChanges(ctx, &v1beta.GetCopilotFileChangesRequest{ diff --git a/cli/azd/extensions/microsoft.azd.demo/internal/cmd/copilot.go b/cli/azd/extensions/microsoft.azd.demo/internal/cmd/copilot.go index 50de2c77c02..04cf4e89ef3 100644 --- a/cli/azd/extensions/microsoft.azd.demo/internal/cmd/copilot.go +++ b/cli/azd/extensions/microsoft.azd.demo/internal/cmd/copilot.go @@ -279,7 +279,7 @@ func showCumulativeMetrics( } usage := metricsResp.Usage - if usage == nil || (usage.InputTokens == 0 && usage.OutputTokens == 0) { + if !hasUsageMetrics(usage) { return } @@ -294,18 +294,20 @@ func showCumulativeMetrics( color.HiBlackString("•"), formatTokens(usage.OutputTokens)) fmt.Printf(" %s Total tokens: %s\n", color.HiBlackString("•"), formatTokens(usage.TotalTokens)) - if usage.BillingRate > 0 { - fmt.Printf(" %s Billing rate: %.0fx per request\n", - color.HiBlackString("•"), usage.BillingRate) + if usage.AiCredits > 0 { + fmt.Printf(" %s AI credits: %s\n", + color.HiBlackString("•"), fmt.Sprintf("%.2f", usage.AiCredits)) } - fmt.Printf(" %s Premium requests: %.0f\n", - color.HiBlackString("•"), usage.PremiumRequests) if usage.DurationMs > 0 { fmt.Printf(" %s API duration: %s\n", color.HiBlackString("•"), formatDuration(usage.DurationMs)) } } +func hasUsageMetrics(usage *v1beta.CopilotUsageMetrics) bool { + return usage != nil && (usage.InputTokens != 0 || usage.OutputTokens != 0 || usage.AiCredits != 0) +} + // showFileChanges displays accumulated file changes from the session. func showFileChanges( ctx context.Context, diff --git a/cli/azd/extensions/microsoft.azd.demo/internal/cmd/copilot_test.go b/cli/azd/extensions/microsoft.azd.demo/internal/cmd/copilot_test.go new file mode 100644 index 00000000000..f5ed4d3ef82 --- /dev/null +++ b/cli/azd/extensions/microsoft.azd.demo/internal/cmd/copilot_test.go @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/require" + + v1beta "github.com/azure/azure-dev/cli/azd/pkg/azdext/contracts/v1beta" +) + +func TestHasUsageMetrics(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + usage *v1beta.CopilotUsageMetrics + expected bool + }{ + {name: "Nil", expected: false}, + {name: "Empty", usage: &v1beta.CopilotUsageMetrics{}, expected: false}, + {name: "InputTokens", usage: &v1beta.CopilotUsageMetrics{InputTokens: 1}, expected: true}, + {name: "OutputTokens", usage: &v1beta.CopilotUsageMetrics{OutputTokens: 1}, expected: true}, + {name: "AICredits", usage: &v1beta.CopilotUsageMetrics{AiCredits: 0.25}, expected: true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + require.Equal(t, test.expected, hasUsageMetrics(test.usage)) + }) + } +} diff --git a/cli/azd/grpc/proto/azd/extensions/v1/ai_model.proto b/cli/azd/grpc/proto/azd/extensions/v1/ai_model.proto index 185bb904e27..ee856756ac9 100644 --- a/cli/azd/grpc/proto/azd/extensions/v1/ai_model.proto +++ b/cli/azd/grpc/proto/azd/extensions/v1/ai_model.proto @@ -40,7 +40,8 @@ service AiModelService { message AiModel { string name = 1; // e.g. "gpt-4o" string format = 2; // e.g. "OpenAI" - string lifecycle_status = 3 [deprecated = true]; // deprecated; always empty; use AiModelVersion.lifecycle_status + // Always empty. Use AiModelVersion.lifecycle_status instead. + string lifecycle_status = 3 [deprecated = true]; repeated string capabilities = 4; // e.g. ["chat", "embeddings"] repeated AiModelVersion versions = 5; repeated string locations = 6; // canonical locations where available diff --git a/cli/azd/grpc/proto/azd/extensions/v1beta/copilot.proto b/cli/azd/grpc/proto/azd/extensions/v1beta/copilot.proto index d0104cc1b6d..c9a47403f73 100644 --- a/cli/azd/grpc/proto/azd/extensions/v1beta/copilot.proto +++ b/cli/azd/grpc/proto/azd/extensions/v1beta/copilot.proto @@ -118,9 +118,12 @@ message CopilotUsageMetrics { double input_tokens = 2; // Total input tokens consumed. double output_tokens = 3; // Total output tokens consumed. double total_tokens = 4; // Sum of input + output tokens. - double billing_rate = 5; // Per-request cost multiplier (e.g., 1.0x, 2.0x). - double premium_requests = 6; // Number of premium requests used. + // Legacy per-request cost multiplier. Use ai_credits instead. + double billing_rate = 5 [deprecated = true]; + // Legacy premium request count. Use ai_credits instead. + double premium_requests = 6 [deprecated = true]; double duration_ms = 7; // Total API duration in milliseconds. + double ai_credits = 8; // Total AI credits consumed. } // --- File change messages --- diff --git a/cli/azd/internal/agent/copilot/copilot_sdk_e2e_test.go b/cli/azd/internal/agent/copilot/copilot_sdk_e2e_test.go index b540428903c..707f3fec5a5 100644 --- a/cli/azd/internal/agent/copilot/copilot_sdk_e2e_test.go +++ b/cli/azd/internal/agent/copilot/copilot_sdk_e2e_test.go @@ -1,19 +1,25 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -package copilot +package copilot_test import ( "context" + "encoding/json" "fmt" "net/http" "os" + "slices" + "strings" + "sync" "testing" "time" copilot "github.com/github/copilot-sdk/go" "github.com/stretchr/testify/require" + "github.com/azure/azure-dev/cli/azd/internal/agent" + agentcopilot "github.com/azure/azure-dev/cli/azd/internal/agent/copilot" "github.com/azure/azure-dev/cli/azd/test/mocks/mockinput" ) @@ -39,8 +45,8 @@ func TestCopilotSDK_E2E(t *testing.T) { t.Setenv("AZD_COPILOT_CLI_PATH", "") // 1. Download the pinned CLI and start it through azd's client manager. - cli := NewCopilotCLI(mockinput.NewMockConsole(), nil, http.DefaultClient) - clientManager := NewCopilotClientManager(&CopilotClientOptions{ + cli := agentcopilot.NewCopilotCLI(mockinput.NewMockConsole(), nil, http.DefaultClient) + clientManager := agentcopilot.NewCopilotClientManager(&agentcopilot.CopilotClientOptions{ LogLevel: "error", }, cli) @@ -58,7 +64,7 @@ func TestCopilotSDK_E2E(t *testing.T) { // 2. Check auth auth, err := clientManager.GetAuthStatus(ctx) require.NoError(t, err) - t.Logf("Auth: authenticated=%v, login=%v", auth.IsAuthenticated, auth.Login) + t.Logf("Auth: authenticated=%v", auth.IsAuthenticated) require.True(t, auth.IsAuthenticated, "not authenticated with GitHub Copilot") // 3. List models @@ -81,7 +87,7 @@ func TestCopilotSDK_E2E(t *testing.T) { OnPermissionRequest: copilot.PermissionHandler.ApproveAll, }) require.NoError(t, err, "CreateSession failed") - t.Logf("Session created: %s", session.WorkspacePath()) + t.Log("Session created") defer func() { if disconnectErr := session.Disconnect(); disconnectErr != nil { t.Logf("session.Destroy error: %v", disconnectErr) @@ -89,50 +95,174 @@ func TestCopilotSDK_E2E(t *testing.T) { }() // 5. Collect events - var events []copilot.SessionEvent + collector := agent.NewHeadlessCollector() + captured := &capturedEvents{} unsubscribe := session.On(func(event copilot.SessionEvent) { - events = append(events, event) t.Logf("Event: type=%s", event.Type()) + aiuFields, err := formatAIUFields(event) + captured.Add(event, err) + if aiuFields != "" { + t.Logf("AIU event: type=%s %s", event.Type(), aiuFields) + } + collector.HandleEvent(event) }) defer unsubscribe() - // 6. Send message and wait for response - t.Log("Sending prompt...") - response, err := session.SendAndWait(ctx, copilot.MessageOptions{ + // 6. Send two messages in the same session and wait for each turn to become idle. + t.Log("Sending first prompt...") + firstEvent := captured.Len() + firstResponse, err := session.SendAndWait(ctx, copilot.MessageOptions{ Prompt: "What is 2+2? Reply with just the number.", }) - require.NoError(t, err, "SendAndWait failed") + require.NoError(t, err, "first SendAndWait failed") + require.NoError(t, collector.WaitForIdle(ctx), "collector did not observe first session idle") + require.NoError(t, captured.Err()) + requireResponseContains(t, firstResponse, captured.Since(firstEvent), "4") + + firstUsage := collector.GetUsageMetrics() + require.Positive(t, firstUsage.AICredits, "expected positive AI credit usage after first turn") + + t.Log("Sending follow-up prompt...") + secondEvent := captured.Len() + secondResponse, err := session.SendAndWait(ctx, copilot.MessageOptions{ + Prompt: "Add 3 to the number you answered in the previous turn. Reply with just the result.", + }) + require.NoError(t, err, "second SendAndWait failed") + require.NoError(t, collector.WaitForIdle(ctx), "collector did not observe second session idle") + require.NoError(t, captured.Err()) + requireResponseContains(t, secondResponse, captured.Since(secondEvent), "7") + + usage := collector.GetUsageMetrics() + require.Greater(t, usage.InputTokens, firstUsage.InputTokens) + require.Greater(t, usage.OutputTokens, firstUsage.OutputTokens) + require.Greater(t, usage.AICredits, firstUsage.AICredits) + t.Logf("Usage: input=%v output=%v AI credits=%v", usage.InputTokens, usage.OutputTokens, usage.AICredits) // 7. Validate response - t.Logf("Received %d events total", len(events)) + t.Logf("Received %d events total", captured.Len()) +} + +type capturedEvents struct { + mu sync.Mutex + events []copilot.SessionEvent + err error +} + +func (c *capturedEvents) Add(event copilot.SessionEvent, err error) { + c.mu.Lock() + defer c.mu.Unlock() + + c.events = append(c.events, event) + if c.err == nil { + c.err = err + } +} + +func (c *capturedEvents) Len() int { + c.mu.Lock() + defer c.mu.Unlock() + + return len(c.events) +} + +func (c *capturedEvents) Since(index int) []copilot.SessionEvent { + c.mu.Lock() + defer c.mu.Unlock() + + return slices.Clone(c.events[index:]) +} + +func (c *capturedEvents) Err() error { + c.mu.Lock() + defer c.mu.Unlock() + + return c.err +} + +func requireResponseContains( + t *testing.T, + response *copilot.SessionEvent, + events []copilot.SessionEvent, + expected string, +) { + t.Helper() + if response != nil { data, ok := response.Data.(*copilot.AssistantMessageData) require.True(t, ok, "expected response.Data to be *copilot.AssistantMessageData, got %T", response.Data) t.Logf("Response content: %s", data.Content) - require.Contains(t, data.Content, "4", - "expected response to contain '4'") + require.Contains(t, data.Content, expected) } else { - // If SendAndWait returned nil, check events for assistant message var found bool - for _, e := range events { - if e.Type() == copilot.SessionEventTypeAssistantMessage { - if data, ok := e.Data.(*copilot.AssistantMessageData); ok { + for _, event := range events { + if event.Type() == copilot.SessionEventTypeAssistantMessage { + if data, ok := event.Data.(*copilot.AssistantMessageData); ok { t.Logf("Found assistant message in events: %s", data.Content) - found = true - break + if strings.Contains(data.Content, expected) { + found = true + break + } } } } if !found { - // Log all event types for debugging - for _, e := range events { + for _, event := range events { detail := "" - if data, ok := e.Data.(*copilot.AssistantMessageData); ok { + if data, ok := event.Data.(*copilot.AssistantMessageData); ok { detail = fmt.Sprintf(" content=%s", truncateForLog(data.Content, 100)) } - t.Logf(" event: type=%s%s", e.Type(), detail) + t.Logf(" event: type=%s%s", event.Type(), detail) + } + t.Fatalf("no assistant message containing %q received", expected) + } + } +} + +func formatAIUFields(event copilot.SessionEvent) (string, error) { + encoded, err := json.Marshal(event.Data) + if err != nil { + return "", fmt.Errorf("marshaling %s event data: %w", event.Type(), err) + } + + var data any + if err := json.Unmarshal(encoded, &data); err != nil { + return "", fmt.Errorf("unmarshaling %s event data: %w", event.Type(), err) + } + + var fields []string + collectAIUFields(data, "data", &fields) + if len(fields) == 0 { + return "", nil + } + if usage, ok := event.Data.(*copilot.AssistantUsageData); ok { + if usage.InputTokens != nil { + fields = append(fields, fmt.Sprintf("data.inputTokens=%d", *usage.InputTokens)) + } + if usage.OutputTokens != nil { + fields = append(fields, fmt.Sprintf("data.outputTokens=%d", *usage.OutputTokens)) + } + } + + slices.Sort(fields) + return strings.Join(fields, ", "), nil +} + +func collectAIUFields(value any, path string, fields *[]string) { + switch value := value.(type) { + case map[string]any: + for name, child := range value { + childPath := path + "." + name + if name == "totalNanoAiu" { + if number, ok := child.(float64); ok { + *fields = append(*fields, fmt.Sprintf("%s=%g", childPath, number)) + } + continue } - t.Fatal("no assistant message received") + collectAIUFields(child, childPath, fields) + } + case []any: + for _, child := range value { + collectAIUFields(child, path+"[]", fields) } } } diff --git a/cli/azd/internal/agent/copilot_agent.go b/cli/azd/internal/agent/copilot_agent.go index 6d0b9cdd518..a252bdc1ced 100644 --- a/cli/azd/internal/agent/copilot_agent.go +++ b/cli/azd/internal/agent/copilot_agent.go @@ -56,7 +56,7 @@ type CopilotAgent struct { // Runtime state clientStarted bool - session *copilot.Session + session copilotSession sessionID string activeCtx atomic.Pointer[context.Context] // current SendMessage context for SDK callbacks display *AgentDisplay // last display for usage metrics (interactive mode) @@ -76,6 +76,12 @@ type cleanupTask struct { fn func() error } +type copilotSession interface { + On(copilot.SessionEventHandler) func() + Send(context.Context, copilot.MessageOptions) (string, error) + GetEvents(context.Context) ([]copilot.SessionEvent, error) +} + const ( selectAIModelMessage = "Select AI model" selectReasoningEffortLevelMessage = "Select reasoning effort level" @@ -271,16 +277,21 @@ func (a *CopilotAgent) sendMessageInteractive( return nil, fmt.Errorf("copilot agent error: %w", err) } - if err := display.WaitForIdle(ctx); err != nil { - return nil, err - } - + err = display.WaitForIdle(ctx) turnUsage := display.GetUsageMetrics() + a.mu.Lock() a.accumulateUsage(turnUsage) - turnFileChanges := a.collectFileChanges(watcher) + var turnFileChanges watch.FileChanges + if err == nil { + turnFileChanges = a.collectFileChanges(watcher) + } a.mu.Unlock() + if err != nil { + return nil, err + } + return &AgentResult{ SessionID: a.sessionID, Usage: turnUsage, @@ -312,16 +323,21 @@ func (a *CopilotAgent) sendMessageHeadless( return nil, fmt.Errorf("copilot agent error: %w", err) } - if err := collector.WaitForIdle(ctx); err != nil { - return nil, err - } - + err = collector.WaitForIdle(ctx) turnUsage := collector.GetUsageMetrics() + a.mu.Lock() a.accumulateUsage(turnUsage) - turnFileChanges := a.collectFileChanges(watcher) + var turnFileChanges watch.FileChanges + if err == nil { + turnFileChanges = a.collectFileChanges(watcher) + } a.mu.Unlock() + if err != nil { + return nil, err + } + return &AgentResult{ SessionID: a.sessionID, Usage: turnUsage, @@ -333,6 +349,7 @@ func (a *CopilotAgent) sendMessageHeadless( func (a *CopilotAgent) accumulateUsage(turn UsageMetrics) { a.cumulativeUsage.InputTokens += turn.InputTokens a.cumulativeUsage.OutputTokens += turn.OutputTokens + a.cumulativeUsage.AICredits += turn.AICredits a.cumulativeUsage.DurationMS += turn.DurationMS a.cumulativeUsage.PremiumRequests += turn.PremiumRequests // These are per-request values, not cumulative — use latest @@ -405,8 +422,7 @@ func (a *CopilotAgent) Stop() error { fields.CopilotMessageModel.String(a.cumulativeUsage.Model), fields.CopilotMessageInputTokens.Float64(a.cumulativeUsage.InputTokens), fields.CopilotMessageOutputTokens.Float64(a.cumulativeUsage.OutputTokens), - fields.CopilotMessageBillingRate.Float64(a.cumulativeUsage.BillingRate), - fields.CopilotMessagePremiumRequests.Float64(a.cumulativeUsage.PremiumRequests), + fields.CopilotMessageAICredits.Float64(a.cumulativeUsage.AICredits), fields.CopilotMessageDurationMs.Float64(a.cumulativeUsage.DurationMS), fields.CopilotConsentApprovedCount.Int(a.consentApprovedCount), fields.CopilotConsentDeniedCount.Int(a.consentDeniedCount), diff --git a/cli/azd/internal/agent/copilot_agent_test.go b/cli/azd/internal/agent/copilot_agent_test.go index b9ce2aa198e..57ad5ab16bf 100644 --- a/cli/azd/internal/agent/copilot_agent_test.go +++ b/cli/azd/internal/agent/copilot_agent_test.go @@ -14,11 +14,112 @@ import ( "github.com/stretchr/testify/require" agentcopilot "github.com/azure/azure-dev/cli/azd/internal/agent/copilot" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/config" "github.com/azure/azure-dev/cli/azd/pkg/input" "github.com/azure/azure-dev/cli/azd/test/mocks" ) +type fakeCopilotSession struct { + handler copilot.SessionEventHandler + send func(context.Context, copilot.MessageOptions) (string, error) +} + +func (s *fakeCopilotSession) On(handler copilot.SessionEventHandler) func() { + s.handler = handler + return func() { s.handler = nil } +} + +func (s *fakeCopilotSession) Send(ctx context.Context, options copilot.MessageOptions) (string, error) { + return s.send(ctx, options) +} + +func (s *fakeCopilotSession) GetEvents(context.Context) ([]copilot.SessionEvent, error) { + return nil, nil +} + +func TestCopilotAgentStopRecordsAICredits(t *testing.T) { + tracing.ResetUsageAttributesForTest() + t.Cleanup(tracing.ResetUsageAttributesForTest) + + agent := &CopilotAgent{ + cumulativeUsage: UsageMetrics{AICredits: 1.25}, + } + require.NoError(t, agent.Stop()) + + for _, attr := range tracing.GetUsageAttributes() { + if attr.Key == fields.CopilotMessageAICredits.Key { + require.Equal(t, 1.25, attr.Value.AsFloat64()) + return + } + } + require.Fail(t, "copilot AI-credit usage attribute was not recorded") +} + +func TestCopilotAgentAccumulateUsage(t *testing.T) { + agent := &CopilotAgent{} + + agent.accumulateUsage(UsageMetrics{ + Model: "gpt-4o", + InputTokens: 100, + OutputTokens: 50, + AICredits: 0.25, + BillingRate: 1.5, + PremiumRequests: 2, + DurationMS: 500, + }) + agent.accumulateUsage(UsageMetrics{ + Model: "gpt-4.1", + InputTokens: 200, + OutputTokens: 75, + AICredits: 0.5, + BillingRate: 2, + PremiumRequests: 3, + DurationMS: 1000, + }) + + usage := agent.GetMetrics().Usage + require.Equal(t, UsageMetrics{ + Model: "gpt-4.1", + InputTokens: 300, + OutputTokens: 125, + AICredits: 0.75, + BillingRate: 2, + PremiumRequests: 5, + DurationMS: 1500, + }, usage) +} + +func TestCopilotAgentSendMessageHeadlessRecordsUsageOnCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + session := &fakeCopilotSession{} + session.send = func(context.Context, copilot.MessageOptions) (string, error) { + session.handler(copilot.SessionEvent{ + Data: &copilot.AssistantUsageData{ + InputTokens: new(int64(100)), + OutputTokens: new(int64(50)), + CopilotUsage: &copilot.AssistantUsageCopilotUsage{TotalNanoAiu: 250_000_000}, + Duration: new(int64(1000)), + }, + }) + cancel() + return "", nil + } + + agent := &CopilotAgent{session: session, sessionID: "test-session"} + result, err := agent.sendMessageHeadless(ctx, "test", AgentModeAutopilot) + + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, result) + require.Equal(t, UsageMetrics{ + InputTokens: 100, + OutputTokens: 50, + AICredits: 0.25, + DurationMS: 1000, + }, agent.GetMetrics().Usage) +} + func TestCopilotAgentPromptModelAndReasoning(t *testing.T) { t.Run("ModelConfigurationAlreadyInConfig", func(t *testing.T) { runCopilotAgentTest(t, copilotAgentTestArgs{ diff --git a/cli/azd/internal/agent/display.go b/cli/azd/internal/agent/display.go index 6131a62fc2c..bf8689a5601 100644 --- a/cli/azd/internal/agent/display.go +++ b/cli/azd/internal/agent/display.go @@ -53,6 +53,7 @@ type AgentDisplay struct { // Usage metrics — accumulated from assistant.usage events totalInputTokens float64 totalOutputTokens float64 + totalNanoAiu float64 billingRate float64 totalDurationMS float64 premiumRequests float64 @@ -332,6 +333,9 @@ func (d *AgentDisplay) HandleEvent(event copilot.SessionEvent) { if data.OutputTokens != nil { d.totalOutputTokens += float64(*data.OutputTokens) } + if data.CopilotUsage != nil { + d.totalNanoAiu += data.CopilotUsage.TotalNanoAiu + } if data.Cost != nil { d.billingRate = *data.Cost // per-request multiplier, not cumulative } @@ -643,6 +647,7 @@ func (d *AgentDisplay) GetUsageMetrics() UsageMetrics { Model: d.lastModel, InputTokens: d.totalInputTokens, OutputTokens: d.totalOutputTokens, + AICredits: nanoAiuToCredits(d.totalNanoAiu), BillingRate: d.billingRate, PremiumRequests: d.premiumRequests, DurationMS: d.totalDurationMS, diff --git a/cli/azd/internal/agent/display_test.go b/cli/azd/internal/agent/display_test.go index 241e8e53ea6..9044393898a 100644 --- a/cli/azd/internal/agent/display_test.go +++ b/cli/azd/internal/agent/display_test.go @@ -111,15 +111,19 @@ func TestGetUsageMetrics(t *testing.T) { outputTokens1 := int64(500) cost1 := float64(1.0) duration1 := int64(5000) + nanoAiu1 := 1_500_000_000.0 // Simulate usage events d.HandleEvent(copilot.SessionEvent{ Data: &copilot.AssistantUsageData{ InputTokens: &inputTokens1, OutputTokens: &outputTokens1, - Cost: &cost1, - Duration: &duration1, - Model: "gpt-4.1", + CopilotUsage: &copilot.AssistantUsageCopilotUsage{ + TotalNanoAiu: nanoAiu1, + }, + Cost: &cost1, + Duration: &duration1, + Model: "gpt-4.1", }, }) @@ -127,19 +131,27 @@ func TestGetUsageMetrics(t *testing.T) { outputTokens2 := int64(800) cost2 := float64(1.0) duration2 := int64(3000) + nanoAiu2 := 1_000_000_000.0 d.HandleEvent(copilot.SessionEvent{ Data: &copilot.AssistantUsageData{ InputTokens: &inputTokens2, OutputTokens: &outputTokens2, - Cost: &cost2, - Duration: &duration2, + CopilotUsage: &copilot.AssistantUsageCopilotUsage{ + TotalNanoAiu: nanoAiu2, + }, + Cost: &cost2, + Duration: &duration2, }, }) + d.HandleEvent(copilot.SessionEvent{Data: &copilot.AssistantMessageData{}}) + d.HandleEvent(copilot.SessionEvent{Data: &copilot.SessionIdleData{}}) + require.NoError(t, d.WaitForIdle(t.Context())) metrics := d.GetUsageMetrics() require.Equal(t, float64(3000), metrics.InputTokens) require.Equal(t, float64(1300), metrics.OutputTokens) + require.Equal(t, 2.5, metrics.AICredits) require.Equal(t, float64(1.0), metrics.BillingRate) // last value, not sum require.Equal(t, float64(8000), metrics.DurationMS) require.Equal(t, "gpt-4.1", metrics.Model) diff --git a/cli/azd/internal/agent/headless_collector.go b/cli/azd/internal/agent/headless_collector.go index 5a0263563fa..d58d84b4dda 100644 --- a/cli/azd/internal/agent/headless_collector.go +++ b/cli/azd/internal/agent/headless_collector.go @@ -20,6 +20,7 @@ type HeadlessCollector struct { // Usage metrics — accumulated from assistant.usage events totalInputTokens float64 totalOutputTokens float64 + totalNanoAiu float64 billingRate float64 totalDurationMS float64 premiumRequests float64 @@ -68,6 +69,9 @@ func (h *HeadlessCollector) HandleEvent(event copilot.SessionEvent) { if data.OutputTokens != nil { h.totalOutputTokens += float64(*data.OutputTokens) } + if data.CopilotUsage != nil { + h.totalNanoAiu += data.CopilotUsage.TotalNanoAiu + } if data.Cost != nil { h.billingRate = *data.Cost } @@ -128,6 +132,7 @@ func (h *HeadlessCollector) GetUsageMetrics() UsageMetrics { Model: h.lastModel, InputTokens: h.totalInputTokens, OutputTokens: h.totalOutputTokens, + AICredits: nanoAiuToCredits(h.totalNanoAiu), BillingRate: h.billingRate, PremiumRequests: h.premiumRequests, DurationMS: h.totalDurationMS, diff --git a/cli/azd/internal/agent/headless_collector_test.go b/cli/azd/internal/agent/headless_collector_test.go index 4c3c4cb925a..feb21750302 100644 --- a/cli/azd/internal/agent/headless_collector_test.go +++ b/cli/azd/internal/agent/headless_collector_test.go @@ -4,6 +4,8 @@ package agent import ( + "encoding/json" + "os" "testing" copilot "github.com/github/copilot-sdk/go" @@ -100,14 +102,39 @@ func TestHeadlessCollector_WaitForIdle_DeferredIdle(t *testing.T) { require.NoError(t, err) } -func TestHeadlessCollector_PremiumRequests(t *testing.T) { +func TestHeadlessCollector_ReplaysCapturedAIUEvents(t *testing.T) { t.Parallel() - collector := NewHeadlessCollector() - collector.HandleEvent(copilot.SessionEvent{ - Data: &copilot.SessionShutdownData{TotalPremiumRequests: new(5.0)}, - }) + contents, err := os.ReadFile("testdata/copilot_usage_events.json") + require.NoError(t, err) - usage := collector.GetUsageMetrics() - require.Equal(t, float64(5), usage.PremiumRequests) + var events []copilot.SessionEvent + require.NoError(t, json.Unmarshal(contents, &events)) + + collector := NewHeadlessCollector() + var usageByTurn []UsageMetrics + var checkpointNanoAiu []float64 + for _, event := range events { + collector.HandleEvent(event) + if checkpoint, ok := event.Data.(*copilot.SessionUsageCheckpointData); ok { + checkpointNanoAiu = append(checkpointNanoAiu, checkpoint.TotalNanoAiu) + } + if event.Type() == copilot.SessionEventTypeSessionIdle { + require.NoError(t, collector.WaitForIdle(t.Context())) + usageByTurn = append(usageByTurn, collector.GetUsageMetrics()) + } + } + + require.Len(t, usageByTurn, 2) + require.Equal(t, 9969.0, usageByTurn[0].InputTokens) + require.Equal(t, 3.0, usageByTurn[0].OutputTokens) + require.Equal(t, 2.49515, usageByTurn[0].AICredits) + require.Equal(t, 20053.0, usageByTurn[1].InputTokens) + require.Equal(t, 6.0, usageByTurn[1].OutputTokens) + require.Equal(t, 2.72664, usageByTurn[1].AICredits) + require.NotContains(t, usageByTurn[1].String(), "Premium requests") + + require.Len(t, checkpointNanoAiu, 2) + require.Equal(t, usageByTurn[0].AICredits, nanoAiuToCredits(checkpointNanoAiu[0])) + require.Equal(t, usageByTurn[1].AICredits, nanoAiuToCredits(checkpointNanoAiu[1])) } diff --git a/cli/azd/internal/agent/testdata/copilot_usage_events.json b/cli/azd/internal/agent/testdata/copilot_usage_events.json new file mode 100644 index 00000000000..b55bd7a9b3d --- /dev/null +++ b/cli/azd/internal/agent/testdata/copilot_usage_events.json @@ -0,0 +1,74 @@ +[ + { + "type": "assistant.turn_start", + "data": {} + }, + { + "type": "assistant.usage", + "data": { + "inputTokens": 9969, + "outputTokens": 3, + "copilotUsage": { + "totalNanoAiu": 2495150000 + } + } + }, + { + "type": "assistant.message", + "data": {} + }, + { + "type": "assistant.turn_end", + "data": {} + }, + { + "type": "session.usage_checkpoint", + "data": { + "totalNanoAiu": 2495150000 + } + }, + { + "type": "assistant.idle", + "data": {} + }, + { + "type": "session.idle", + "data": {} + }, + { + "type": "assistant.turn_start", + "data": {} + }, + { + "type": "assistant.usage", + "data": { + "inputTokens": 10084, + "outputTokens": 3, + "copilotUsage": { + "totalNanoAiu": 231490000 + } + } + }, + { + "type": "assistant.message", + "data": {} + }, + { + "type": "assistant.turn_end", + "data": {} + }, + { + "type": "session.usage_checkpoint", + "data": { + "totalNanoAiu": 2726640000 + } + }, + { + "type": "assistant.idle", + "data": {} + }, + { + "type": "session.idle", + "data": {} + } +] diff --git a/cli/azd/internal/agent/types.go b/cli/azd/internal/agent/types.go index 18a2c9544b7..8f974779dff 100644 --- a/cli/azd/internal/agent/types.go +++ b/cli/azd/internal/agent/types.go @@ -26,10 +26,17 @@ type AgentResult struct { // UsageMetrics tracks resource consumption for an agent session. type UsageMetrics struct { - Model string - InputTokens float64 - OutputTokens float64 - BillingRate float64 // per-request cost multiplier (e.g., 1.0x, 2.0x) + Model string + InputTokens float64 + OutputTokens float64 + + // AICredits is the de-billionized AI-credit value derived from the SDK's raw + // nano-AIU totals. It should match what you see as "AIC" in other Copilots. + AICredits float64 + + // BillingRate is deprecated legacy metadata. Prefer AICredits for billing. + BillingRate float64 + // PremiumRequests is deprecated legacy metadata. Prefer AICredits for billing. PremiumRequests float64 DurationMS float64 } @@ -39,9 +46,13 @@ func (u UsageMetrics) TotalTokens() float64 { return u.InputTokens + u.OutputTokens } +func nanoAiuToCredits(nanoAiu float64) float64 { + return nanoAiu / 1_000_000_000 +} + // String returns a multi-line formatted string for display. func (u UsageMetrics) String() string { - if u.InputTokens == 0 && u.OutputTokens == 0 { + if u.InputTokens == 0 && u.OutputTokens == 0 && u.AICredits == 0 { return "" } @@ -55,11 +66,10 @@ func (u UsageMetrics) String() string { lines = append(lines, output.WithGrayFormat(" • Input tokens: %s", formatTokenCount(u.InputTokens))) lines = append(lines, output.WithGrayFormat(" • Output tokens: %s", formatTokenCount(u.OutputTokens))) lines = append(lines, output.WithGrayFormat(" • Total tokens: %s", formatTokenCount(u.TotalTokens()))) - - if u.BillingRate > 0 { - lines = append(lines, output.WithGrayFormat(" • Billing rate: %.0fx per request", u.BillingRate)) + if u.AICredits > 0 { + lines = append(lines, output.WithGrayFormat(" • AI credits: %.2f AIC", u.AICredits)) } - lines = append(lines, output.WithGrayFormat(" • Premium requests: %.0f", u.PremiumRequests)) + if u.DurationMS > 0 { seconds := u.DurationMS / 1000 if seconds >= 60 { diff --git a/cli/azd/internal/agent/types_test.go b/cli/azd/internal/agent/types_test.go index 67e0b4fb29c..e7962d39a35 100644 --- a/cli/azd/internal/agent/types_test.go +++ b/cli/azd/internal/agent/types_test.go @@ -37,16 +37,18 @@ func TestUsageMetrics_Format(t *testing.T) { require.Contains(t, result, "claude-sonnet-4.5") }) - t.Run("WithCostAndPremium", func(t *testing.T) { + t.Run("LegacyBillingFieldsAreHidden", func(t *testing.T) { u := UsageMetrics{ InputTokens: 50000, OutputTokens: 20000, + AICredits: 0.25, BillingRate: 2.0, PremiumRequests: 15, } result := u.String() - require.Contains(t, result, "2x per request") - require.Contains(t, result, "15") + require.Contains(t, result, "AI credits: 0.25 AIC") + require.NotContains(t, result, "Premium requests") + require.NotContains(t, result, "2x per request") }) t.Run("DurationSeconds", func(t *testing.T) { diff --git a/cli/azd/internal/grpcserver/copilot_service.go b/cli/azd/internal/grpcserver/copilot_service.go index d8cfe537a48..b477c32dae5 100644 --- a/cli/azd/internal/grpcserver/copilot_service.go +++ b/cli/azd/internal/grpcserver/copilot_service.go @@ -315,6 +315,7 @@ func convertUsageMetrics(usage agent.UsageMetrics) *azdext.CopilotUsageMetrics { BillingRate: usage.BillingRate, PremiumRequests: usage.PremiumRequests, DurationMs: usage.DurationMS, + AiCredits: usage.AICredits, } } diff --git a/cli/azd/internal/grpcserver/copilot_service_test.go b/cli/azd/internal/grpcserver/copilot_service_test.go index 850f95da14d..70bd8ac9698 100644 --- a/cli/azd/internal/grpcserver/copilot_service_test.go +++ b/cli/azd/internal/grpcserver/copilot_service_test.go @@ -218,7 +218,7 @@ func TestCopilotService_GetUsageMetrics_ValidSession(t *testing.T) { }, nil) mockAgent.On("GetMetrics").Return(agent.AgentMetrics{ Usage: agent.UsageMetrics{ - Model: "gpt-4o", InputTokens: 500, OutputTokens: 250, DurationMS: 3000, + Model: "gpt-4o", InputTokens: 500, OutputTokens: 250, AICredits: 1.25, DurationMS: 3000, }, }) @@ -238,6 +238,7 @@ func TestCopilotService_GetUsageMetrics_ValidSession(t *testing.T) { require.Equal(t, "gpt-4o", resp.Usage.Model) require.Equal(t, float64(500), resp.Usage.InputTokens) require.Equal(t, float64(250), resp.Usage.OutputTokens) + require.Equal(t, 1.25, resp.Usage.AiCredits) require.Equal(t, float64(3000), resp.Usage.DurationMs) } diff --git a/cli/azd/internal/grpcserver/prompt_service_test.go b/cli/azd/internal/grpcserver/prompt_service_test.go index 52eca5df2d1..a0ae846e14e 100644 --- a/cli/azd/internal/grpcserver/prompt_service_test.go +++ b/cli/azd/internal/grpcserver/prompt_service_test.go @@ -1471,20 +1471,16 @@ func TestConvertFileChanges_WithChanges(t *testing.T) { func TestConvertUsageMetrics(t *testing.T) { t.Parallel() usage := agent.UsageMetrics{ - Model: "gpt-4o", - InputTokens: 100, - OutputTokens: 50, - BillingRate: 0.5, - PremiumRequests: 2, - DurationMS: 1500, + Model: "gpt-4o", + InputTokens: 100, + OutputTokens: 50, + DurationMS: 1500, } result := convertUsageMetrics(usage) require.Equal(t, "gpt-4o", result.Model) require.Equal(t, float64(100), result.InputTokens) require.Equal(t, float64(50), result.OutputTokens) require.Equal(t, float64(150), result.TotalTokens) // 100 + 50 - require.Equal(t, 0.5, result.BillingRate) - require.Equal(t, float64(2), result.PremiumRequests) require.Equal(t, float64(1500), result.DurationMs) } diff --git a/cli/azd/internal/telemetry/telemetry_test.go b/cli/azd/internal/telemetry/telemetry_test.go index 48799d49cb9..687da333b00 100644 --- a/cli/azd/internal/telemetry/telemetry_test.go +++ b/cli/azd/internal/telemetry/telemetry_test.go @@ -15,6 +15,8 @@ import ( ) func TestGetTelemetrySystem(t *testing.T) { + t.Setenv("AZD_CONFIG_DIR", t.TempDir()) + devEndpointConfig, err := appinsightsexporter.NewEndpointConfig(devConnectionString) require.NoError(t, err) prodEndpointConfig, err := appinsightsexporter.NewEndpointConfig(prodConnectionString) @@ -100,7 +102,11 @@ func TestGetTelemetrySystem(t *testing.T) { } func TestTelemetrySystem_RunBackgroundUpload(t *testing.T) { - t.Parallel() + resetTelemetryForTest() + t.Cleanup(resetTelemetryForTest) + t.Setenv("AZD_CONFIG_DIR", t.TempDir()) + t.Setenv(collectTelemetryEnvVar, "yes") + type args struct { ctx context.Context enableDebugLogging bool diff --git a/cli/azd/internal/tracing/fields/fields.go b/cli/azd/internal/tracing/fields/fields.go index aacf54c77bd..86a41a8706c 100644 --- a/cli/azd/internal/tracing/fields/fields.go +++ b/cli/azd/internal/tracing/fields/fields.go @@ -1482,16 +1482,9 @@ var ( Purpose: PerformanceAndHealth, IsMeasurement: true, } - // CopilotMessageBillingRate is the billing rate multiplier per message. - CopilotMessageBillingRate = AttributeKey{ - Key: attribute.Key("copilot.message.billingRate"), - Classification: SystemMetadata, - Purpose: BusinessInsight, - IsMeasurement: true, - } - // CopilotMessagePremiumRequests is the number of premium requests used per message. - CopilotMessagePremiumRequests = AttributeKey{ - Key: attribute.Key("copilot.message.premiumRequests"), + // CopilotMessageAICredits is the number of AI credits consumed per session. + CopilotMessageAICredits = AttributeKey{ + Key: attribute.Key("copilot.message.aiCredits"), Classification: SystemMetadata, Purpose: BusinessInsight, IsMeasurement: true, diff --git a/cli/azd/pkg/azdext/contracts/v1/ai_model.pb.go b/cli/azd/pkg/azdext/contracts/v1/ai_model.pb.go index ebc254d762c..25f76e402ff 100644 --- a/cli/azd/pkg/azdext/contracts/v1/ai_model.pb.go +++ b/cli/azd/pkg/azdext/contracts/v1/ai_model.pb.go @@ -28,9 +28,11 @@ type AiModel struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // e.g. "gpt-4o" Format string `protobuf:"bytes,2,opt,name=format,proto3" json:"format,omitempty"` // e.g. "OpenAI" + // Always empty. Use AiModelVersion.lifecycle_status instead. + // // Deprecated: Marked as deprecated in azd/extensions/v1/ai_model.proto. - LifecycleStatus string `protobuf:"bytes,3,opt,name=lifecycle_status,json=lifecycleStatus,proto3" json:"lifecycle_status,omitempty"` // deprecated; always empty; use AiModelVersion.lifecycle_status - Capabilities []string `protobuf:"bytes,4,rep,name=capabilities,proto3" json:"capabilities,omitempty"` // e.g. ["chat", "embeddings"] + LifecycleStatus string `protobuf:"bytes,3,opt,name=lifecycle_status,json=lifecycleStatus,proto3" json:"lifecycle_status,omitempty"` + Capabilities []string `protobuf:"bytes,4,rep,name=capabilities,proto3" json:"capabilities,omitempty"` // e.g. ["chat", "embeddings"] Versions []*AiModelVersion `protobuf:"bytes,5,rep,name=versions,proto3" json:"versions,omitempty"` Locations []string `protobuf:"bytes,6,rep,name=locations,proto3" json:"locations,omitempty"` // canonical locations where available unknownFields protoimpl.UnknownFields diff --git a/cli/azd/pkg/azdext/contracts/v1beta/copilot.pb.go b/cli/azd/pkg/azdext/contracts/v1beta/copilot.pb.go index ab0e7fc09cf..d305fba99e3 100644 --- a/cli/azd/pkg/azdext/contracts/v1beta/copilot.pb.go +++ b/cli/azd/pkg/azdext/contracts/v1beta/copilot.pb.go @@ -600,14 +600,21 @@ func (x *GetCopilotUsageMetricsResponse) GetUsage() *CopilotUsageMetrics { // CopilotUsageMetrics tracks resource consumption for a Copilot session. type CopilotUsageMetrics struct { - state protoimpl.MessageState `protogen:"open.v1"` - Model string `protobuf:"bytes,1,opt,name=model,proto3" json:"model,omitempty"` // Model used. - InputTokens float64 `protobuf:"fixed64,2,opt,name=input_tokens,json=inputTokens,proto3" json:"input_tokens,omitempty"` // Total input tokens consumed. - OutputTokens float64 `protobuf:"fixed64,3,opt,name=output_tokens,json=outputTokens,proto3" json:"output_tokens,omitempty"` // Total output tokens consumed. - TotalTokens float64 `protobuf:"fixed64,4,opt,name=total_tokens,json=totalTokens,proto3" json:"total_tokens,omitempty"` // Sum of input + output tokens. - BillingRate float64 `protobuf:"fixed64,5,opt,name=billing_rate,json=billingRate,proto3" json:"billing_rate,omitempty"` // Per-request cost multiplier (e.g., 1.0x, 2.0x). - PremiumRequests float64 `protobuf:"fixed64,6,opt,name=premium_requests,json=premiumRequests,proto3" json:"premium_requests,omitempty"` // Number of premium requests used. - DurationMs float64 `protobuf:"fixed64,7,opt,name=duration_ms,json=durationMs,proto3" json:"duration_ms,omitempty"` // Total API duration in milliseconds. + state protoimpl.MessageState `protogen:"open.v1"` + Model string `protobuf:"bytes,1,opt,name=model,proto3" json:"model,omitempty"` // Model used. + InputTokens float64 `protobuf:"fixed64,2,opt,name=input_tokens,json=inputTokens,proto3" json:"input_tokens,omitempty"` // Total input tokens consumed. + OutputTokens float64 `protobuf:"fixed64,3,opt,name=output_tokens,json=outputTokens,proto3" json:"output_tokens,omitempty"` // Total output tokens consumed. + TotalTokens float64 `protobuf:"fixed64,4,opt,name=total_tokens,json=totalTokens,proto3" json:"total_tokens,omitempty"` // Sum of input + output tokens. + // Legacy per-request cost multiplier. Use ai_credits instead. + // + // Deprecated: Marked as deprecated in azd/extensions/v1beta/copilot.proto. + BillingRate float64 `protobuf:"fixed64,5,opt,name=billing_rate,json=billingRate,proto3" json:"billing_rate,omitempty"` + // Legacy premium request count. Use ai_credits instead. + // + // Deprecated: Marked as deprecated in azd/extensions/v1beta/copilot.proto. + PremiumRequests float64 `protobuf:"fixed64,6,opt,name=premium_requests,json=premiumRequests,proto3" json:"premium_requests,omitempty"` + DurationMs float64 `protobuf:"fixed64,7,opt,name=duration_ms,json=durationMs,proto3" json:"duration_ms,omitempty"` // Total API duration in milliseconds. + AiCredits float64 `protobuf:"fixed64,8,opt,name=ai_credits,json=aiCredits,proto3" json:"ai_credits,omitempty"` // Total AI credits consumed. unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -670,6 +677,7 @@ func (x *CopilotUsageMetrics) GetTotalTokens() float64 { return 0 } +// Deprecated: Marked as deprecated in azd/extensions/v1beta/copilot.proto. func (x *CopilotUsageMetrics) GetBillingRate() float64 { if x != nil { return x.BillingRate @@ -677,6 +685,7 @@ func (x *CopilotUsageMetrics) GetBillingRate() float64 { return 0 } +// Deprecated: Marked as deprecated in azd/extensions/v1beta/copilot.proto. func (x *CopilotUsageMetrics) GetPremiumRequests() float64 { if x != nil { return x.PremiumRequests @@ -691,6 +700,13 @@ func (x *CopilotUsageMetrics) GetDurationMs() float64 { return 0 } +func (x *CopilotUsageMetrics) GetAiCredits() float64 { + if x != nil { + return x.AiCredits + } + return 0 +} + // GetCopilotFileChangesRequest requests cached file changes for a session. type GetCopilotFileChangesRequest struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1073,16 +1089,18 @@ const file_azd_extensions_v1beta_copilot_proto_rawDesc = "" + "\n" + "session_id\x18\x01 \x01(\tR\tsessionId\"b\n" + "\x1eGetCopilotUsageMetricsResponse\x12@\n" + - "\x05usage\x18\x01 \x01(\v2*.azd.extensions.v1beta.CopilotUsageMetricsR\x05usage\"\x85\x02\n" + + "\x05usage\x18\x01 \x01(\v2*.azd.extensions.v1beta.CopilotUsageMetricsR\x05usage\"\xac\x02\n" + "\x13CopilotUsageMetrics\x12\x14\n" + "\x05model\x18\x01 \x01(\tR\x05model\x12!\n" + "\finput_tokens\x18\x02 \x01(\x01R\vinputTokens\x12#\n" + "\routput_tokens\x18\x03 \x01(\x01R\foutputTokens\x12!\n" + - "\ftotal_tokens\x18\x04 \x01(\x01R\vtotalTokens\x12!\n" + - "\fbilling_rate\x18\x05 \x01(\x01R\vbillingRate\x12)\n" + - "\x10premium_requests\x18\x06 \x01(\x01R\x0fpremiumRequests\x12\x1f\n" + + "\ftotal_tokens\x18\x04 \x01(\x01R\vtotalTokens\x12%\n" + + "\fbilling_rate\x18\x05 \x01(\x01B\x02\x18\x01R\vbillingRate\x12-\n" + + "\x10premium_requests\x18\x06 \x01(\x01B\x02\x18\x01R\x0fpremiumRequests\x12\x1f\n" + "\vduration_ms\x18\a \x01(\x01R\n" + - "durationMs\"=\n" + + "durationMs\x12\x1d\n" + + "\n" + + "ai_credits\x18\b \x01(\x01R\taiCredits\"=\n" + "\x1cGetCopilotFileChangesRequest\x12\x1d\n" + "\n" + "session_id\x18\x01 \x01(\tR\tsessionId\"l\n" + diff --git a/docs/reference/telemetry-data.md b/docs/reference/telemetry-data.md index 240ebb695b1..794710f7b53 100644 --- a/docs/reference/telemetry-data.md +++ b/docs/reference/telemetry-data.md @@ -474,8 +474,7 @@ Emitted at provision start by the `microsoft.foundry` provisioning provider (the | `copilot.message.model` | string | Model for specific message | | `copilot.message.inputTokens` | measurement | Input token count | | `copilot.message.outputTokens` | measurement | Output token count | -| `copilot.message.billingRate` | measurement | Billing rate | -| `copilot.message.premiumRequests` | measurement | Premium request count | +| `copilot.message.aiCredits` | measurement | AI credits consumed during the session | | `copilot.message.durationMs` | measurement | Message duration | | `copilot.consent.approvedCount` | measurement | Approved consent actions | | `copilot.consent.deniedCount` | measurement | Denied consent actions | diff --git a/docs/specs/metrics-audit/feature-telemetry-matrix.md b/docs/specs/metrics-audit/feature-telemetry-matrix.md index d23449e379b..3c31b379443 100644 --- a/docs/specs/metrics-audit/feature-telemetry-matrix.md +++ b/docs/specs/metrics-audit/feature-telemetry-matrix.md @@ -33,7 +33,7 @@ These commands emit attributes or events beyond the global middleware span. | Extensions (dynamic) | `extension.id`, `extension.version`, `extension.event`, `extension.version.from`, `extension.version.to`, `extension.source.category`, `extension.source.kind`, `extension.source.category.from`, `extension.source.category.to`, `extension.installed.source.category`, `extension.dependency_of`, `extension.dependency_update_count`, `extension.update.outcome`, `extension.update.duration_ms`, `extension.grpc.legacy_call_count`, `error.chain.types`, `error.extension.cause_types`, `error.mapper.source.type`, `error.mapper.destination.type` + trace-context propagation to child process | Covers `ext.run`, `ext.install`, `ext.update`, `ext.promote`, source registration, installed inventory, failed-invocation attribution, and temporary legacy gRPC bridge use without emitting source names or locations | | `mcp start` | Per-tool spans via `tracing.Start` with `mcp.client.name`, `mcp.client.version` | MCP event prefix `mcp.*` | | `tool install` / `tool update` / `tool uninstall` / `tool check` / `tool list` / `tool show` | `tool.id`, `tool.ids`, `tool.dry_run`, `tool.install.strategy`, `tool.install.success`, `tool.install.success_count`, `tool.install.failure_count`, `tool.install.failed_ids`, `tool.install.duration_ms`, `tool.update.from_version`, `tool.update.to_version`, `tool.check.updates_available` | Comprehensive coverage in `cli/azd/cmd/tool.go`; install/update emit `tools.pack.build` spans for pack-based tools | -| `copilot` (agent) | `copilot.initialize` event (model + reasoning config), `copilot.session` event (session create/resume) | Emitted from `internal/agent/copilot_agent.go`; covers the experimental copilot agent surface | +| `copilot` (agent) | `copilot.initialize` event (model + reasoning config), `copilot.session` event (session create/resume), `copilot.message.aiCredits` | Emitted from `internal/agent/copilot_agent.go`; covers the experimental copilot agent surface and cumulative AI-credit usage | | `provision` | `validation.provision` event (provision validation outcome + 6 fields), 8 `arm.*` events (subscription / resource-group deploy / stack-deploy / what-if / validate), `aks.postprovision.skip`, per-layer `provision.layer.*` counts (`count`, `max_parallel`, `safe_fallback_count`, `explicit_dependson_count`) when multi-layer infra is used | Telemetry added across `internal/cmd/provision_*.go` and the ARM deployment client | | `deploy` / `publish` / `package` | `deploy.appservice.zip` event (zip-deploy outcome), `container.credentials` / `container.publish` / `container.remotebuild` events for container-based services | Per-service-target instrumentation; container events emitted from container-app and ACR push paths | | `hooks run` (and all hook-running commands) | `hooks.exec` event with `hooks.name` (hashed unless built-in lifecycle name), `hooks.type` (project / service / **layer**), `hooks.kind` (script runtime — `sh` / `pwsh` / `js` / `ts` / `python` / `dotnet`) | `hooks.type=layer` was added with multi-layer provision; pre/post is encoded in `hooks.name` (e.g., `prebuild` / `postbuild`); emitted from the hooks runner on every lifecycle command | @@ -103,7 +103,7 @@ These commands emit attributes or events beyond the global middleware span. | `tool check` | — | ✅ | ✅ | ❌ | `tool.check.updates_available` (count) | | `tool show` | — | ✅ | ✅ | ❌ | `tool.id` | | **Copilot (Agent)** | | | | | | -| `copilot` | — | ✅ | ✅ | ✅ | `copilot.initialize` event captures model + reasoning config; `copilot.session` event tracks session create/resume | +| `copilot` | — | ✅ | ✅ | ✅ | `copilot.initialize` captures model + reasoning config; `copilot.session` tracks session create/resume; `copilot.message.aiCredits` measures cumulative usage | | **Disabled** | | | | | | | `version` | — | 🚫 | — | — | Intentionally disabled | | `telemetry upload` | — | 🚫 | — | — | Intentionally disabled | diff --git a/docs/specs/metrics-audit/telemetry-schema.md b/docs/specs/metrics-audit/telemetry-schema.md index 1b4f8ecbdba..c8c57bacc39 100644 --- a/docs/specs/metrics-audit/telemetry-schema.md +++ b/docs/specs/metrics-audit/telemetry-schema.md @@ -354,8 +354,7 @@ set on its own. | Model | `copilot.message.model` | SystemMetadata | FeatureInsight | | | Input tokens | `copilot.message.inputTokens` | SystemMetadata | PerformanceAndHealth | **Measurement** | | Output tokens | `copilot.message.outputTokens` | SystemMetadata | PerformanceAndHealth | **Measurement** | -| Billing rate | `copilot.message.billingRate` | SystemMetadata | BusinessInsight | **Measurement** | -| Premium requests | `copilot.message.premiumRequests` | SystemMetadata | BusinessInsight | **Measurement** | +| AI credits | `copilot.message.aiCredits` | SystemMetadata | BusinessInsight | **Measurement** | | Duration (ms) | `copilot.message.durationMs` | SystemMetadata | PerformanceAndHealth | **Measurement** | ### Copilot Consent