diff --git a/cli/azd/cmd/telemetry_test.go b/cli/azd/cmd/telemetry_test.go index cea6bf3b082..aefda1895c6 100644 --- a/cli/azd/cmd/telemetry_test.go +++ b/cli/azd/cmd/telemetry_test.go @@ -25,6 +25,10 @@ import ( func TestTelemetryEventConstants(t *testing.T) { t.Parallel() require.Equal(t, "ext.update", events.ExtensionUpdateEvent) + // Agent operation classifications use values of the existing extension.event + // field on ext.usage; they must not require another host span or attribute. + require.Equal(t, "ext.usage", events.ExtensionUsageEvent) + require.Equal(t, "extension.event", string(fields.ExtensionEvent.Key)) } // TestTelemetryFieldConstants verifies that all telemetry field constants added for diff --git a/cli/azd/extensions/azure.ai.agents/README.md b/cli/azd/extensions/azure.ai.agents/README.md index 23bd2e21d3f..3059f615580 100644 --- a/cli/azd/extensions/azure.ai.agents/README.md +++ b/cli/azd/extensions/azure.ai.agents/README.md @@ -44,6 +44,15 @@ the OpenTelemetry operation ID. A project with multiple agent classifications reports one row for each classification. The event never includes agent names, service keys, paths, URLs, prompts, or other customer content. +### Operation classification markers + +Init, provision and deploy also emit bounded +`agent.operation.v1...` values in the existing +`extension.event` field of `ext.usage`, with no additional attributes. Existing +`agent.context.resolved` and command results are unchanged. See +[operation statistics](docs/operation-telemetry.md) for the vocabulary, query and +coverage limits. Marker success must not be used as command success. + ## Non-interactive automation See the shared [AI extension non-interactive input reference](../ai-non-interactive.md) diff --git a/cli/azd/extensions/azure.ai.agents/docs/operation-telemetry-fixture.kql b/cli/azd/extensions/azure.ai.agents/docs/operation-telemetry-fixture.kql new file mode 100644 index 00000000000..13fe3eb3fe8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/docs/operation-telemetry-fixture.kql @@ -0,0 +1,29 @@ +// Synthetic query validation: no customer telemetry or ingestion changes. +// Expected: observed=4, successes=2, failures=2, missingClassification=1, cancellations=1. +// The mixed operation has two marker rows but remains ONE completed command. +let Completed = datatable(operation_Id:string, id:string, operation:string, success:bool, resultCode:string) +[ + "one", "1", "init", true, "Success", + "two", "2", "init", false, "ext.user.cancelled", + "three", "3", "deploy", true, "Success", + "four", "4", "provision", false, "internal.error" +]; +let RawMarkers = datatable(operation_Id:string, eventName:string) +[ + "one", "agent.operation.v1.init.voice_managed.none", + "three", "agent.operation.v1.deploy.hosted_invocations_ws.none", + "three", "agent.operation.v1.deploy.voice_hosted_wrapper.none", + "three", "agent.operation.v1.deploy.voice_hosted_wrapper.none", + "four", "agent.operation.v1.provision.voice_byom.enabled" +]; +let Markers = RawMarkers +| parse eventName with "agent.operation.v1." operation "." category "." telephony +| summarize categories=make_set(category) by operation_Id, operation; +Completed +| join kind=leftouter Markers on operation_Id, operation +| summarize observed=count(), successes=countif(success), failures=countif(not(success)), + missingClassification=countif(isnull(categories)), + cancellations=countif(resultCode startswith "user.canceled" + or resultCode in ("ext.user.cancelled", "internal.operation_cancelled", "internal.operation_aborted")) +| extend passed = observed == 4 and successes == 2 and failures == 2 + and missingClassification == 1 and cancellations == 1 diff --git a/cli/azd/extensions/azure.ai.agents/docs/operation-telemetry.md b/cli/azd/extensions/azure.ai.agents/docs/operation-telemetry.md new file mode 100644 index 00000000000..b2913bb3baf --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/docs/operation-telemetry.md @@ -0,0 +1,137 @@ +# Agent operation statistics + + + +Reuse existing command completion records for totals, successes and failures. +The only additional data is bounded text in the existing `extension.event` field +of `ext.usage`: `agent.operation.v1...`. +There are no new attributes, core spans, result codes, exporters or pipelines. +Existing `agent.context.resolved` fields and deduplication are unchanged. + +| Segment | Fixed values | +|---|---| +| operation | `init`, `provision`, `deploy` | +| category | `hosted`, `hosted_invocations_ws`, `prompt`, `workflow`, `voice_managed`, `voice_byom`, `voice_hosted_wrapper`, `unknown` | +| telephony | `none`, `enabled`, `unknown` | + +Voice aliases normalize to the same category. BYOM is the configured model mode, +not a guess from a customer model/deployment name. `invocations_ws` denotes a +transport, not proof of audio usage. Telephony means configured bindings, not +successful binding creation or phone calls. No names, paths, model names, +identifiers, credentials, prompts or configuration payloads are emitted. + +## Coverage and behavior + +- Extension init collects explicit kind intent, refining it at existing selection, + definition/adoption and reuse points. RunE return attempts to report the last + classification on success **or failure**. Unknown intent is not inferred from + later success. Failure before RunE, process termination or unavailable telemetry + can have no marker. Success/failure remain owned by existing command telemetry. +- Agent preprovision/predeploy handlers report in-memory service classifications + before their existing work. Failures before the hooks (such as package or config + errors) may lack markers. Skipped services that never enter the hook are absent. +- No additional project/file/Azure queries are made for classification. Unresolved + root `$ref` or an external definition override is unknown; no speculative reads. +- Distinct operation/category/telephony tuples are attempted once per reporter + process. Provision cannot suppress deploy, and init refinements do not double + count earlier intent. The original reporter's behavior is unchanged. +- Uses the existing best-effort reporter, no retries, one-second total deadline + per batch. Init preserves trace metadata with a bounded uncancelled reporting + context on return; telemetry failures never replace command errors/return values. + There is bounded latency, not zero overhead. No global background worker is added. +- **Core `azd init` is not the extension command.** Its totals/results already + exist, but its agent type remains unknown in this extension-only change. Reuse + inside `azd ai agent init` can be classified from its existing project read. + +## Counting contract + +Count completed command spans, **not marker rows or agents**. For mixed projects, +keep a sorted category combination and one command count; never assign one +project failure as each service's individual outcome. A hosted voice project may +contain both `hosted_invocations_ws` and `voice_hosted_wrapper`. + +The left join below preserves failures without markers. Unclassified core commands +include both non-agent projects and early-failed agent projects: they cannot be +claimed as agent-only usage. Same-operation commands in a shared trace are marked +ambiguous instead of guessing which service/marker belongs to which command. +Do not add parent `up` and child operations together into one total. + +Telemetry opt-out, nonofficial ZIP/dev installs, old hosts, the shared 100-event +invocation budget, crashes and ingestion delays can lose markers/completions. +The additional vocabulary is bounded to 72 possible tuples across three operations. +These are counts of **observed completions**, not an absolute census of users. +Report unknown/ambiguous coverage alongside classified rates, and use a fixed +host/extension release cohort. Installation alone does not prove agent involvement. + +## KQL — existing Application Insights requests shape + +No shared ingestion function needs modification. Validate this query against the +authorized destination and adapt table/column aliases for cooked Kusto/LENS data. +It has not been run against production customer data in this change. +The [synthetic fixture](operation-telemetry-fixture.kql) can be run without customer +tables: it expects four completions, two successes, two failures, one cancellation +and one unclassified failure, even though one mixed deploy has duplicate/multiple marker rows. This is +a query-engine acceptance fixture, not a claim of local Kusto execution. + +```kusto +let since = ago(7d); +let Markers = requests +| where timestamp >= since - 1d +| where name == "ext.usage" +| where tostring(customDimensions["extension.id"]) == "azure.ai.agents" +| extend marker = tostring(customDimensions["extension.event"]) +| parse marker with "agent.operation.v1." operation "." category "." telephony +| where operation in ("init", "provision", "deploy") +| where category in ("hosted", "hosted_invocations_ws", "prompt", "workflow", + "voice_managed", "voice_byom", "voice_hosted_wrapper", "unknown") +| where telephony in ("none", "enabled", "unknown") +| summarize categories=make_set(category), phones=make_set(telephony) + by operation_Id, operation; +let Completed = requests +| where timestamp >= since +| extend command = tostring(customDimensions["cmd.entry"]) +| where (name == "ext.run" and command == "cmd.ai.agent.init") + or name in ("cmd.init", "cmd.provision", "cmd.deploy") +| summarize arg_max(timestamp, *) by operation_Id, id +| extend operation = case(name == "ext.run", "init", name == "cmd.init", "init", + name == "cmd.provision", "provision", "deploy") +| extend scope = iff(name == "ext.run", "agent_extension", "core"); +let Cardinality = Completed +| summarize commandSpans=count() by operation_Id, operation, scope; +Completed +| join kind=leftouter Cardinality on operation_Id, operation, scope +| join kind=leftouter Markers on operation_Id, operation +| extend attribution = case(name == "cmd.init", "core_init_unclassified", + commandSpans != 1, "ambiguous_trace", + isnull(categories), "no_marker", "classified") +| extend agentTypes = iff(attribution == "classified", + strcat_array(array_sort_asc(categories), "+"), "unknown") +| extend phoneConfiguration = iff(attribution == "classified", + strcat_array(array_sort_asc(phones), "+"), "unknown") +| extend succeeded = tobool(success), code = tostring(resultCode) +| summarize total=count(), successes=countif(succeeded == true), + failures=countif(succeeded == false), missingResult=countif(isnull(succeeded)), + cancellations=countif(code startswith "user.canceled" + or code in ("ext.user.cancelled", "internal.operation_cancelled", "internal.operation_aborted")) + by scope, operation, attribution, agentTypes, phoneConfiguration +| extend successRate = iff(successes + failures > 0, todouble(successes)/(successes+failures), real(null)), + failureRate = iff(successes + failures > 0, todouble(failures)/(successes+failures), real(null)) +``` + +`total = successes + failures + missingResult`. Cancellations are a diagnostic +subset, not an extra addition to total. The host's existing success/resultCode +semantics are preserved (a graceful nil-error cancellation can remain success). +`ext.usage.success` is never used to determine business outcome. A marker without +a completion is not proof of failure. Unknown types before classification remain +unknown; these rates must not be advertised as complete per-type population rates. + +## Validation and rollout + +Fake-host tests check actual submitted event names, nil attribute maps, +per-operation deduplication, concurrent calls, type privacy and unchanged init +failure behavior. Existing telemetry tests continue to assert the original contract. +Test official-registry ingestion and trace joins before using a production KPI; +local ZIP sources remain deliberately rejected by host admission. These additional +event values and their product metadata need normal extension telemetry review; +using existing fields does not bypass privacy rules. No customer telemetry was +accessed or uploaded by the unit tests. diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go index e1586a6a469..2f7d5225e57 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go @@ -1332,6 +1332,10 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, --image registry.example.com/agents/my-agent:v1 --registry-connection production-registry`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + // Record bounded intent before validation so failures are not a success-only sample. + ctx := withInitOperationContext(azdext.WithAccessToken(cmd.Context()), flags.kind, + flags.manifestPointer != "" || len(args) > 0) + defer reportInitOperation(ctx) flags.noPrompt = extCtx.NoPrompt if flags.env == "" { flags.env = extCtx.Environment @@ -1361,7 +1365,6 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, } } - ctx := azdext.WithAccessToken(cmd.Context()) azdClient, err := azdext.NewAzdClient() if err != nil { return exterrors.Internal(exterrors.CodeAzdClientFailed, fmt.Sprintf("failed to create azd client: %s", err)) @@ -1522,6 +1525,7 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, // and code scaffolding. The image is wired into azure.yaml and ACR is // skipped by the existing --image handling in InitAction.Run. if flags.image != "" && flags.manifestPointer == "" { + recordInitProperties(ctx, map[string]any{"kind": "hosted"}) // Validate early so we fail before initializing a project/template. if err := validateImageFlag(flags.image, flags.deployMode); err != nil { return err @@ -1676,6 +1680,7 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, useExisting = *confirmResp.Value } if useExisting { + recordInitProject(ctx, detection.project) if err := runReuseProjectAgentServices( ctx, flags, azdClient, detection.services, ); err != nil { @@ -1974,6 +1979,7 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, } case initModeVoice: + recordInitProperties(ctx, map[string]any{"kind": "voice", "modelType": "managed"}) // User chose to create a declarative (managed) voice agent. // Resolve the agent name, synthesize a prompt-voice manifest, // and route it through the manifest flow — the same path as @@ -2324,6 +2330,7 @@ func (a *InitAction) Run(ctx context.Context) error { if err != nil { return fmt.Errorf("downloading agent.yaml: %w", err) } + recordInitDefinition(ctx, agentManifest.Template) // Prompt for deploy mode (code vs container) for hosted agents. // Code deploy is supported for Python and .NET projects. if hostedAgent, ok := agentManifest.Template.(agent_yaml.ContainerAgent); ok { diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_adopt.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_adopt.go index 7ee2eadb10c..614fa0393e9 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_adopt.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_adopt.go @@ -978,6 +978,7 @@ func runInitFromAzureYaml( content []byte, ) error { projectName := foundryProjectName(content) + recordInitProjectContent(ctx, content) agentNameOverride, err := adoptedAgentNameOverride(flags) if err != nil { return err @@ -1022,6 +1023,7 @@ func runInitFromAzureYaml( return err } promptOnly := stagedInfo.promptOnly() + recordInitProjectContent(ctx, stagedContent) if agentNameOverride != "" { // Validate against the fully staged template so services whose host lives // inside a local $ref are counted the same way azd-core will load them. diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code.go index 2ea0654b9c6..388fa624f31 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code.go @@ -43,6 +43,7 @@ type InitFromCodeAction struct { } func (a *InitFromCodeAction) Run(ctx context.Context) error { + recordInitProperties(ctx, map[string]any{"kind": "hosted"}) var err error a.projectConfig, err = a.ensureProject(ctx) if err != nil { @@ -93,6 +94,7 @@ func (a *InitFromCodeAction) Run(ctx context.Context) error { } if localDefinition != nil { + recordInitDefinition(ctx, localDefinition) // Generate .agentignore. The agent definition is written into the // azure.yaml service entry below, not to an on-disk agent.yaml. diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code_reuse.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code_reuse.go index 447c7316ea7..833fd282c2d 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code_reuse.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code_reuse.go @@ -79,6 +79,7 @@ func runReuseDefinition( fmt.Sprintf("Fix %s and retry, or remove the file to start a fresh init.", displayPath), ) } + recordInitDefinition(ctx, def) fmt.Println(color.HiBlackString( "Detected existing agent definition: %s (name: %s).", diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_managed.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_managed.go index 2ae1e24b1ac..8f31c8c9c95 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_managed.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_managed.go @@ -166,6 +166,7 @@ func runInitManaged( harness string, manifest *promptAgentManifest, ) error { + recordInitProperties(ctx, map[string]any{"kind": "prompt"}) // Every prompt-agent init converges here — interactive picker, --kind prompt, // and manifest adoption alike — so this is the one place the preview notice // reaches all of them. Emitted before validation so it is seen even when the diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_reuse_project_agent.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_reuse_project_agent.go index 6d8d5fe8d5c..8912a3a4cf4 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_reuse_project_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_reuse_project_agent.go @@ -35,6 +35,7 @@ type projectAgentService struct { type projectAgentDetection struct { services []projectAgentService projectRoot string + project *azdext.ProjectConfig } // detectProjectAgentServices returns the agent services the azd host reports for @@ -67,6 +68,7 @@ func detectProjectAgentServices(ctx context.Context, azdClient *azdext.AzdClient } return projectAgentDetection{ + project: project, services: services, projectRoot: project.GetPath(), } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/listen.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/listen.go index bc271d2cddd..06efab9ab15 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/listen.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/listen.go @@ -20,9 +20,11 @@ import ( "azureaiagent/internal/pkg/agents/optimize_api" "azureaiagent/internal/pkg/envkey" "azureaiagent/internal/project" + agentTelemetry "azureaiagent/internal/telemetry" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/azure/azure-dev/cli/azd/pkg/azdext" + foundryTelemetry "github.com/azure/azure-dev/cli/azd/pkg/foundry/telemetry" "github.com/azure/azure-dev/cli/azd/pkg/output" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/structpb" @@ -38,6 +40,7 @@ func configureExtensionHost(host *azdext.ExtensionHost) { func configureExtensionHostWithTelemetry(host *azdext.ExtensionHost, telemetryReporter *agentContextReporter) { azdClient := host.Client() + operationReporter := newOperationReporter() // IMPORTANT: service target name here must match the name used in the extension manifest. host. @@ -46,6 +49,10 @@ func configureExtensionHostWithTelemetry(host *azdext.ExtensionHost, telemetryRe }). WithProjectEventHandler("preprovision", func(ctx context.Context, args *azdext.ProjectEventArgs) error { telemetryReporter.reportProjectConfig(ctx, azdClient.Telemetry(), args.Project, "provision") + usage := foundryTelemetry.NewReporter(azdClient.Telemetry(), nil) + if classes := operationProjectClasses(args.Project); len(classes) > 0 { + operationReporter.report(ctx, usage, "provision", classes) + } return preprovisionHandler(ctx, azdClient, args) }). WithProjectEventHandler("postprovision", func(ctx context.Context, args *azdext.ProjectEventArgs) error { @@ -53,6 +60,10 @@ func configureExtensionHostWithTelemetry(host *azdext.ExtensionHost, telemetryRe }). WithServiceEventHandler("predeploy", func(ctx context.Context, args *azdext.ServiceEventArgs) error { telemetryReporter.reportService(ctx, azdClient.Telemetry(), args.Project, args.Service, "deploy") + usage := foundryTelemetry.NewReporter(azdClient.Telemetry(), nil) + operationReporter.report(ctx, usage, "deploy", []agentTelemetry.OperationClass{ + operationServiceClass(args.Service), + }) return predeployHandler(ctx, azdClient, args) }, &azdext.ServiceEventOptions{Host: AiAgentHost}). WithServiceEventHandler("postdeploy", func(ctx context.Context, args *azdext.ServiceEventArgs) error { diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/telemetry_operation.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/telemetry_operation.go new file mode 100644 index 00000000000..a97cb65036c --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/telemetry_operation.go @@ -0,0 +1,162 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "maps" + "os" + "slices" + "strings" + "sync" + "time" + + "azureaiagent/internal/project" + agentTelemetry "azureaiagent/internal/telemetry" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + foundryTelemetry "github.com/azure/azure-dev/cli/azd/pkg/foundry/telemetry" + "go.yaml.in/yaml/v3" +) + +type initOperationContextKey struct{} + +// initOperationContext stores bounded classifications only. It never owns or +// changes command results; refinements replace intent instead of double-counting it. +type initOperationContext struct { + classes []agentTelemetry.OperationClass +} + +func withInitOperationContext(ctx context.Context, kind string, explicitInput bool) context.Context { + state := &initOperationContext{} + if !explicitInput && kind != "" { + state.classes = []agentTelemetry.OperationClass{ + agentTelemetry.ClassifyOperation(map[string]any{"kind": kind}), + } + } + return context.WithValue(ctx, initOperationContextKey{}, state) +} + +func recordInitProperties(ctx context.Context, properties map[string]any) { + if state, ok := ctx.Value(initOperationContextKey{}).(*initOperationContext); ok { + state.classes = []agentTelemetry.OperationClass{agentTelemetry.ClassifyOperation(properties)} + } +} + +func recordInitDefinition(ctx context.Context, definition any) { + if ctx.Value(initOperationContextKey{}) == nil { + return + } + data, err := json.Marshal(definition) + if err != nil { + return + } + var properties map[string]any + if json.Unmarshal(data, &properties) == nil { + recordInitProperties(ctx, properties) + } +} + +func recordInitProjectContent(ctx context.Context, content []byte) { + state, ok := ctx.Value(initOperationContextKey{}).(*initOperationContext) + if !ok { + return + } + var doc struct { + Services map[string]map[string]any `yaml:"services"` + } + if yaml.Unmarshal(content, &doc) != nil { + return + } + var classes []agentTelemetry.OperationClass + for _, properties := range doc.Services { + if properties["host"] == AiAgentHost { + classes = append(classes, agentTelemetry.ClassifyOperation(properties)) + } + } + if len(classes) > 0 { + state.classes = classes + } +} + +func recordInitProject(ctx context.Context, project *azdext.ProjectConfig) { + if state, ok := ctx.Value(initOperationContextKey{}).(*initOperationContext); ok { + state.classes = operationProjectClasses(project) + } +} + +func reportInitOperation(ctx context.Context) { + state, ok := ctx.Value(initOperationContextKey{}).(*initOperationContext) + if !ok { + return + } + // A cancelled command must remain cancelled. A separate, bounded reporting + // context preserves its trace metadata but never retries the operation. + reportCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Second) + defer cancel() + client, err := azdext.NewAzdClient() + if err != nil { + return + } + defer client.Close() + reporter := foundryTelemetry.NewReporter(client.Telemetry(), nil) + newOperationReporter().report(reportCtx, reporter, "init", state.classes) +} + +type operationReporter struct { + mu sync.Mutex + seen map[string]bool +} + +func newOperationReporter() *operationReporter { + return &operationReporter{seen: map[string]bool{}} +} + +func (r *operationReporter) report( + ctx context.Context, reporter foundryTelemetry.Reporter, operation string, classes []agentTelemetry.OperationClass, +) { + if len(classes) == 0 { + classes = []agentTelemetry.OperationClass{{Category: "unknown", Telephony: "unknown"}} + } + // Deterministic order and one shared budget bound the total telemetry delay, + // not one second per service. Keep the original context reporter untouched. + ctx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + names := map[string]foundryTelemetry.Event{} + for _, class := range classes { + if event, ok := agentTelemetry.OperationClassified(operation, class); ok { + names[event.Name] = event + } + } + keys := slices.Sorted(maps.Keys(names)) + for _, name := range keys { + r.mu.Lock() + seen := r.seen[name] + r.seen[name] = true + r.mu.Unlock() + if !seen && ctx.Err() == nil && reporter != nil { + reporter.Report(ctx, names[name]) + } + } +} + +func operationServiceClass(svc *azdext.ServiceConfig) agentTelemetry.OperationClass { + unknown := agentTelemetry.OperationClass{Category: "unknown", Telephony: "unknown"} + if svc == nil || strings.TrimSpace(os.Getenv("AGENT_DEFINITION_PATH")) != "" { + return unknown // do not read external definitions just to collect telemetry + } + properties := project.ServiceConfigProps(svc) + return agentTelemetry.ClassifyOperation(properties.AsMap()) +} + +func operationProjectClasses(project *azdext.ProjectConfig) []agentTelemetry.OperationClass { + var classes []agentTelemetry.OperationClass + for _, svc := range project.GetServices() { + if svc.GetHost() == AiAgentHost { + classes = append(classes, operationServiceClass(svc)) + } + } + return classes +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/telemetry_operation_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/telemetry_operation_test.go new file mode 100644 index 00000000000..d181c77118e --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/telemetry_operation_test.go @@ -0,0 +1,251 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net" + "sync" + "testing" + "time" + + "azureaiagent/internal/pkg/agents/agent_yaml" + agentTelemetry "azureaiagent/internal/telemetry" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + foundryTelemetry "github.com/azure/azure-dev/cli/azd/pkg/foundry/telemetry" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/structpb" +) + +type operationRecordingReporter struct { + mu sync.Mutex + events []foundryTelemetry.Event +} + +func (r *operationRecordingReporter) Report(ctx context.Context, event foundryTelemetry.Event) { + r.mu.Lock() + defer r.mu.Unlock() + r.events = append(r.events, event) +} + +func TestOperationReporterDeduplicatesPerOperation(t *testing.T) { + t.Parallel() + r := newOperationReporter() + capture := &operationRecordingReporter{} + classes := []agentTelemetry.OperationClass{{Category: "hosted", Telephony: "none"}} + var wg sync.WaitGroup + for range 20 { + wg.Go(func() { + r.report(t.Context(), capture, "provision", classes) + r.report(t.Context(), capture, "deploy", classes) + }) + } + wg.Wait() + require.Len(t, capture.events, 2) + require.NotEqual(t, capture.events[0].Name, capture.events[1].Name) + for _, event := range capture.events { + require.Empty(t, event.Attributes) + } + // Unsupported operations do not report and nil reporters are harmless. + r.report(t.Context(), capture, "down", classes) + r.report(t.Context(), nil, "init", classes) + require.Len(t, capture.events, 2) +} + +func TestInitOperationRefinesIntentWithoutChangingContext(t *testing.T) { + t.Parallel() + ctx := withInitOperationContext(t.Context(), "hosted", false) + recordInitDefinition(ctx, agent_yaml.VoiceAgent{ + AgentDefinition: agent_yaml.AgentDefinition{Kind: agent_yaml.AgentKindVoice, Name: "private-name"}, + ModelType: agent_yaml.VoiceModelTypeSelfDeployed, + }) + state, ok := ctx.Value(initOperationContextKey{}).(*initOperationContext) + require.True(t, ok) + require.Equal(t, []agentTelemetry.OperationClass{{Category: "voice_byom", Telephony: "none"}}, state.classes) + recordInitProjectContent(ctx, []byte(`services: + secret-target: + host: azure.ai.agent + kind: hosted + protocols: [{protocol: invocations_ws}] + secret-wrapper: + host: azure.ai.agent + kind: voice + conversationEngine: {type: hosted_agent, name: secret-target} +`)) + require.ElementsMatch(t, []agentTelemetry.OperationClass{ + {Category: "hosted_invocations_ws", Telephony: "none"}, + {Category: "voice_hosted_wrapper", Telephony: "none"}, + }, state.classes) + unknown := withInitOperationContext(t.Context(), "hosted", true) + require.Empty(t, unknown.Value(initOperationContextKey{}).(*initOperationContext).classes) + recordInitProperties(t.Context(), map[string]any{"kind": "voice"}) // no state: no-op + recordInitProjectContent(ctx, []byte("[invalid")) // telemetry parsing never surfaces an error + require.Len(t, state.classes, 2) +} + +func TestOperationProjectClassDoesNotReadDefinitions(t *testing.T) { + props, err := structpb.NewStruct(map[string]any{"kind": "voice"}) + require.NoError(t, err) + svc := &azdext.ServiceConfig{Host: AiAgentHost, AdditionalProperties: props} + t.Setenv("AGENT_DEFINITION_PATH", "missing-sensitive-file.yaml") + require.Equal(t, "unknown", operationServiceClass(svc).Category) + t.Setenv("AGENT_DEFINITION_PATH", "") + require.Equal(t, "voice_managed", operationServiceClass(svc).Category) + require.Empty(t, operationProjectClasses(nil)) + ref, err := structpb.NewStruct(map[string]any{"$ref": "missing-sensitive-file.yaml"}) + require.NoError(t, err) + svc.AdditionalProperties = ref + require.Equal(t, "unknown", operationServiceClass(svc).Category) + // Reuse uses only the project already loaded by the existing flow. + ctx := withInitOperationContext(t.Context(), "", false) + recordInitProject(ctx, &azdext.ProjectConfig{Services: map[string]*azdext.ServiceConfig{ + "agent": {Host: AiAgentHost, AdditionalProperties: props}, + "web": {Host: "containerapp"}, + }}) + state, ok := ctx.Value(initOperationContextKey{}).(*initOperationContext) + require.True(t, ok) + require.Equal(t, []agentTelemetry.OperationClass{{Category: "voice_managed", Telephony: "none"}}, state.classes) +} + +func TestOperationMarkerDoesNotChangeOriginalContextContract(t *testing.T) { + t.Parallel() + props, err := structpb.NewStruct(map[string]any{ + "kind": "voice", "modelType": "self_deployed", "instructions": "private instructions", + "telephony": map[string]any{"bindings": []any{map[string]any{"identifier": "private number"}}}, + }) + require.NoError(t, err) + project := &azdext.ProjectConfig{Services: map[string]*azdext.ServiceConfig{ + "private-name": {Host: AiAgentHost, AdditionalProperties: props}, + }} + before, err := json.Marshal(project) + require.NoError(t, err) + original := &telemetryRecordingClient{} + r := newAgentContextReporter() + r.reportProjectConfig(t.Context(), original, project, "provision") + additional := &operationRecordingReporter{} + newOperationReporter().report(t.Context(), additional, "provision", operationProjectClasses(project)) + r.reportProjectConfig(t.Context(), original, project, "deploy") + require.Len(t, original.requests, 1, "existing process-wide deduplication must not change") + require.Equal(t, agentContextResolvedEvent, original.requests[0].EventName) + require.Equal(t, map[string]string{ + agentKindAttribute: "voice", agentHarnessAttribute: "none", agentOperationAttribute: "provision", + }, original.requests[0].Attributes) + require.Len(t, additional.events, 1) + require.Nil(t, additional.events[0].Attributes) + require.Equal(t, "agent.operation.v1.provision.voice_byom.enabled", additional.events[0].Name) + after, err := json.Marshal(project) + require.NoError(t, err) + require.Equal(t, before, after) +} + +func TestOperationServiceClassPropertyPrecedence(t *testing.T) { + t.Setenv("AGENT_DEFINITION_PATH", "") + for _, tt := range []struct { + name string + inline, legacy map[string]any + want string + }{ + {"legacy-with-unrelated-inline", map[string]any{"custom": "private-value"}, + map[string]any{"kind": "voice", "modelType": "self_deployed"}, "voice_byom"}, + {"inline-kind-wins", map[string]any{"kind": "hosted"}, + map[string]any{"kind": "voice"}, "hosted"}, + {"legacy-only", nil, map[string]any{"kind": "prompt"}, "prompt"}, + {"no-kind", map[string]any{"custom": "private-value"}, nil, "unknown"}, + {"unresolved-ref", map[string]any{"kind": "voice", "$ref": "private-path"}, + map[string]any{"kind": "prompt"}, "unknown"}, + } { + t.Run(tt.name, func(t *testing.T) { + inline, err := structpb.NewStruct(tt.inline) + require.NoError(t, err) + legacy, err := structpb.NewStruct(tt.legacy) + require.NoError(t, err) + svc := &azdext.ServiceConfig{Host: AiAgentHost, AdditionalProperties: inline, Config: legacy} + before, err := json.Marshal(svc) + require.NoError(t, err) + require.Equal(t, tt.want, operationServiceClass(svc).Category) + after, err := json.Marshal(svc) + require.NoError(t, err) + require.Equal(t, before, after) + }) + } +} + +type operationTelemetryServer struct { + azdext.UnimplementedTelemetryServiceServer + mu sync.Mutex + events []*azdext.ReportUsageRequest + err error + block bool +} + +func (s *operationTelemetryServer) ReportUsage( + ctx context.Context, req *azdext.ReportUsageRequest, +) (*azdext.ReportUsageResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.events = append(s.events, req) + if s.block { + <-ctx.Done() + return nil, ctx.Err() + } + return &azdext.ReportUsageResponse{Accepted: false}, s.err +} + +func TestFailedInitReportsIntentWithoutChangingFailure(t *testing.T) { + for _, tt := range []struct { + name string + err error + block bool + }{ + {"not-accepted", nil, false}, + {"rpc-failure", errors.New("sensitive transport text"), false}, + {"old-host", status.Error(codes.Unimplemented, "unavailable"), false}, + {"deadline", nil, true}, + } { + t.Run(tt.name, func(t *testing.T) { + server := grpc.NewServer() + capture := &operationTelemetryServer{err: tt.err, block: tt.block} + azdext.RegisterTelemetryServiceServer(server, capture) + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + go func() { _ = server.Serve(listener) }() + t.Cleanup(func() { server.Stop(); _ = listener.Close() }) + t.Setenv("AZD_SERVER", listener.Addr().String()) + cmd := newInitCommand(nil) + var output bytes.Buffer + cmd.SetOut(&output) + cmd.SetErr(&output) + cmd.SetArgs([]string{"--kind", "prompt-voice", "--runtime", "python_3_13"}) + start := time.Now() + err = cmd.Execute() + require.Less(t, time.Since(start), 3*time.Second, "telemetry must have a bounded delay and no retries") + require.ErrorContains(t, err, "new prompt voice agents cannot use these init inputs") + capture.mu.Lock() + defer capture.mu.Unlock() + require.Len(t, capture.events, 1) + require.Equal(t, "agent.operation.v1.init.voice_managed.none", capture.events[0].EventName) + require.Empty(t, capture.events[0].Attributes) + require.NotContains(t, output.String(), "sensitive transport text") + }) + } +} + +func TestOperationReporterHonorsCancellation(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + capture := &operationRecordingReporter{} + newOperationReporter().report(ctx, capture, "deploy", nil) + require.Empty(t, capture.events) + start := time.Now() + reportInitOperation(ctx) // no state: no connection or wait + require.Less(t, time.Since(start), time.Second) +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/telemetry/operation.go b/cli/azd/extensions/azure.ai.agents/internal/telemetry/operation.go new file mode 100644 index 00000000000..e5b8dd2805f --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/internal/telemetry/operation.go @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package telemetry + +import ( + "slices" + "strings" + + foundryTelemetry "github.com/azure/azure-dev/cli/azd/pkg/foundry/telemetry" +) + +// OperationEventPrefix uses the existing extension.event field, not new attributes. +// Names contain only allowlisted product classifications, never customer strings. +const OperationEventPrefix = "agent.operation.v1." + +// OperationClass describes an in-memory configuration, not an operation outcome. +type OperationClass struct { + Category string + Telephony string +} + +// ClassifyOperation reads only supplied properties. It never resolves references, +// accesses files, expands environment variables, validates, or mutates a service. +func ClassifyOperation(properties map[string]any) OperationClass { + result := OperationClass{Category: "unknown", Telephony: "unknown"} + if _, unresolved := properties["$ref"]; unresolved { + return result + } + text := func(key string) string { + value, _ := properties[key].(string) + return strings.ToLower(strings.TrimSpace(value)) + } + switch text("kind") { + case "hosted": + result.Category = "hosted" + protocols, _ := properties["protocols"].([]any) + for _, value := range protocols { + protocol, _ := value.(map[string]any) + name, _ := protocol["protocol"].(string) + if name == "invocations_ws" { + result.Category = "hosted_invocations_ws" + } + } + case "prompt": + result.Category = "prompt" + case "workflow": + result.Category = "workflow" + case "voice", "prompt-voice": + engine, hasEngine := properties["conversationEngine"] + if !hasEngine { + engine, hasEngine = properties["conversation_engine"] + } + if hasEngine { + fields, _ := engine.(map[string]any) + if fields["type"] == "hosted_agent" { + result.Category = "voice_hosted_wrapper" + } + } else { + value, present := properties["modelType"] + if !present { + value, present = properties["model_type"] + } + mode, valid := value.(string) + if present && !valid { + return result + } + mode = strings.ToLower(strings.TrimSpace(mode)) + switch mode { + case "", "managed": + result.Category = "voice_managed" + case "self_deployed": + result.Category = "voice_byom" + } + } + } + if result.Category == "unknown" { + return result + } + result.Telephony = "none" + if value, found := properties["telephony"]; found { + fields, ok := value.(map[string]any) + if !ok { + result.Telephony = "unknown" + } else if value, present := fields["bindings"]; present { + bindings, ok := value.([]any) + if !ok { + result.Telephony = "unknown" + } else if len(bindings) > 0 { + result.Telephony = "enabled" + } + } + } + return result +} + +// OperationClassified returns a bounded event, or false for an unrelated operation. +// Outcomes remain exclusively owned by the existing command completion telemetry. +func OperationClassified(operation string, class OperationClass) (foundryTelemetry.Event, bool) { + if !slices.Contains([]string{"init", "provision", "deploy"}, operation) { + return foundryTelemetry.Event{}, false + } + if !slices.Contains([]string{ + "hosted", "hosted_invocations_ws", "prompt", "workflow", + "voice_managed", "voice_byom", "voice_hosted_wrapper", "unknown", + }, class.Category) { + class.Category = "unknown" + } + if !slices.Contains([]string{"none", "enabled", "unknown"}, class.Telephony) { + class.Telephony = "unknown" + } + return foundryTelemetry.Event{ + Name: OperationEventPrefix + operation + "." + class.Category + "." + class.Telephony, + }, true +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/telemetry/operation_test.go b/cli/azd/extensions/azure.ai.agents/internal/telemetry/operation_test.go new file mode 100644 index 00000000000..b2eca233b23 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/internal/telemetry/operation_test.go @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package telemetry + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestOperationClassification(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + name string + properties map[string]any + want OperationClass + }{ + {"empty", nil, OperationClass{"unknown", "unknown"}}, + {"unrecognized", map[string]any{"kind": "customer-secret"}, OperationClass{"unknown", "unknown"}}, + {"hosted", map[string]any{"kind": "hosted", "name": "customer-secret"}, OperationClass{"hosted", "none"}}, + {"websocket", map[string]any{"kind": "hosted", "protocols": []any{ + map[string]any{"protocol": "responses"}, map[string]any{"protocol": "invocations_ws"}, + }}, OperationClass{"hosted_invocations_ws", "none"}}, + {"prompt", map[string]any{"kind": "prompt"}, OperationClass{"prompt", "none"}}, + {"workflow", map[string]any{"kind": "workflow"}, OperationClass{"workflow", "none"}}, + {"voice-default", map[string]any{"kind": "voice"}, OperationClass{"voice_managed", "none"}}, + {"voice-alias", map[string]any{"kind": "prompt-voice", "modelType": "managed"}, + OperationClass{"voice_managed", "none"}}, + {"byom", map[string]any{"kind": "voice", "modelType": "self_deployed"}, OperationClass{"voice_byom", "none"}}, + {"byom-legacy", map[string]any{"kind": "prompt-voice", "model_type": "self_deployed"}, + OperationClass{"voice_byom", "none"}}, + {"wrapper", map[string]any{"kind": "voice", "conversationEngine": map[string]any{"type": "hosted_agent"}}, + OperationClass{"voice_hosted_wrapper", "none"}}, + {"wrapper-legacy", map[string]any{"kind": "prompt-voice", + "conversation_engine": map[string]any{"type": "hosted_agent"}}, OperationClass{"voice_hosted_wrapper", "none"}}, + {"unknown-engine", map[string]any{"kind": "voice", "conversationEngine": nil}, + OperationClass{"unknown", "unknown"}}, + {"unknown-mode", map[string]any{"kind": "voice", "modelType": "customer-secret"}, + OperationClass{"unknown", "unknown"}}, + {"malformed-mode", map[string]any{"kind": "voice", "modelType": 42}, + OperationClass{"unknown", "unknown"}}, + {"unresolved-ref", map[string]any{"kind": "voice", "$ref": "customer-secret"}, + OperationClass{"unknown", "unknown"}}, + {"phone", map[string]any{"kind": "voice", "telephony": map[string]any{"bindings": []any{ + map[string]any{"identifier": "customer-secret", "connection": "customer-secret"}, + }}}, OperationClass{"voice_managed", "enabled"}}, + {"phone-empty", map[string]any{"kind": "voice", "telephony": map[string]any{"bindings": []any{}}}, + OperationClass{"voice_managed", "none"}}, + {"phone-malformed", map[string]any{"kind": "voice", "telephony": map[string]any{"bindings": "secret"}}, + OperationClass{"voice_managed", "unknown"}}, + } { + t.Run(tt.name, func(t *testing.T) { + before, err := json.Marshal(tt.properties) + require.NoError(t, err) + class := ClassifyOperation(tt.properties) + require.Equal(t, tt.want, class) + after, err := json.Marshal(tt.properties) + require.NoError(t, err) + require.Equal(t, before, after) + event, ok := OperationClassified("deploy", class) + require.True(t, ok) + require.Empty(t, event.Attributes, "no new telemetry attributes") + require.NotContains(t, event.Name, "customer-secret") + require.LessOrEqual(t, len(event.Name), 128) + }) + } +} + +func TestOperationEventVocabulary(t *testing.T) { + t.Parallel() + for _, operation := range []string{"init", "provision", "deploy"} { + event, ok := OperationClassified(operation, OperationClass{"unexpected-user-value", "secret"}) + require.True(t, ok) + require.Equal(t, "agent.operation.v1."+operation+".unknown.unknown", event.Name) + require.Nil(t, event.Attributes) + } + for _, operation := range []string{"", "invoke", "down", "customer-secret"} { + _, ok := OperationClassified(operation, OperationClass{}) + require.False(t, ok) + } +} diff --git a/docs/reference/telemetry-data.md b/docs/reference/telemetry-data.md index 240ebb695b1..8d9f51883e9 100644 --- a/docs/reference/telemetry-data.md +++ b/docs/reference/telemetry-data.md @@ -536,6 +536,7 @@ Reviewed first-party extension usage events currently include: |-----------|-------------------|---------|--------------------| | `azure.ai.agents` | `agent.context.resolved` | An agent command or lifecycle operation resolves an `azure.ai.agent` service | `ext.agent.kind`: `hosted`, `prompt`, `prompt-voice`, `voice`, `workflow`, or `unknown`; `ext.agent.harness`: `none`, `github_copilot_preview`, or `other`; `ext.agent.operation`: fixed extension command path; no agent names or customer content | | `azure.ai.agents` | `local_client.route.selected` | `azd ai agent run` resolves the service and protocol profile; emitted before client availability, agent startup, and client launch | `ext.route`: `inspector`, `playground`, or `suppressed`; suppression takes precedence | +| `azure.ai.agents` | `agent.operation.v1...` | Extension init returns or agent preprovision/predeploy is reached | No additional attributes. Bounded operation/category/telephony tokens in existing `extension.event`; join existing command results, never marker success. See [vocabulary, query and coverage limits](../../cli/azd/extensions/azure.ai.agents/docs/operation-telemetry.md). | | `azure.ai.inspector` | `inspector.funnel.stage` | The Inspector SPA sends `setViewReady` after mounting | `ext.stage=ui_ready`; `ext.outcome=succeeded`; this does not indicate agent connection | Source-category fields are classified from the configured source type and location, not the user-defined source name. diff --git a/docs/specs/metrics-audit/feature-telemetry-matrix.md b/docs/specs/metrics-audit/feature-telemetry-matrix.md index d23449e379b..f08d1e0f414 100644 --- a/docs/specs/metrics-audit/feature-telemetry-matrix.md +++ b/docs/specs/metrics-audit/feature-telemetry-matrix.md @@ -149,6 +149,12 @@ captures the operation type, making the attribute redundant: ## Cross-Cutting Subsystems +Agent operation classification additionally uses the existing `ext.usage` channel +with fixed `extension.event` values `agent.operation.v1...`. +No attributes are added. Coverage is partial: extension init return and agent +preprovision/predeploy hooks; core init and early failures may remain unknown. +This is not per-service outcome telemetry. See the [query and exact vocabulary](../../../cli/azd/extensions/azure.ai.agents/docs/operation-telemetry.md). + These telemetry surfaces are not tied to a single command — they emit from middleware or shared infrastructure invoked by many commands. They are included here so the privacy review covers every emission point. Dormant surfaces are retained to document diff --git a/docs/specs/metrics-audit/privacy-review-checklist.md b/docs/specs/metrics-audit/privacy-review-checklist.md index 87039a8ffb9..8b4a26fe7c8 100644 --- a/docs/specs/metrics-audit/privacy-review-checklist.md +++ b/docs/specs/metrics-audit/privacy-review-checklist.md @@ -25,6 +25,16 @@ attribution, tool-name normalization, and extension identity attribution. ## When to Trigger a Privacy Review +Agent operation marker proposal: `azure.ai.agents` adds bounded +`agent.operation.v1...` values to existing +`extension.event` on `ext.usage`. No new attributes, output payloads, model names, +paths, identifiers, credentials or hashing changes. Values are classified only +from already-held input/configuration; exact vocabulary is in the +[extension reference](../../../cli/azd/extensions/azure.ai.agents/docs/operation-telemetry.md). +The existing official-source gate and event budget remain enforced. Normal +extension privacy review and destination/query validation are required before +rollout; this note records the review scope, not completed approval. + A privacy review **must** be triggered when any of the following conditions are met: 1. **New telemetry field** — Any new attribute key added to diff --git a/docs/specs/metrics-audit/telemetry-schema.md b/docs/specs/metrics-audit/telemetry-schema.md index 1b4f8ecbdba..11acdc0c61e 100644 --- a/docs/specs/metrics-audit/telemetry-schema.md +++ b/docs/specs/metrics-audit/telemetry-schema.md @@ -294,12 +294,15 @@ guarantees about the whole class: | Trust | `extension.id` and `extension.version` are derived from host-signed claims; `extension.source` and eligibility are checked against the installed record and verified source config, never from the request | | Review | Extension telemetry is reviewed when the extension is admitted to the official registry, under the same documentation, classification, and privacy rules as core fields. The eligibility rule above is what ties recording to that review | + + Reviewed first-party event contracts: | Extension | `extension.event` | Trigger | Extension attributes | |-----------|-------------------|---------|----------------------| | `azure.ai.agents` | `agent.context.resolved` | An agent command or lifecycle operation resolves an `azure.ai.agent` service; one event per distinct kind/harness classification in the invocation | `ext.agent.kind`: fixed enum `hosted`, `prompt`, `prompt-voice`, `voice`, `workflow`, or `unknown`; `ext.agent.harness`: fixed enum `none`, `github_copilot_preview`, or `other`; `ext.agent.operation`: fixed extension command path; values contain no agent names or customer content | | `azure.ai.agents` | `local_client.route.selected` | `azd ai agent run` resolves the service and protocol profile; this precedes client availability, agent startup, and client launch | `ext.route`: fixed enum `inspector`, `playground`, or `suppressed`; suppression takes precedence | +| `azure.ai.agents` | `agent.operation.v1...` | Extension init RunE returns or an agent preprovision/predeploy hook is reached; deduplicated by the complete operation/classification tuple | **No extension attributes.** Existing `extension.event` encodes operation (`init`, `provision`, `deploy`), category (`hosted`, `hosted_invocations_ws`, `prompt`, `workflow`, `voice_managed`, `voice_byom`, `voice_hosted_wrapper`, `unknown`) and telephony (`none`, `enabled`, `unknown`). All values are allowlisted. Join existing command results; marker success is not business success. | | `azure.ai.inspector` | `inspector.funnel.stage` | The Inspector SPA sends `setViewReady` after mounting | `ext.stage`: fixed enum `ui_ready`; `ext.outcome`: fixed enum `succeeded`; this does not indicate agent connection | Because `ext.usage` spans share the command's trace, they join the originating @@ -479,6 +482,13 @@ Telemetry for the `infra.layers[]` parallel provisioning feature, emitted from ` ## Data Classifications +Agent operation markers add only fixed values to the existing `extension.event` +SystemMetadata / FeatureInsight field on `ext.usage`, not new fields or span names. +Format: `agent.operation.v1...`; the allowlisted +vocabulary and coverage are defined in the [extension operation reference](../../../cli/azd/extensions/azure.ai.agents/docs/operation-telemetry.md). +No customer strings, dynamic attributes or hashing changes are introduced; existing +agent-context and completion-result semantics remain unchanged. + Classifications are defined in `cli/azd/internal/tracing/fields/fields.go` and control how data is stored, retained, and who may access it.