diff --git a/.github/scripts/Test-SkillIndex.ps1 b/.github/scripts/Test-SkillIndex.ps1 index ab560067..f4b48eeb 100644 --- a/.github/scripts/Test-SkillIndex.ps1 +++ b/.github/scripts/Test-SkillIndex.ps1 @@ -110,9 +110,12 @@ try { throw 'A knowledge-only request must exclude review actions.' } # Entry applies READ's non-empty technology intersection to indexed actions. - $alKnowledge = @($knowledgeCandidates | Where-Object { 'al' -in $_.filters.technologies }) - if ($alKnowledge.Count -ne 1 -or $alKnowledge[0].id -ne 'al-knowledge') { - throw 'Entry must admit knowledge-query with technologies: [al].' + # Control add-in consultations may supply only JavaScript context. + foreach ($technology in 'al', 'javascript') { + $admitted = @($knowledgeCandidates | Where-Object { $technology -in $_.filters.technologies }) + if ($admitted.Count -ne 1 -or $admitted[0].id -ne 'al-knowledge') { + throw "Entry must admit knowledge-query with technologies: [$technology]." + } } foreach ($dimension in 'bc-version', 'countries', 'application-area') { $expected = if ($dimension -eq 'countries') { 'w1' } else { 'all' } diff --git a/.github/scripts/test_knowledge_response_semantics.py b/.github/scripts/test_knowledge_response_semantics.py index 42a79912..93c5c46b 100644 --- a/.github/scripts/test_knowledge_response_semantics.py +++ b/.github/scripts/test_knowledge_response_semantics.py @@ -265,6 +265,27 @@ def test_duplicate_read_evidence(self): self.evidence["reads"].append(copy.deepcopy(self.evidence["reads"][0])) self.assertTrue(self.errors()) + def test_duplicate_citation_with_contradictory_applicability(self): + duplicate = copy.deepcopy(self.response["references"][0]) + duplicate.update(applicability="conditional", unknown=["bc-version"]) + self.response["references"].append(duplicate) + self.assertEqual(self.errors(), [f"Duplicate citation path: {self.path}"]) + + def test_cited_article_cannot_be_suppressed(self): + for reason in ("layer-precedence", "configuration"): + with self.subTest(reason=reason): + response = copy.deepcopy(self.response) + response["suppressed"] = [{"path": self.path, "reason": reason}] + errors = module.validate_response(response, question=self.question, root=self.root, + evidence=self.evidence, run_id="test-run") + self.assertEqual(errors, [f"Path is both cited and suppressed: {self.path}"]) + + def test_duplicate_suppression_path(self): + self.response.update(outcome="no-knowledge", references=[], suppressed=[ + {"path": self.path, "reason": "layer-precedence"}, + {"path": self.path, "reason": "configuration"}]) + self.assertEqual(self.errors(evidence=False), [f"Duplicate suppression path: {self.path}"]) + def test_cli_preserves_question_and_rejects_changes(self): question = self.root / "question.txt" question.write_bytes(self.question.encode()) diff --git a/docs/knowledge-response-validation.md b/docs/knowledge-response-validation.md index 49b4c659..ba812a34 100644 --- a/docs/knowledge-response-validation.md +++ b/docs/knowledge-response-validation.md @@ -80,8 +80,10 @@ python tools/validate_knowledge_response.py response.json ` The tool checks the response schema, exact question, existing canonical citation paths within the corpus, and complete-read evidence with matching current file -hashes and byte counts. A changed article, truncated read, missing evidence, -different-layer path with the same filename, or mismatched run is rejected. +hashes and byte counts. Each article path may appear only once across +`references` and `suppressed`. A changed article, truncated read, missing +evidence, different-layer path with the same filename, mismatched run, duplicate +path, or article both cited and suppressed is rejected. The tool never repairs or rewrites the response. Exit codes: `0` means these checks passed, `1` means validation failed, and `2` diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 66164717..1da23b00 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -13,8 +13,9 @@ start with `/`. | The CLI has no `plugin` command | Update Copilot CLI using its installation method. Confirm `copilot plugin --help` works. | | Sign-in, entitlement, or organization-policy error | Start `copilot`, use `/login`, and confirm your account is allowed to use Copilot CLI. Ask your administrator about organization restrictions; BCQuality cannot override them. | | Plugin installation cannot reach the repository | Confirm access to `https://github.com/microsoft/BCQuality` and follow your organization's proxy/network guidance. Do not disable certificate checks. | -| The plugin installed, but the skill is missing | Run `copilot plugin list` in the terminal. Enable it with `copilot plugin enable bcquality` if disabled, then start a new session. In the session, use `/skills list` and look for `al-code-review` and `al-plan-guidance`. If only `al-code-review` appears, run `copilot plugin update bcquality`. | +| The plugin installed, but the skill is missing | Run `copilot plugin list` in the terminal. Enable it with `copilot plugin enable bcquality` if disabled, then start a new session. In the session, use `/skills list` and look for `al-code-review`, `al-knowledge`, and `al-plan-guidance`. If `al-knowledge` or `al-plan-guidance` is missing, run `copilot plugin update bcquality`, then start a new session. | | The agent performs a generic review | Name the **installed `al-code-review` skill** explicitly, as in the quick start. Ask which skill and BCQuality source it used. Another plugin or host may expose a similarly named operation. | +| The agent answers a question without BCQuality citations | Name the **installed `al-knowledge` skill** explicitly, as in the [knowledge consultation example](knowledge-consultation.md#ask-a-question), and ask for the `knowledge-response` unchanged. An answer without cited articles is not BCQuality guidance. | | Installation works in the terminal, but not in the editor | Plugin discovery is host-specific. Follow the editor's installation instructions; a CLI installation is not proof that another host loaded the plugin. | | `pwsh` is missing, or index generation fails | The index is an accelerator, not required knowledge. The review can fall back to discovery from folders. For faster discovery, install [PowerShell 7](https://learn.microsoft.com/en-us/powershell/scripting/install/installing-powershell), or resolve the reported filesystem error. | | An update or local edit is not visible | Run `copilot plugin update bcquality` for a repository-installed plugin, then start a fresh session. For a directly installed local folder, reinstall that folder to refresh the cached copy; see [customizing](customizing-bcquality.md#use-your-fork). | @@ -26,7 +27,7 @@ start with `/`. | `partial`, a timeout, or an unfinished review | Read `outcome-reason` and domain reports. Retry the incomplete scope in a fresh session, use smaller app folders or a focused review, or select a host/model with sufficient capacity. Keep the limited scope visible; do not relabel it a complete app review. | | `failed` | Resolve the stated problem, such as inaccessible input, a failed invocation, or an unverifiable reference, before using that report. A failed domain's findings are not reliable. | | `no-match` or `not-applicable` | Confirm you supplied AL source, the intended folder/file/diff, and an appropriate goal. Check [disabled skills and layers](customizing-bcquality.md#select-layers-or-disable-a-review). | -| `no-knowledge` | Check the target BC version, selected domain, enabled layers, and whether the relevant knowledge files are present. No applicable rules is different from no defects. | +| `no-knowledge` | Check the target BC version, selected domain, enabled layers, and whether the relevant knowledge files are present. No applicable rules is different from no defects. From `al-knowledge`, it means no installed article answers the question: a coverage gap, not a negative answer. | | `completed` with no findings | This can be a valid clean result for the selected scope. Confirm the intended files and domain reports are included. If you have a concrete missed defect, report it with a minimal example. | | JSON rather than a readable summary | JSON is the shared output format. Ask the host to summarize the existing reports, preserving outcomes, locations, severity, confidence, and references. | | A surprising finding | Open its guidance and samples, inspect surrounding code, and confirm version/localization assumptions. Ask the agent to explain the evidence; do not apply a suggestion solely because it has high confidence. | diff --git a/docs/using-bcquality.md b/docs/using-bcquality.md index b6606767..ebf93760 100644 --- a/docs/using-bcquality.md +++ b/docs/using-bcquality.md @@ -12,7 +12,7 @@ and rendering; BCQuality does not install a BC extension or an agent. | Path | What to do | | --- | --- | | Read the knowledge yourself | Browse [knowledge by domain](#knowledge-by-domain), or search the repository for an AL concept. Read the article and its samples. No installation required. | -| Use a supplied skill | Follow the [plugin quick start](../README.md#quick-start). `al-code-review` reviews code and returns findings; the preview `al-plan-guidance` enriches an existing development plan before coding. | +| Use a supplied skill | Follow the [plugin quick start](../README.md#quick-start). `al-code-review` reviews code and returns findings; `al-knowledge` answers a development question with cited knowledge; the preview `al-plan-guidance` enriches an existing development plan before coding. | | Use your own agent or workflow | Supply selected articles as context, as described below, or use the [integration bootstrap](agent-consumption.md#try-a-minimal-integration) to execute BCQuality action skills without the plugin. | | Get guidance before coding (preview) | Ask the plugin's `al-plan-guidance` skill to enrich an existing plan, then review the finished change. See [plan guidance](../README.md#preview-guidance-before-implementation). | @@ -66,8 +66,8 @@ integration protocol. The remaining sections describe the review workflow. The [quick start](../README.md#quick-start) documents GitHub Copilot CLI. Use a current CLI release with plugin support and sign in to an account allowed to use it. In an interactive CLI session, `/skills list` should include -`al-code-review` and `al-plan-guidance`; in the terminal, `copilot plugin list` -should include `bcquality`. +`al-code-review`, `al-knowledge`, and `al-plan-guidance`; in the terminal, +`copilot plugin list` should include `bcquality`. Do not assume a CLI installation also installs the plugin into VS Code, another editor, or another agent host. Follow that host's plugin instructions diff --git a/microsoft/skills/knowledge/al-knowledge.md b/microsoft/skills/knowledge/al-knowledge.md index 4f32c727..42d45c86 100644 --- a/microsoft/skills/knowledge/al-knowledge.md +++ b/microsoft/skills/knowledge/al-knowledge.md @@ -7,7 +7,7 @@ description: Answer a Business Central development question with cited knowledge inputs: [knowledge-query] outputs: [knowledge-response] bc-version: [all] -technologies: [al] +technologies: [al, javascript] countries: [w1] application-area: [all] --- diff --git a/skills/do.md b/skills/do.md index 4a3721b7..d47a2eff 100644 --- a/skills/do.md +++ b/skills/do.md @@ -502,6 +502,7 @@ an expected pin. A conditional reference lists each unknown applicability dimension in `unknown[]`; fully applicable references use `applicable` with an empty `unknown[]`. READ defines matching and conflict precedence. Record displaced articles in `suppressed[]` without citing them as supporting guidance. +Each article path appears at most once across `references[]` and `suppressed[]`. `completed` requires a useful answer to the bound question and at least one verified supporting reference. Applicability under READ does not establish diff --git a/tools/validate_knowledge_response.py b/tools/validate_knowledge_response.py index cab9776c..4e47e896 100644 --- a/tools/validate_knowledge_response.py +++ b/tools/validate_knowledge_response.py @@ -10,6 +10,7 @@ import argparse import hashlib import json +from collections import Counter from pathlib import Path from jsonschema import Draft7Validator @@ -24,6 +25,11 @@ def validate_response(response, *, question, root, evidence=None, run_id, enable return errors if response["question"] != question: errors.append("question must equal the bound input exactly, including whitespace") + cited = Counter(reference["path"] for reference in response["references"]) + excluded = Counter(entry["path"] for entry in response["suppressed"]) + errors += [f"Duplicate citation path: {name}" for name in sorted(cited) if cited[name] > 1] + errors += [f"Duplicate suppression path: {name}" for name in sorted(excluded) if excluded[name] > 1] + errors += [f"Path is both cited and suppressed: {name}" for name in sorted(cited.keys() & excluded.keys())] known_layers = {"microsoft", "community", "custom"} enabled_layers = set(enabled_layers) if not enabled_layers <= known_layers: