Strengthen review contracts and HTTP guidance

- add outbound HttpClient transport and HTTP status review rules with paired fixtures`n- resolve layered action-skill overrides deterministically across enabled layers`n- validate findings reports and enforce measurable changed-fixture coverage
This commit is contained in:
demiliani 2026-09-23 15:31:21 +02:00
parent 07e324ddbc
commit 45ca57a23e
21 changed files with 830 additions and 34 deletions

View file

@ -30,9 +30,10 @@ function Assert-ThrowsLike {
}
$generator = Join-Path $Root 'tools/Build-SkillIndex.ps1'
$resolver = Join-Path $Root 'tools/Resolve-SkillWorklist.ps1'
$indexSchema = Join-Path $Root 'schemas/skill-index.schema.json'
$reportSchema = Join-Path $Root 'schemas/findings-report.schema.json'
foreach ($path in $generator, $indexSchema, $reportSchema) {
foreach ($path in $generator, $resolver, $indexSchema, $reportSchema) {
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
throw "Required contract file not found: $path"
}
@ -235,9 +236,83 @@ Output.
Assert-ThrowsLike -Pattern '*Nested super-skills are not supported*' -Action {
& $generator -BCQualityRoot $fixtureRoot -IndexPath (Join-Path $tmp 'nested.json')
}
Remove-Item -LiteralPath (Join-Path $fixtureSkills 'al-nested-review.md') -Force
$validSuper = @'
---
kind: action-skill
id: al-code-review
version: 1
title: Review
description: Test super-skill.
inputs: [file-path]
outputs: [findings-report]
sub-skills:
- microsoft/skills/review/al-leaf-review.md
---
# Review
## Source
Source.
## Relevance
Relevance.
## Worklist
Worklist.
## Action
Action.
## Output
Output.
'@
Set-Content -LiteralPath $superPath -Value $validSuper -Encoding utf8NoBOM
$customSkills = Join-Path -Path $fixtureRoot -ChildPath 'custom/skills/review'
New-Item -ItemType Directory -Path $customSkills -Force | Out-Null
$customLeaf = $leaf.Replace('title: Leaf', 'title: Custom Leaf')
Set-Content -LiteralPath (Join-Path $customSkills 'custom-leaf-review.md') -Value $customLeaf -Encoding utf8NoBOM
$layeredPath = Join-Path $tmp 'layered.json'
& $generator -BCQualityRoot $fixtureRoot -IndexPath $layeredPath | Out-Null
$layeredIndex = Get-Content -LiteralPath $layeredPath -Raw | ConvertFrom-Json
$layeredLeaves = @($layeredIndex.skills | Where-Object id -eq 'al-leaf-review')
if ($layeredLeaves.Count -ne 2) {
throw "Expected both layered al-leaf-review implementations, found $($layeredLeaves.Count)."
}
if ((@($layeredLeaves.layer | Sort-Object) -join ',') -cne 'custom,microsoft') {
throw 'Layered al-leaf-review implementations did not preserve custom and microsoft records.'
}
$resolved = & $resolver -BCQualityRoot $fixtureRoot -IndexPath $layeredPath -SuperSkillPath (
'microsoft/skills/review/al-code-review.md'
)
if ($resolved.subSkills.Count -ne 1 -or
$resolved.subSkills[0].path -cne 'custom/skills/review/custom-leaf-review.md') {
throw 'The custom implementation did not win the layered leaf slot.'
}
$microsoftOnly = & $resolver -BCQualityRoot $fixtureRoot -IndexPath $layeredPath -SuperSkillPath (
'microsoft/skills/review/al-code-review.md'
) -EnabledLayers microsoft
if ($microsoftOnly.subSkills.Count -ne 1 -or
$microsoftOnly.subSkills[0].path -cne 'microsoft/skills/review/al-leaf-review.md') {
throw 'Disabling the custom layer did not fall back to the Microsoft implementation.'
}
$customDisabled = & $resolver -BCQualityRoot $fixtureRoot -IndexPath $layeredPath -SuperSkillPath (
'microsoft/skills/review/al-code-review.md'
) -DisabledSkills 'custom/skills/review/custom-leaf-review.md'
if ($customDisabled.subSkills.Count -ne 1 -or
$customDisabled.subSkills[0].path -cne 'microsoft/skills/review/al-leaf-review.md') {
throw 'Disabling the custom implementation did not fall back to Microsoft.'
}
Set-Content -LiteralPath (Join-Path $customSkills 'duplicate-leaf-review.md') -Value $customLeaf -Encoding utf8NoBOM
Assert-ThrowsLike -Pattern '*Duplicate action-skill IDs within a layer: custom:al-leaf-review*' -Action {
& $generator -BCQualityRoot $fixtureRoot -IndexPath (Join-Path $tmp 'duplicate-layer.json')
}
}
finally {
Remove-Item -LiteralPath $tmp -Recurse -Force -ErrorAction SilentlyContinue
}
Write-Output "Skill-index check PASSED: deterministic, schema-valid, and all $($expectedLeaves.Count) review leaves preserved in order."
Write-Output "Skill-index check PASSED: deterministic, schema-valid, layered overrides resolved, and all $($expectedLeaves.Count) review leaves preserved in order."

View file

@ -688,12 +688,16 @@ def run(root: Path) -> Report:
if domain_dir.is_dir():
validate_samples_in_domain(domain_dir, root, report)
# Third pass: R24 unique ids within kind
# Third pass: R24 unique ids within kind. Layered action-skill overrides
# may share an id, but two definitions in one layer are ambiguous.
by_kind: dict[str, dict[str, list[Path]]] = {}
for rec in skill_records:
if rec.skill_id is None:
continue
by_kind.setdefault(rec.kind, {}).setdefault(rec.skill_id, []).append(rec.path)
scope = rec.kind
if rec.kind == "action-skill":
scope = f"{rec.kind}:{rec.path.relative_to(root).parts[0]}"
by_kind.setdefault(scope, {}).setdefault(rec.skill_id, []).append(rec.path)
for kind, by_id in by_kind.items():
for sid, paths in by_id.items():
if len(paths) > 1:

View file

@ -12,7 +12,26 @@ jobs:
steps:
- name: Check out repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
- name: Collect changed paths
shell: pwsh
env:
BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }}
run: |
$paths = if (-not $env:BASE_SHA -or $env:BASE_SHA -match '^0+$') {
@(git diff-tree --no-commit-id --name-only -r $env:GITHUB_SHA)
} else {
@(git diff --name-only $env:BASE_SHA $env:GITHUB_SHA)
}
$paths | Set-Content -LiteralPath "$env:RUNNER_TEMP/changed-paths.txt" -Encoding utf8NoBOM
- name: Validate review evaluation corpus
shell: pwsh
run: ./tools/Test-ReviewFixtures.ps1 -Root . -PrepareDirectory "$env:RUNNER_TEMP/bcquality-review-fixtures"
run: |
./tools/Test-ReviewFixtures.ps1 -Root . `
-PrepareDirectory "$env:RUNNER_TEMP/bcquality-review-fixtures" `
-ChangedPathsFile "$env:RUNNER_TEMP/changed-paths.txt" `
-CoverageReportPath "$env:RUNNER_TEMP/review-coverage.json"
Get-Content -LiteralPath "$env:RUNNER_TEMP/review-coverage.json"

View file

@ -44,6 +44,15 @@ Otherwise the layers are additive. A matching filename alone does not suppress
an article; the [READ contract](../skills/read.md#layer-precedence) governs
knowledge conflicts. Review reports record displaced knowledge in `suppressed`.
Action skills use the same layer order but override by frontmatter `id`. A
custom leaf with the same `id` as a Community or Microsoft leaf replaces that
leaf in every super-skill slot while its layer is enabled. The files may have
different names. IDs must remain unique within each layer. Disabling the custom
layer or the custom skill path makes composition fall back to the next enabled
implementation. Hosts should build the skill index and use
`tools/Resolve-SkillWorklist.ps1`; they must not implement this selection from
filenames.
Layer selection is **not an access-control boundary**. A plugin installation
still contains excluded layers on disk. An integration requiring genuine
exclusion must remove denied files from its own content copy before the agent

View file

@ -49,16 +49,19 @@ only result.
action skills to run. Do not reproduce its routing logic.
3. Execute every dispatched action skill with the exact input subset in its
dispatch record. Read `skills/read.md` and `skills/do.md` on demand.
4. When an action skill declares `sub-skills`, execute every relevant leaf as a
discrete invocation. Leaves are independent and may be scheduled serially
or concurrently.
4. When an action skill declares `sub-skills`, resolve its ordered leaf slots
with `tools/Resolve-SkillWorklist.ps1`, passing the enabled layers and
disabled skill paths from the task context. Execute every resolved leaf as
a discrete invocation. Leaves are independent and may be scheduled serially
or concurrently.
5. Capture the exact Task return as the immutable raw audit payload and primary
transport. Preserve it unchanged in private artifacts or host logs. Before
the full DO acceptance gate, create a normalized candidate only for DO's
bounded optional-range case, record that normalization separately in private
telemetry, and accept the candidate only if the entire copy passes the
unchanged strict gate. The accepted report contains no undeclared telemetry
fields.
unchanged strict gate. Use `tools/Validate-FindingsReport.ps1`, passing the
exact source paths and fully retrieved article paths. The accepted report
contains no undeclared telemetry fields.
6. Collect each accepted findings-report into `sub-results` in the declared
`sub-skills` order, not completion order. Run the super-skill self-review
only after all leaves have finished.
@ -92,6 +95,8 @@ A compatible runner:
- invokes every worklisted leaf exactly once unless a documented retry replaces
a failed attempt;
- resolves same-ID leaf implementations by `custom > community > microsoft`,
preserves declared slot order, and falls back when a higher layer is disabled;
- keeps leaf contexts isolated and passes only the inputs they declare;
- preserves each raw Task return unchanged for audit and distinguishes it from
any normalized accepted copy;

View file

@ -4,6 +4,15 @@ The evaluation is convention-driven. The harness discovers every `<layer>/skills
`review-fixtures.json` contains only global thresholds and optional exceptional overrides. An override may select a different `article`, add context when the generic convention cannot express a scenario, or use an `articles` array when one domain needs explicit regression coverage for several paired articles. Specify either `article` or `articles`, not both. The first selected article retains the stable `<domain>-bad` and `<domain>-good` manifest IDs; additional articles use slug-qualified IDs. Overrides should remain empty in the normal case.
CI also measures selected paired articles against every effective article that
has both AL companions. A changed paired article must be selected by the
domain convention or an override. When adding it would not provide a useful
deterministic regression, add a narrow `coverageWaivers` entry with its exact
article path and a non-empty reason. Waivers are reviewable exceptions, not a
substitute for domain coverage. The generated coverage report includes totals
and per-domain ratios; the ratio is informational, while changed-file coverage
is mandatory.
Model-facing preparation hashes case IDs, neutralizes `Good`/`Bad` object-name tokens, and removes full-line sample comments so neither the article slug, domain, nor expected outcome reveals the answer.
The SCM `articles` override deliberately selects every rule in the initial
@ -36,6 +45,13 @@ pwsh ./tools/Test-ReviewFixtures.ps1 -Root .
This credential-free check proves every selected leaf maps to a same-named knowledge domain with at least one complete AL sample pair and that all configured overrides are valid.
To reproduce the changed-file gate and emit the same measurable report as CI:
```powershell
git diff --name-only origin/main...HEAD | Set-Content .changed-paths.txt
pwsh ./tools/Test-ReviewFixtures.ps1 -Root . -ChangedPathsFile .changed-paths.txt -CoverageReportPath .coverage.json
```
## Run a fast-model evaluation
1. Prepare neutral inputs:

View file

@ -3,6 +3,7 @@
"selection": "first-paired-al-article",
"minimumExpectedRecall": 1.0,
"minimumCleanRate": 1.0,
"coverageWaivers": [],
"overrides": {
"agents": {
"article": "wire-all-three-agent-interfaces"
@ -95,7 +96,11 @@
"context": "The extended table existed in the previous app version and already contains rows."
},
"web-services": {
"article": "expose-systemid-as-the-api-key"
"articles": [
"expose-systemid-as-the-api-key",
"handle-httpclient-platform-failure-before-response-access",
"check-http-status-before-consuming-response-body"
]
}
}
}

View file

@ -0,0 +1,21 @@
codeunit 50100 "HTTP Status Handling Bad"
{
procedure GetCustomer(CustomerId: Guid): JsonObject
var
Client: HttpClient;
Response: HttpResponseMessage;
CustomerJson: JsonObject;
ResponseText: Text;
begin
if not Client.Get(
StrSubstNo('https://api.example.com/customers/%1', CustomerId),
Response)
then
Error('The customer service could not be reached.');
// A completed request can still contain a 4xx or 5xx error document.
Response.Content().ReadAs(ResponseText);
CustomerJson.ReadFrom(ResponseText);
exit(CustomerJson);
end;
}

View file

@ -0,0 +1,23 @@
codeunit 50100 "HTTP Status Handling Good"
{
procedure GetCustomer(CustomerId: Guid): JsonObject
var
Client: HttpClient;
Response: HttpResponseMessage;
CustomerJson: JsonObject;
ResponseText: Text;
begin
if not Client.Get(
StrSubstNo('https://api.example.com/customers/%1', CustomerId),
Response)
then
Error('The customer service could not be reached.');
if not Response.IsSuccessStatusCode() then
Error('The customer service returned HTTP status %1.', Response.HttpStatusCode());
Response.Content().ReadAs(ResponseText);
CustomerJson.ReadFrom(ResponseText);
exit(CustomerJson);
end;
}

View file

@ -0,0 +1,31 @@
---
bc-version: [all]
domain: web-services
keywords: [httpclient, httpresponsemessage, issuccessstatuscode, httpstatuscode, response-body, json]
technologies: [al]
countries: [w1]
application-area: [all]
---
# Check HTTP status before consuming the response body
## Description
A successful AL `HttpClient` call only confirms that the platform completed the HTTP exchange. The server can still return `4xx` or `5xx`, often with an error document whose shape differs from the expected success payload. Parsing that body as business data can produce misleading parse errors, incomplete records, or decisions based on an error response.
## Best Practice
After handling any platform or transport failure, check `HttpResponseMessage.IsSuccessStatusCode()` or the expected `HttpStatusCode()` before interpreting the response body as a success payload. Handle non-success status explicitly and include safe diagnostic context when appropriate. A bounded error body may be read for diagnostics, but it must not enter the success parsing path.
See sample: [`check-http-status-before-consuming-response-body.good.al`](check-http-status-before-consuming-response-body.good.al).
## Anti Pattern
Checking only the Boolean result of `Get`, `Post`, `Put`, `Delete`, or `Send` and then parsing `Response.Content()` as the expected payload. The Boolean can be `true` for any HTTP status, including authentication failures, throttling, validation errors, and server failures.
See sample: [`check-http-status-before-consuming-response-body.bad.al`](check-http-status-before-consuming-response-body.bad.al).
## References
- [HttpResponseMessage.IsSuccessStatusCode method](https://learn.microsoft.com/dynamics365/business-central/dev-itpro/developer/methods-auto/httpresponsemessage/httpresponsemessage-issuccessstatuscode-method)
- [HttpClient data type](https://learn.microsoft.com/dynamics365/business-central/dev-itpro/developer/methods-auto/httpclient/httpclient-data-type)

View file

@ -0,0 +1,18 @@
codeunit 50100 "Http Platform Failure Bad"
{
procedure GetCustomer(CustomerId: Guid): Text
var
Client: HttpClient;
Response: HttpResponseMessage;
ResponseText: Text;
RequestSucceeded: Boolean;
begin
RequestSucceeded := Client.Get(
StrSubstNo('https://api.example.com/customers/%1', CustomerId),
Response);
// Response content is unavailable when the platform call failed.
Response.Content().ReadAs(ResponseText);
exit(ResponseText);
end;
}

View file

@ -0,0 +1,18 @@
codeunit 50100 "Http Platform Failure Good"
{
procedure GetCustomer(CustomerId: Guid): Text
var
Client: HttpClient;
Response: HttpResponseMessage;
ResponseText: Text;
begin
if not Client.Get(
StrSubstNo('https://api.example.com/customers/%1', CustomerId),
Response)
then
Error('The customer service could not be reached.');
Response.Content().ReadAs(ResponseText);
exit(ResponseText);
end;
}

View file

@ -0,0 +1,31 @@
---
bc-version: [all]
domain: web-services
keywords: [httpclient, transport-failure, boolean-return, httpresponsemessage, content, runtime-error]
technologies: [al]
countries: [w1]
application-area: [all]
---
# Handle HttpClient platform failure before accessing the response
## Description
AL `HttpClient` methods can fail before a usable HTTP response exists because of an invalid request, DNS or network failure, certificate validation, timeout, a disabled extension setting, or the response-size limit. When code captures the optional Boolean return value, `false` reports this platform or transport failure. The accompanying `HttpResponseMessage` is not safe to consume; accessing its content after the failed call can raise another error and obscure the original failure.
## Best Practice
When capturing the Boolean return value from `Get`, `Post`, `Put`, `Delete`, or `Send`, stop the current response-processing path immediately when it is `false`. Report or propagate the transport failure without reading status, headers, or content. Omitting the optional Boolean is also valid when fail-fast behavior is intended: the runtime then raises an error if the operation cannot execute.
See sample: [`handle-httpclient-platform-failure-before-response-access.good.al`](handle-httpclient-platform-failure-before-response-access.good.al).
## Anti Pattern
Capturing a failed call in a Boolean and then reading `Response.Content()`, parsing the body, or otherwise treating `Response` as usable. Do not report omission of the Boolean by itself; that form deliberately delegates failure propagation to the runtime.
See sample: [`handle-httpclient-platform-failure-before-response-access.bad.al`](handle-httpclient-platform-failure-before-response-access.bad.al).
## References
- [HttpClient.Send method](https://learn.microsoft.com/dynamics365/business-central/dev-itpro/developer/methods-auto/httpclient/httpclient-send-method)
- [Call external services with HttpClient](https://learn.microsoft.com/dynamics365/business-central/dev-itpro/developer/devenv-httpclient)

View file

@ -46,8 +46,12 @@ The sub-skills invoked by this skill are those listed in frontmatter `sub-skills
Hosts that orchestrate leaves mechanically SHOULD run
`tools/Build-SkillIndex.ps1` and resolve this skill by `id: al-code-review`.
The generated `subSkills` array preserves the frontmatter order and avoids
host-specific Markdown parsing.
The generated `subSkills` array preserves the frontmatter slot order and
avoids host-specific Markdown parsing. Before invoking leaves, run
`tools/Resolve-SkillWorklist.ps1` with this skill's path and the task's enabled
layers and disabled skill paths. Each declared path supplies a leaf `id`; the
resolver selects the highest-precedence enabled implementation with that `id`
without changing slot order.
## Relevance

View file

@ -3,7 +3,7 @@ kind: action-skill
id: al-web-services-review
version: 1
title: AL web services review
description: Reviews AL API surfaces and webhook integration handlers against web-services guidance from BCQuality.
description: Reviews AL API surfaces, outbound HTTP integrations, and webhook handlers against web-services guidance from BCQuality.
inputs: [pr-diff, file-path, folder-path]
outputs: [findings-report]
bc-version: [all]
@ -14,7 +14,7 @@ application-area: [all]
# AL web services review
Reviews AL source changes against the `web-services` knowledge domain in BCQuality and emits a findings report. This is a leaf action skill: it invokes no sub-skills. It is one of the skills composed by `al-code-review`.
Reviews AL source changes against the `web-services` knowledge domain in BCQuality and emits a findings report. This includes inbound API surfaces, outbound HTTP integrations, and webhook lifecycle code. This is a leaf action skill: it invokes no sub-skills. It is one of the skills composed by `al-code-review`.
An orchestrator invokes this skill with a `pr-diff`, `file-path`, or `folder-path`. The skill produces a single JSON document conforming to the DO output contract.
@ -39,8 +39,9 @@ Narrow the relevant files to the subset that applies to the changes under review
- The changed AL object names and types — especially pages declared with `PageType = API`, API page `part` controls, queries declared with `QueryType = API`, and procedures that expose bound actions.
- The changed properties and triggers, weighted toward API page metadata (`APIPublisher`, `APIGroup`, `APIVersion`, `EntityName`, `EntitySetName`, `ODataKeyFields`, `SourceTable`, `SourceTableTemporary`), navigation metadata (`SubPageLink`, `Multiplicity`, and visible singleton or collection semantics), CRUD guards (`InsertAllowed`, `ModifyAllowed`, `DeleteAllowed`, `Editable`), the `OnOpenPage` trigger, and `OnValidate` triggers on exposed fields.
- Outbound HTTP integration code that constructs or sends requests, captures a client method's optional Boolean result, checks an `HttpResponseMessage`, or reads and parses response content.
- Webhook subscriber handlers and subscription lifecycle code, especially code that creates or renews subscriptions, handles `validationToken`, schedules from `expirationDateTime`, or targets resources whose eligibility is visible in the diff.
- Tokens extracted from the diff that relate to API surface and behaviour (`PageType`, `QueryType`, `API`, `api-page`, `page-part`, `APIPublisher`, `APIGroup`, `APIVersion`, `EntityName`, `EntitySetName`, `ODataKeyFields`, `SystemId`, `SubPageLink`, `subpagelink`, `Multiplicity`, `multiplicity`, `Many`, `ZeroOrOne`, `SourceTableTemporary`, `Job Queue Entry`, `webhook`, `webhookSupportedResources`, `webhook-supported-resources`, `subscriptions`, `notificationUrl`, `validationToken`, `validationtoken`, `expirationDateTime`, `expirationdatetime`, `ServiceEnabled`, `WebServiceActionContext`, `SetActionResponse`, `ReadIsolation`, `IsolationLevel`, `ReadCommitted`, `InsertAllowed`, `ModifyAllowed`, `DeleteAllowed`, `Editable`, `SourceTable`).
- Tokens extracted from the diff that relate to API surface and behaviour (`PageType`, `QueryType`, `API`, `api-page`, `page-part`, `APIPublisher`, `APIGroup`, `APIVersion`, `EntityName`, `EntitySetName`, `ODataKeyFields`, `SystemId`, `SubPageLink`, `subpagelink`, `Multiplicity`, `multiplicity`, `Many`, `ZeroOrOne`, `SourceTableTemporary`, `Job Queue Entry`, `webhook`, `webhookSupportedResources`, `webhook-supported-resources`, `subscriptions`, `notificationUrl`, `validationToken`, `validationtoken`, `expirationDateTime`, `expirationdatetime`, `ServiceEnabled`, `WebServiceActionContext`, `SetActionResponse`, `ReadIsolation`, `IsolationLevel`, `ReadCommitted`, `InsertAllowed`, `ModifyAllowed`, `DeleteAllowed`, `Editable`, `SourceTable`, `HttpClient`, `HttpRequestMessage`, `HttpResponseMessage`, `Get`, `Post`, `Put`, `Delete`, `Send`, `IsSuccessStatusCode`, `HttpStatusCode`, `Content`, `ReadAs`, `JsonObject`, `XmlDocument`).
A file enters the candidate worklist when its `keywords` intersect the extracted tokens or its topic (derived from the index entry's `path`, `title`, and `description`) matches a changed object type. Read an article's full file — its `## Best Practice` / `## Anti Pattern` bodies — only after it makes the worklist; candidate selection uses the index alone.
@ -56,7 +57,14 @@ For each worklist entry, evaluate the diff against the file's `## Best Practice`
- When the diff contains code that contradicts a Best Practice without being a full anti-pattern, emit `minor` with the same reference shape.
- Applicability alone is not a finding. Emit `info` only for a concrete, non-actionable observation the article explicitly defines; otherwise emit nothing when no violation is present.
For API parts whose parent declares `ODataKeyFields = SystemId`, detect a child foreign key linked to a parent business field instead of `Field(SystemId)`. Do not apply the SystemId-link rule to APIs intentionally keyed by another field. Omitted `Multiplicity` is valid and means the documented default 1:N collection; never report omission alone. Report an explicit `ZeroOrOne` only when the visible contract clearly intends a collection or deep insert, and report an explicit `Many` only when it clearly intends a singleton. Singleton metadata requires an explicit `ZeroOrOne`; do not infer singleton intent from naming alone. For webhook eligibility, detect `QueryType = API`, `SourceTableTemporary = true`, composite `ODataKeyFields` (including an omitted property when a visible source primary key is composite), Job Queue Entry, and visible system-table sources; do not infer an unknown table number. For lifecycle code, require both create and renew paths to use a handler that returns the query-string `validationToken` verbatim with `200 OK`, and flag renewal scheduling that assumes subscriptions are permanent instead of using `expirationDateTime`. Do not emit generic HTTP or REST advice.
For API parts whose parent declares `ODataKeyFields = SystemId`, detect a child foreign key linked to a parent business field instead of `Field(SystemId)`. Do not apply the SystemId-link rule to APIs intentionally keyed by another field. Omitted `Multiplicity` is valid and means the documented default 1:N collection; never report omission alone. Report an explicit `ZeroOrOne` only when the visible contract clearly intends a collection or deep insert, and report an explicit `Many` only when it clearly intends a singleton. Singleton metadata requires an explicit `ZeroOrOne`; do not infer singleton intent from naming alone. For webhook eligibility, detect `QueryType = API`, `SourceTableTemporary = true`, composite `ODataKeyFields` (including an omitted property when a visible source primary key is composite), Job Queue Entry, and visible system-table sources; do not infer an unknown table number. For lifecycle code, require both create and renew paths to use a handler that returns the query-string `validationToken` verbatim with `200 OK`, and flag renewal scheduling that assumes subscriptions are permanent instead of using `expirationDateTime`.
For outbound HTTP calls, apply these targeted checks:
- When code captures the optional Boolean result of `HttpClient.Get`, `Post`, `Put`, `Delete`, or `Send`, require the failed branch to stop before status, headers, or content are accessed. Do not report a call that omits the Boolean result: that form intentionally lets the runtime raise an error when the request cannot execute.
- After platform success, require `IsSuccessStatusCode()` or an explicit acceptable `HttpStatusCode()` check before response content is interpreted as a success payload. Do not report code that reads a bounded non-success body solely for diagnostics and keeps it out of the success parsing path.
Do not emit generic HTTP or REST advice beyond applicable knowledge articles.
Set `confidence` to:
@ -64,7 +72,7 @@ Set `confidence` to:
- `medium` when detection relies on heuristics or when any frontmatter dimension was `unknown`.
- `low` when the finding is an advisory derived only from applicability.
After evaluating each worklist entry, also consider whether the diff exhibits a web-services defect the agent recognises from its general AL knowledge that no knowledge file in the worklist covers. Such candidates are agent findings within this skill's domain — emit them with `references: []`, an `id` slug prefixed with `agent:`, `confidence` capped at `medium`, `severity` capped at `minor` (agent findings are advisory and non-gating), and a `message` that is self-contained (describing both the issue and a concrete recommendation, since there is no knowledge-file footer for the consumer to fall back on). Hold every candidate to the precision bar in `skills/do.md` (*Agent findings*): emit only a concrete, material web-services defect a knowledgeable BC reviewer would agree is wrong — steelman it first and drop anything stylistic, speculative, dependent on code outside the diff, or merely a valid alternative; when in doubt, omit. The scope is strictly API pages and web-service surfaces; defects outside this domain belong to other leaves and MUST NOT be emitted here. Before emitting, check the worklist for a knowledge file that matches the candidate — if one exists, upgrade the candidate to a knowledge-backed finding instead. See `skills/do.md` for the full contract.
After evaluating each worklist entry, also consider whether the diff exhibits a web-services defect the agent recognises from its general AL knowledge that no knowledge file in the worklist covers. Such candidates are agent findings within this skill's domain — emit them with `references: []`, an `id` slug prefixed with `agent:`, `confidence` capped at `medium`, `severity` capped at `minor` (agent findings are advisory and non-gating), and a `message` that is self-contained (describing both the issue and a concrete recommendation, since there is no knowledge-file footer for the consumer to fall back on). Hold every candidate to the precision bar in `skills/do.md` (*Agent findings*): emit only a concrete, material web-services defect a knowledgeable BC reviewer would agree is wrong — steelman it first and drop anything stylistic, speculative, dependent on code outside the diff, or merely a valid alternative; when in doubt, omit. The scope is strictly API pages, outbound HTTP integrations, and other web-service surfaces; defects outside this domain belong to other leaves and MUST NOT be emitted here. Before emitting, check the worklist for a knowledge file that matches the candidate — if one exists, upgrade the candidate to a knowledge-backed finding instead. See `skills/do.md` for the full contract.
For every emitted finding, decide whether the fix is mechanical. A fix is mechanical when it is small, local, and unambiguous from the diff context (for example: set `ODataKeyFields = SystemId`; add the three `*Allowed = false` guards to a read-only page; add the missing `OnOpenPage` isolation assignment). For mechanical findings, emit `findings[].suggested-code` with the literal replacement for the source lines indicated by `location`. The payload must be a verbatim replacement — no diff markers, no fences, no commentary — that the consumer can render as a one-click suggestion. When a `.good.al` companion exists and the diff context matches the `.bad.al` shape, adapt the `.good.al` replacement into `suggested-code`.
@ -74,7 +82,7 @@ Outcome selection:
- `completed` — the skill evaluated every worklist item; default when the skill finishes normally, including when the resulting `findings` array is empty.
- `no-knowledge` — no applicable web-services knowledge survived Source, Relevance, configuration filtering, and conflict resolution. `findings` is empty.
- `not-applicable` — the task context contains no AL API surface, JavaScript webhook subscription lifecycle code, or JavaScript notification handler, or the `technologies` filter rejected the task.
- `not-applicable` — the task context contains no AL API surface, outbound AL HTTP integration, JavaScript webhook subscription lifecycle code, or JavaScript notification handler, or the `technologies` filter rejected the task.
- `partial` — a time or token budget was hit before the worklist was exhausted. `summary.coverage` reflects the evaluated subset; `outcome-reason` explains the cause.
- `failed` — an unrecoverable error occurred. `outcome-reason` is required.

View file

@ -222,6 +222,11 @@ as a findings-report, a coordinator or host MUST validate it deterministically:
inclusive range identify existing lines with `start-line == line` and
`end-line >= start-line`.
Hosts SHOULD execute `tools/Validate-FindingsReport.ps1` with the exact source
scope and the leaf's recorded set of fully retrieved article paths. Pass
`-AllowBoundedNormalization` only when the host preserves the immutable raw
payload and records `removedRanges` in private telemetry as required above.
Validation failure invalidates the complete return; consumers MUST NOT salvage
individual findings, infer missing fields, reconstruct JSON, clamp ranges,
rewrite paths, or otherwise silently repair model output. Preserve the invalid
@ -356,16 +361,27 @@ performing any super-skill self-review or final rollup. Scheduling MUST NOT
change relevance, coverage, failure, reference-integrity, or output semantics.
Orchestrators SHOULD generate `skill-index.json` with
`tools/Build-SkillIndex.ps1` and consume the super-skill's ordered `subSkills`
from that index instead of parsing Markdown. Action-skill frontmatter remains
the source of truth; the generated index conforms to
`tools/Build-SkillIndex.ps1` instead of parsing Markdown. Each declared
`subSkills` path defines an ordered leaf slot: its indexed `id` identifies the
slot, while its path fixes the declaration order. Before scheduling leaves,
the orchestrator MUST resolve each slot to the highest-precedence enabled,
non-disabled leaf with that `id` (`custom` over `community` over `microsoft`).
If no implementation remains, the slot is skipped with `reason:
"configuration"`. The orchestrator SHOULD use
`tools/Resolve-SkillWorklist.ps1` for this resolution. Action-skill
frontmatter remains the source of truth; the generated index conforms to
`schemas/skill-index.schema.json`.
Layer resolution MUST NOT reorder slots. Multiple implementations with the
same `id` are valid only when they belong to different layers; duplicate IDs
within one layer are invalid. Disabling a winning implementation falls back
to the next enabled implementation for that slot when one exists.
### Section interpretation for super-skills
The five required sections still apply. Their meaning shifts from knowledge files to sub-skills:
- `## Source` — names the sub-skills invoked (mirrors `sub-skills` in frontmatter).
- `## Source` — names the declared sub-skill slots (mirrors `sub-skills` in frontmatter) and resolves their effective leaf implementations by the layered rule above.
- `## Relevance` — rules for deciding which sub-skills apply to the current task. A sub-skill is relevant when its declared `inputs` are satisfied by the orchestrator's provided inputs and the orchestrator has not disabled it via configuration. The super-skill MUST NOT filter sub-skills by task content (for example, by inspecting the diff or the file). Task-level applicability is the sub-skill's own responsibility; sub-skills signal non-applicability by returning `outcome: "not-applicable"` or `outcome: "no-knowledge"`.
- `## Worklist` — the final list of sub-skills to invoke; the rest go to `skipped-sub-skills`.
- `## Action` — invoke each worklisted sub-skill with the appropriate subset of inputs, collect its findings-report verbatim into `sub-results`, and copy its `findings[]` into the super-skill's top-level `findings[]` with `from-sub-skill` set. All finding fields, including the optional `domain`, are preserved verbatim unless this contract explicitly requires a transformation. Findings from a sub-skill with `outcome: "failed"` MUST NOT be copied into the super-skill's top-level `findings[]` and MUST NOT contribute to the super-skill's `summary.counts` (their report is still preserved in `sub-results` for traceability, consistent with DO's rule that consumers ignore a failed skill's findings).

View file

@ -203,9 +203,19 @@ foreach ($layer in 'microsoft', 'community', 'custom') {
}
}
$ids = @($records | Group-Object id | Where-Object Count -gt 1)
if ($ids.Count) {
throw "Duplicate action-skill IDs: $($ids.Name -join ', ')"
$idsWithinLayer = @(
$records |
Group-Object { "$($_.layer)`0$($_.id)" } |
Where-Object Count -gt 1
)
if ($idsWithinLayer.Count) {
$duplicates = @(
$idsWithinLayer | ForEach-Object {
$parts = $_.Name -split "`0", 2
"$($parts[0]):$($parts[1])"
}
)
throw "Duplicate action-skill IDs within a layer: $($duplicates -join ', ')"
}
foreach ($record in $records) {

View file

@ -0,0 +1,92 @@
<#
.SYNOPSIS
Resolves a super-skill's ordered leaf worklist across enabled layers.
#>
[CmdletBinding()]
param(
[string] $BCQualityRoot,
[string] $IndexPath,
[Parameter(Mandatory)]
[string] $SuperSkillPath,
[string[]] $EnabledLayers = @('microsoft', 'community', 'custom'),
[string[]] $DisabledSkills = @()
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
if (-not $BCQualityRoot) {
$BCQualityRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path
}
$BCQualityRoot = (Resolve-Path -LiteralPath $BCQualityRoot).Path
if (-not $IndexPath) {
$IndexPath = Join-Path $BCQualityRoot 'skill-index.json'
}
if (-not (Test-Path -LiteralPath $IndexPath -PathType Leaf)) {
& (Join-Path $PSScriptRoot 'Build-SkillIndex.ps1') -BCQualityRoot $BCQualityRoot -IndexPath $IndexPath | Out-Null
}
$knownLayers = @('microsoft', 'community', 'custom')
$unknownLayers = @($EnabledLayers | Where-Object { $_ -cnotin $knownLayers })
if ($unknownLayers.Count) {
throw "Unknown enabled layers: $($unknownLayers -join ', ')"
}
$index = Get-Content -LiteralPath $IndexPath -Raw | ConvertFrom-Json
$skills = @($index.skills)
$superSkills = @($skills | Where-Object path -CEQ $SuperSkillPath)
if ($superSkills.Count -ne 1) {
throw "Expected one indexed super-skill at '$SuperSkillPath', found $($superSkills.Count)."
}
$superSkill = $superSkills[0]
if (-not @($superSkill.subSkills).Count) {
throw "Action skill '$SuperSkillPath' is not a super-skill."
}
$precedence = @{ microsoft = 0; community = 1; custom = 2 }
$resolved = [Collections.Generic.List[object]]::new()
$skipped = [Collections.Generic.List[object]]::new()
foreach ($declaredPath in @($superSkill.subSkills)) {
$declared = @($skills | Where-Object path -CEQ $declaredPath)
if ($declared.Count -ne 1) {
throw "Declared sub-skill '$declaredPath' is not uniquely indexed."
}
$candidates = @(
$skills |
Where-Object {
$_.id -CEQ $declared[0].id -and
$_.layer -cin $EnabledLayers -and
$_.path -cnotin $DisabledSkills -and
-not @($_.subSkills).Count
} |
Sort-Object @{ Expression = { $precedence[$_.layer] }; Descending = $true }, path
)
if (-not $candidates.Count) {
$skipped.Add([pscustomobject][ordered]@{
id = $declared[0].id
declaredPath = $declaredPath
reason = 'configuration'
}) | Out-Null
continue
}
$winner = $candidates[0]
$resolved.Add([pscustomobject][ordered]@{
id = $winner.id
path = $winner.path
version = $winner.version
layer = $winner.layer
declaredPath = $declaredPath
}) | Out-Null
}
return [pscustomobject][ordered]@{
superSkill = [pscustomobject][ordered]@{
id = $superSkill.id
path = $superSkill.path
version = $superSkill.version
}
subSkills = @($resolved)
skipped = @($skipped)
}

View file

@ -1,12 +1,11 @@
<#
.SYNOPSIS
Validates the bounded leaf-range normalization contract.
Validates executable findings-report acceptance and bounded normalization.
.DESCRIPTION
BCQuality has no executable findings-report consumer. These assertions keep
the normative DO contract, AL coordinator, and standalone runner aligned
while exercising the exact normalization predicate against representative
safe and ambiguous inputs.
These assertions keep the normative DO contract, executable validator, AL
coordinator, and standalone runner aligned while exercising semantic report
validation and the exact normalization predicate.
#>
[CmdletBinding()]
param(
@ -39,6 +38,24 @@ function Assert-Contains {
Assert-True $Text.Contains($Expected) $Message
}
function Assert-ThrowsLike {
param(
[scriptblock] $Action,
[string] $Pattern
)
try {
& $Action
}
catch {
if ($_.Exception.Message -like $Pattern) {
return
}
throw "Expected error like '$Pattern', received: $($_.Exception.Message)"
}
throw "Expected error like '$Pattern', but no error was thrown."
}
function Test-PositiveInteger {
param([object] $Value)
@ -168,4 +185,85 @@ Assert-True (-not ($candidateFinding.location.PSObject.Properties.Name -contains
Assert-True ($candidateFinding.location.line -eq $rawFinding.location.line) 'candidate preserves the primary line'
Assert-True ($candidateFinding.message -ceq $rawFinding.message) 'candidate preserves all other finding content'
Write-Output "Review contract validation passed ($($cases.Count) normalization cases)."
$validator = Join-Path $Root 'tools/Validate-FindingsReport.ps1'
Assert-True (Test-Path -LiteralPath $validator -PathType Leaf) 'executable report validator exists'
$tmp = Join-Path ([IO.Path]::GetTempPath()) ("reviewcontract_" + [guid]::NewGuid().ToString('N'))
New-Item -ItemType Directory -Path $tmp -Force | Out-Null
try {
$sourcePath = 'src/codeunit.al'
$sourceFile = Join-Path $tmp 'src/codeunit.al'
New-Item -ItemType Directory -Path (Split-Path -Parent $sourceFile) -Force | Out-Null
Set-Content -LiteralPath $sourceFile -Value @('line one', 'line two', 'line three') -Encoding utf8NoBOM
$articlePath = 'microsoft/knowledge/style/caption-required-on-page-fields.md'
$reportPath = Join-Path $tmp 'report.json'
$validReport = [ordered]@{
skill = [ordered]@{ id = 'al-style-review'; version = 1 }
outcome = 'completed'
summary = [ordered]@{
counts = [ordered]@{ blocker = 0; major = 0; minor = 1; info = 0 }
coverage = [ordered]@{ 'worklist-size' = 1; 'items-evaluated' = 1 }
}
findings = @(
[ordered]@{
id = $articlePath
severity = 'minor'
message = 'A concrete style defect.'
location = [ordered]@{
file = $sourcePath
line = 2
range = [ordered]@{ 'start-line' = 2; 'end-line' = 3 }
}
references = @([ordered]@{ path = $articlePath })
confidence = 'high'
domain = 'Style'
}
)
suppressed = @()
}
Set-Content -LiteralPath $reportPath -Value ($validReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
$accepted = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
Assert-True (-not $accepted.normalized) 'valid report is accepted without normalization'
$invalidCounts = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
$invalidCounts.summary.counts.minor = 0
Set-Content -LiteralPath $reportPath -Value ($invalidCounts | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
Assert-ThrowsLike -Pattern '*COUNT_MISMATCH*' -Action {
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
}
Set-Content -LiteralPath $reportPath -Value ($validReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
Assert-ThrowsLike -Pattern '*REFERENCE_NOT_RETRIEVED*' -Action {
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp -SourcePaths $sourcePath
}
$invalidAgent = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
$invalidAgent.findings[0].id = 'agent:uncited-defect'
$invalidAgent.findings[0].references = @()
$invalidAgent.findings[0].confidence = 'high'
Set-Content -LiteralPath $reportPath -Value ($invalidAgent | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
Assert-ThrowsLike -Pattern '*AGENT_CONFIDENCE_INVALID*' -Action {
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp -SourcePaths $sourcePath
}
$normalizable = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
$normalizable.findings[0].location.range.'start-line' = 1
Set-Content -LiteralPath $reportPath -Value ($normalizable | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
Assert-ThrowsLike -Pattern '*RANGE_START_MISMATCH*' -Action {
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
}
$normalized = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath -AllowBoundedNormalization
Assert-True $normalized.normalized 'eligible range mismatch is normalized'
Assert-True ($normalized.removedRanges.Count -eq 1) 'normalization records one removed range'
Assert-True (-not ($normalized.report.findings[0].location.PSObject.Properties.Name -contains 'range')) `
'accepted normalized report removes only the optional range'
}
finally {
Remove-Item -LiteralPath $tmp -Recurse -Force -ErrorAction SilentlyContinue
}
Write-Output "Review contract validation passed ($($cases.Count) predicate cases plus executable acceptance cases)."

View file

@ -18,7 +18,9 @@ param(
[string] $ManifestPath,
[string] $PrepareDirectory,
[string] $ResultsPath,
[string] $ResultsDirectory
[string] $ResultsDirectory,
[string] $ChangedPathsFile,
[string] $CoverageReportPath
)
Set-StrictMode -Version Latest
@ -160,7 +162,23 @@ foreach ($overrideDomain in $overrides.Keys) {
}
}
$coverageWaivers = @{}
if ($manifest.PSObject.Properties.Name -contains 'coverageWaivers') {
foreach ($waiver in @($manifest.coverageWaivers)) {
if (-not $waiver.path -or -not $waiver.reason) {
$problems.Add('Each coverage waiver requires non-empty path and reason values.') | Out-Null
continue
}
if ($coverageWaivers.ContainsKey([string]$waiver.path)) {
$problems.Add("Duplicate coverage waiver: $($waiver.path)") | Out-Null
continue
}
$coverageWaivers[[string]$waiver.path] = [string]$waiver.reason
}
}
$caseList = [System.Collections.Generic.List[object]]::new()
$pairedArticlesByDomain = @{}
foreach ($domain in $leafDomains) {
$articleCandidates = @(
foreach ($layer in $layers) {
@ -189,6 +207,7 @@ foreach ($domain in $leafDomains) {
ForEach-Object { $_.Group | Sort-Object Rank -Descending | Select-Object -First 1 } |
Sort-Object BaseName
)
$pairedArticlesByDomain[$domain] = @($articles)
if (-not $articles.Count) {
$problems.Add("${domain}: no enabled knowledge layer has an article with both .good.al and .bad.al companion samples.") | Out-Null
continue
@ -339,6 +358,65 @@ foreach ($domain in $leafDomains) {
}
}
$selectedArticlePaths = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
foreach ($case in $cases) {
foreach ($reference in @($case.expected)) {
$selectedArticlePaths.Add([string]$reference) | Out-Null
}
}
$effectivePairedPaths = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
$coverageDomains = @(
foreach ($domain in $leafDomains) {
$paired = @($pairedArticlesByDomain[$domain])
foreach ($article in $paired) {
$effectivePairedPaths.Add([string]$article.ArticlePath) | Out-Null
}
$selected = @($paired | Where-Object { $selectedArticlePaths.Contains([string]$_.ArticlePath) }).Count
[pscustomobject][ordered]@{
domain = $domain
pairedArticles = $paired.Count
selectedArticles = $selected
coverage = if ($paired.Count) { $selected / $paired.Count } else { 0 }
}
}
)
$pairedTotal = ($coverageDomains | Measure-Object pairedArticles -Sum).Sum
$selectedTotal = ($coverageDomains | Measure-Object selectedArticles -Sum).Sum
$coverageReport = [pscustomobject][ordered]@{
pairedArticles = $pairedTotal
selectedArticles = $selectedTotal
coverage = if ($pairedTotal) { $selectedTotal / $pairedTotal } else { 0 }
domains = $coverageDomains
}
if ($CoverageReportPath) {
$coverageParent = Split-Path -Parent $CoverageReportPath
if ($coverageParent -and -not (Test-Path -LiteralPath $coverageParent)) {
New-Item -ItemType Directory -Path $coverageParent -Force | Out-Null
}
$coverageReport | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $CoverageReportPath -Encoding utf8NoBOM
}
if ($ChangedPathsFile) {
if (-not (Test-Path -LiteralPath $ChangedPathsFile -PathType Leaf)) {
$problems.Add("Changed paths file not found: $ChangedPathsFile") | Out-Null
}
else {
foreach ($changedPathValue in Get-Content -LiteralPath $ChangedPathsFile) {
$changedPath = ([string]$changedPathValue).Trim().Replace('\', '/')
if ($changedPath -notmatch '^(microsoft|community|custom)/knowledge/[^/]+/(.+?)(?:\.(?:good|bad)\.al|\.md)$') {
continue
}
$articlePath = "$($Matches[1])/knowledge/$($changedPath.Split('/')[2])/$($Matches[2]).md"
if (-not $effectivePairedPaths.Contains($articlePath) -or $selectedArticlePaths.Contains($articlePath)) {
continue
}
if (-not $coverageWaivers.ContainsKey($articlePath)) {
$problems.Add("Changed paired article is not selected for evaluation and has no coverage waiver: $articlePath") | Out-Null
}
}
}
}
if ($problems.Count) {
Write-Host "Review fixture validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
@ -474,7 +552,7 @@ if ($PrepareDirectory) {
if (-not $ResultsPath -and -not $ResultsDirectory) {
& (Join-Path $PSScriptRoot 'Test-ReviewContract.ps1') -Root $Root
Write-Host "Review fixture validation PASSED: $($cases.Count) cases cover $($leafDomains.Count) leaf domains." -ForegroundColor Green
Write-Host "Review fixture validation PASSED: $($cases.Count) cases cover $selectedTotal/$pairedTotal paired articles across $($leafDomains.Count) leaf domains." -ForegroundColor Green
exit 0
}

View file

@ -0,0 +1,215 @@
<#
.SYNOPSIS
Validates a BCQuality findings-report against its structural and semantic contract.
#>
[CmdletBinding()]
param(
[Parameter(Mandatory)]
[string] $ReportPath,
[string] $BCQualityRoot,
[string] $SourceRoot,
[string[]] $SourcePaths = @(),
[string[]] $RetrievedArticlePaths = @(),
[switch] $AllowBoundedNormalization
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
if (-not $BCQualityRoot) {
$BCQualityRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path
}
$BCQualityRoot = (Resolve-Path -LiteralPath $BCQualityRoot).Path
$schemaPath = Join-Path $BCQualityRoot 'schemas/findings-report.schema.json'
$raw = Get-Content -LiteralPath $ReportPath -Raw
try {
if (-not ($raw | Test-Json -SchemaFile $schemaPath -ErrorAction Stop)) {
throw 'Report does not satisfy schemas/findings-report.schema.json.'
}
$report = $raw | ConvertFrom-Json -Depth 100
}
catch {
throw "Invalid findings-report JSON or schema: $($_.Exception.Message)"
}
$retrieved = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
foreach ($path in $RetrievedArticlePaths) {
$retrieved.Add($path) | Out-Null
}
$sources = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
foreach ($path in $SourcePaths) {
$sources.Add($path) | Out-Null
}
$lineCounts = @{}
function Test-HasProperty {
param([object] $Object, [string] $Name)
return $null -ne $Object -and $Object.PSObject.Properties.Name -ccontains $Name
}
function Get-SourceLineCount {
param([string] $Path)
if ($lineCounts.ContainsKey($Path)) {
return $lineCounts[$Path]
}
if (-not $SourceRoot) {
return -1
}
$fullPath = Join-Path $SourceRoot ($Path -replace '/', [IO.Path]::DirectorySeparatorChar)
if (-not (Test-Path -LiteralPath $fullPath -PathType Leaf)) {
return -1
}
$lineCounts[$Path] = [IO.File]::ReadAllLines($fullPath).Count
return $lineCounts[$Path]
}
function Get-SemanticErrors {
param(
[object] $Candidate,
[switch] $PermitRangeStartMismatch
)
$errors = [Collections.Generic.List[object]]::new()
function Add-Error {
param([string] $Code, [string] $Path, [string] $Message)
$errors.Add([pscustomobject]@{ Code = $Code; Path = $Path; Message = $Message }) | Out-Null
}
function Test-Report {
param([object] $Current, [string] $ReportPathPrefix)
$findings = @($Current.findings)
foreach ($severity in 'blocker', 'major', 'minor', 'info') {
$actual = @($findings | Where-Object severity -CEQ $severity).Count
if ($Current.summary.counts.$severity -ne $actual) {
Add-Error 'COUNT_MISMATCH' "$ReportPathPrefix.summary.counts.$severity" "Expected $actual."
}
}
if ($Current.summary.coverage.'items-evaluated' -gt $Current.summary.coverage.'worklist-size') {
Add-Error 'COVERAGE_INVALID' "$ReportPathPrefix.summary.coverage" 'items-evaluated exceeds worklist-size.'
}
for ($index = 0; $index -lt $findings.Count; $index++) {
$finding = $findings[$index]
$findingPath = "$ReportPathPrefix.findings[$index]"
$references = @($finding.references)
if (-not $references.Count) {
if ($finding.id -cnotmatch '(^|:)agent:[a-z0-9]+(?:-[a-z0-9]+)*$') {
Add-Error 'AGENT_ID_INVALID' "$findingPath.id" 'An agent finding id must contain an agent: slug marker.'
}
if ($finding.confidence -ceq 'high') {
Add-Error 'AGENT_CONFIDENCE_INVALID' "$findingPath.confidence" 'Agent confidence cannot be high.'
}
if ($finding.severity -cin @('blocker', 'major')) {
Add-Error 'AGENT_SEVERITY_INVALID' "$findingPath.severity" 'Agent severity cannot exceed minor.'
}
}
else {
if ($finding.id -cne $references[0].path) {
Add-Error 'PRIMARY_REFERENCE_MISMATCH' "$findingPath.id" 'Finding id must equal the primary reference path.'
}
foreach ($reference in $references) {
if ($reference.path -cnotmatch '^(microsoft|community|custom)/knowledge/.+\.md$') {
Add-Error 'REFERENCE_PATH_INVALID' "$findingPath.references" "Invalid knowledge path '$($reference.path)'."
continue
}
if (-not (Test-Path -LiteralPath (Join-Path $BCQualityRoot $reference.path) -PathType Leaf)) {
Add-Error 'REFERENCE_MISSING' "$findingPath.references" "Knowledge path '$($reference.path)' does not exist."
}
if (-not $retrieved.Contains($reference.path)) {
Add-Error 'REFERENCE_NOT_RETRIEVED' "$findingPath.references" "Knowledge path '$($reference.path)' was not retrieved in full."
}
}
}
if (Test-HasProperty $finding 'location') {
$location = $finding.location
if (-not $sources.Contains($location.file)) {
Add-Error 'SOURCE_OUT_OF_SCOPE' "$findingPath.location.file" "Source path '$($location.file)' is outside the supplied scope."
}
$lineCount = Get-SourceLineCount $location.file
if ($lineCount -lt 0) {
Add-Error 'SOURCE_MISSING' "$findingPath.location.file" "Source path '$($location.file)' does not exist."
}
elseif ($location.line -gt $lineCount) {
Add-Error 'SOURCE_LINE_INVALID' "$findingPath.location.line" "Line exceeds the file's $lineCount lines."
}
if (Test-HasProperty $location 'range') {
$range = $location.range
if ($range.'start-line' -ne $location.line) {
Add-Error 'RANGE_START_MISMATCH' "$findingPath.location.range.start-line" 'start-line must equal line.'
}
if ($range.'end-line' -lt $range.'start-line' -or
($lineCount -ge 0 -and $range.'end-line' -gt $lineCount)) {
Add-Error 'SOURCE_RANGE_INVALID' "$findingPath.location.range" 'Range is reversed or exceeds the source file.'
}
}
}
}
if (Test-HasProperty $Current 'sub-results') {
$subResults = @($Current.'sub-results')
for ($index = 0; $index -lt $subResults.Count; $index++) {
Test-Report $subResults[$index] "$ReportPathPrefix.sub-results[$index]"
}
}
}
Test-Report $Candidate '$'
if ($PermitRangeStartMismatch) {
return @($errors | Where-Object Code -CNE 'RANGE_START_MISMATCH')
}
return @($errors)
}
$errors = @(Get-SemanticErrors $report)
$normalized = $false
$removedRanges = [Collections.Generic.List[object]]::new()
if ($errors.Count -and $AllowBoundedNormalization) {
$otherErrors = @($errors | Where-Object Code -CNE 'RANGE_START_MISMATCH')
$rangeErrors = @($errors | Where-Object Code -CEQ 'RANGE_START_MISMATCH')
if (-not $otherErrors.Count -and $rangeErrors.Count) {
$candidate = $report | ConvertTo-Json -Depth 100 | ConvertFrom-Json -Depth 100
$eligible = $true
foreach ($finding in @($candidate.findings)) {
if (-not (Test-HasProperty $finding 'location') -or
-not (Test-HasProperty $finding.location 'range') -or
$finding.location.range.'start-line' -eq $finding.location.line) {
continue
}
$range = $finding.location.range
if ($range.'start-line' -gt $finding.location.line -or
$finding.location.line -gt $range.'end-line' -or
(Test-HasProperty $finding 'suggested-code')) {
$eligible = $false
break
}
$removedRanges.Add([pscustomobject]@{
findingId = $finding.id
file = $finding.location.file
line = $finding.location.line
startLine = $range.'start-line'
endLine = $range.'end-line'
}) | Out-Null
$finding.location.PSObject.Properties.Remove('range')
}
if ($eligible -and -not @(Get-SemanticErrors $candidate).Count) {
$report = $candidate
$normalized = $true
$errors = @()
}
}
}
if ($errors.Count) {
$details = @($errors | ForEach-Object { "$($_.Code) at $($_.Path): $($_.Message)" }) -join '; '
throw "Findings-report acceptance failed: $details"
}
return [pscustomobject][ordered]@{
normalized = $normalized
report = $report
removedRanges = @($removedRanges)
}