Strengthen review contracts and HTTP guidance

- add outbound HttpClient transport and HTTP status review rules with paired fixtures`n- resolve layered action-skill overrides deterministically across enabled layers`n- validate findings reports and enforce measurable changed-fixture coverage
This commit is contained in:
demiliani 2026-09-23 15:31:21 +02:00
parent 07e324ddbc
commit 45ca57a23e
21 changed files with 830 additions and 34 deletions

View file

@ -30,9 +30,10 @@ function Assert-ThrowsLike {
}
$generator = Join-Path $Root 'tools/Build-SkillIndex.ps1'
$resolver = Join-Path $Root 'tools/Resolve-SkillWorklist.ps1'
$indexSchema = Join-Path $Root 'schemas/skill-index.schema.json'
$reportSchema = Join-Path $Root 'schemas/findings-report.schema.json'
foreach ($path in $generator, $indexSchema, $reportSchema) {
foreach ($path in $generator, $resolver, $indexSchema, $reportSchema) {
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
throw "Required contract file not found: $path"
}
@ -235,9 +236,83 @@ Output.
Assert-ThrowsLike -Pattern '*Nested super-skills are not supported*' -Action {
& $generator -BCQualityRoot $fixtureRoot -IndexPath (Join-Path $tmp 'nested.json')
}
Remove-Item -LiteralPath (Join-Path $fixtureSkills 'al-nested-review.md') -Force
$validSuper = @'
---
kind: action-skill
id: al-code-review
version: 1
title: Review
description: Test super-skill.
inputs: [file-path]
outputs: [findings-report]
sub-skills:
- microsoft/skills/review/al-leaf-review.md
---
# Review
## Source
Source.
## Relevance
Relevance.
## Worklist
Worklist.
## Action
Action.
## Output
Output.
'@
Set-Content -LiteralPath $superPath -Value $validSuper -Encoding utf8NoBOM
$customSkills = Join-Path -Path $fixtureRoot -ChildPath 'custom/skills/review'
New-Item -ItemType Directory -Path $customSkills -Force | Out-Null
$customLeaf = $leaf.Replace('title: Leaf', 'title: Custom Leaf')
Set-Content -LiteralPath (Join-Path $customSkills 'custom-leaf-review.md') -Value $customLeaf -Encoding utf8NoBOM
$layeredPath = Join-Path $tmp 'layered.json'
& $generator -BCQualityRoot $fixtureRoot -IndexPath $layeredPath | Out-Null
$layeredIndex = Get-Content -LiteralPath $layeredPath -Raw | ConvertFrom-Json
$layeredLeaves = @($layeredIndex.skills | Where-Object id -eq 'al-leaf-review')
if ($layeredLeaves.Count -ne 2) {
throw "Expected both layered al-leaf-review implementations, found $($layeredLeaves.Count)."
}
if ((@($layeredLeaves.layer | Sort-Object) -join ',') -cne 'custom,microsoft') {
throw 'Layered al-leaf-review implementations did not preserve custom and microsoft records.'
}
$resolved = & $resolver -BCQualityRoot $fixtureRoot -IndexPath $layeredPath -SuperSkillPath (
'microsoft/skills/review/al-code-review.md'
)
if ($resolved.subSkills.Count -ne 1 -or
$resolved.subSkills[0].path -cne 'custom/skills/review/custom-leaf-review.md') {
throw 'The custom implementation did not win the layered leaf slot.'
}
$microsoftOnly = & $resolver -BCQualityRoot $fixtureRoot -IndexPath $layeredPath -SuperSkillPath (
'microsoft/skills/review/al-code-review.md'
) -EnabledLayers microsoft
if ($microsoftOnly.subSkills.Count -ne 1 -or
$microsoftOnly.subSkills[0].path -cne 'microsoft/skills/review/al-leaf-review.md') {
throw 'Disabling the custom layer did not fall back to the Microsoft implementation.'
}
$customDisabled = & $resolver -BCQualityRoot $fixtureRoot -IndexPath $layeredPath -SuperSkillPath (
'microsoft/skills/review/al-code-review.md'
) -DisabledSkills 'custom/skills/review/custom-leaf-review.md'
if ($customDisabled.subSkills.Count -ne 1 -or
$customDisabled.subSkills[0].path -cne 'microsoft/skills/review/al-leaf-review.md') {
throw 'Disabling the custom implementation did not fall back to Microsoft.'
}
Set-Content -LiteralPath (Join-Path $customSkills 'duplicate-leaf-review.md') -Value $customLeaf -Encoding utf8NoBOM
Assert-ThrowsLike -Pattern '*Duplicate action-skill IDs within a layer: custom:al-leaf-review*' -Action {
& $generator -BCQualityRoot $fixtureRoot -IndexPath (Join-Path $tmp 'duplicate-layer.json')
}
}
finally {
Remove-Item -LiteralPath $tmp -Recurse -Force -ErrorAction SilentlyContinue
}
Write-Output "Skill-index check PASSED: deterministic, schema-valid, and all $($expectedLeaves.Count) review leaves preserved in order."
Write-Output "Skill-index check PASSED: deterministic, schema-valid, layered overrides resolved, and all $($expectedLeaves.Count) review leaves preserved in order."

View file

@ -688,12 +688,16 @@ def run(root: Path) -> Report:
if domain_dir.is_dir():
validate_samples_in_domain(domain_dir, root, report)
# Third pass: R24 unique ids within kind
# Third pass: R24 unique ids within kind. Layered action-skill overrides
# may share an id, but two definitions in one layer are ambiguous.
by_kind: dict[str, dict[str, list[Path]]] = {}
for rec in skill_records:
if rec.skill_id is None:
continue
by_kind.setdefault(rec.kind, {}).setdefault(rec.skill_id, []).append(rec.path)
scope = rec.kind
if rec.kind == "action-skill":
scope = f"{rec.kind}:{rec.path.relative_to(root).parts[0]}"
by_kind.setdefault(scope, {}).setdefault(rec.skill_id, []).append(rec.path)
for kind, by_id in by_kind.items():
for sid, paths in by_id.items():
if len(paths) > 1:

View file

@ -12,7 +12,26 @@ jobs:
steps:
- name: Check out repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
- name: Collect changed paths
shell: pwsh
env:
BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }}
run: |
$paths = if (-not $env:BASE_SHA -or $env:BASE_SHA -match '^0+$') {
@(git diff-tree --no-commit-id --name-only -r $env:GITHUB_SHA)
} else {
@(git diff --name-only $env:BASE_SHA $env:GITHUB_SHA)
}
$paths | Set-Content -LiteralPath "$env:RUNNER_TEMP/changed-paths.txt" -Encoding utf8NoBOM
- name: Validate review evaluation corpus
shell: pwsh
run: ./tools/Test-ReviewFixtures.ps1 -Root . -PrepareDirectory "$env:RUNNER_TEMP/bcquality-review-fixtures"
run: |
./tools/Test-ReviewFixtures.ps1 -Root . `
-PrepareDirectory "$env:RUNNER_TEMP/bcquality-review-fixtures" `
-ChangedPathsFile "$env:RUNNER_TEMP/changed-paths.txt" `
-CoverageReportPath "$env:RUNNER_TEMP/review-coverage.json"
Get-Content -LiteralPath "$env:RUNNER_TEMP/review-coverage.json"