mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-05 22:56:55 +01:00
Add machine-readable review contracts (#182)
Generate a deterministic action-skill index from frontmatter, publish structural schemas for orchestration and findings, and validate flat review composition in CI. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: dayland <dayland@microsoft.com> Copilot-Session: 76eb42c4-2acd-4f9c-a898-f4a44f9d46f7
This commit is contained in:
dayland
GitHub
parent
852a676285
commit
b74967bc5b
8 changed files with 803 additions and 3 deletions
240
.github/scripts/Test-SkillIndex.ps1
vendored
Normal file
240
.github/scripts/Test-SkillIndex.ps1
vendored
Normal file
|
|
@ -0,0 +1,240 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates the BCQuality action-skill index generator and shared schemas.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $Root = (Resolve-Path (Join-Path -Path $PSScriptRoot -ChildPath '..' '..'))
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
||||
|
||||
function Assert-ThrowsLike {
|
||||
param(
|
||||
[scriptblock] $Action,
|
||||
[string] $Pattern
|
||||
)
|
||||
|
||||
try {
|
||||
& $Action
|
||||
}
|
||||
catch {
|
||||
if ($_.Exception.Message -like $Pattern) {
|
||||
return
|
||||
}
|
||||
throw "Expected error like '$Pattern', received: $($_.Exception.Message)"
|
||||
}
|
||||
throw "Expected error like '$Pattern', but no error was thrown."
|
||||
}
|
||||
|
||||
$generator = Join-Path $Root 'tools/Build-SkillIndex.ps1'
|
||||
$indexSchema = Join-Path $Root 'schemas/skill-index.schema.json'
|
||||
$reportSchema = Join-Path $Root 'schemas/findings-report.schema.json'
|
||||
foreach ($path in $generator, $indexSchema, $reportSchema) {
|
||||
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
|
||||
throw "Required contract file not found: $path"
|
||||
}
|
||||
}
|
||||
|
||||
$tmp = Join-Path ([IO.Path]::GetTempPath()) ("skillindex_" + [guid]::NewGuid().ToString('N'))
|
||||
New-Item -ItemType Directory -Path $tmp -Force | Out-Null
|
||||
try {
|
||||
$first = Join-Path $tmp 'first.json'
|
||||
$second = Join-Path $tmp 'second.json'
|
||||
& $generator -BCQualityRoot $Root -IndexPath $first | Out-Null
|
||||
& $generator -BCQualityRoot $Root -IndexPath $second | Out-Null
|
||||
|
||||
$normalize = {
|
||||
param([string] $Path)
|
||||
return ((Get-Content -LiteralPath $Path -Raw) -replace '"generatedAt":"[^"]*"', '"generatedAt":"<timestamp>"')
|
||||
}
|
||||
if ((& $normalize $first) -ne (& $normalize $second)) {
|
||||
throw 'Skill index is not deterministic beyond generatedAt.'
|
||||
}
|
||||
|
||||
$raw = Get-Content -LiteralPath $first -Raw
|
||||
if (-not ($raw | Test-Json -SchemaFile $indexSchema -ErrorAction Stop)) {
|
||||
throw 'Generated skill index does not satisfy schemas/skill-index.schema.json.'
|
||||
}
|
||||
|
||||
$index = $raw | ConvertFrom-Json
|
||||
$skills = @($index.skills)
|
||||
if ($index.skillCount -ne $skills.Count) {
|
||||
throw "skillCount is $($index.skillCount), but the index contains $($skills.Count) records."
|
||||
}
|
||||
|
||||
$paths = @($skills.path)
|
||||
$duplicates = @($paths | Group-Object | Where-Object Count -gt 1)
|
||||
if ($duplicates.Count) {
|
||||
throw "Duplicate skill paths: $($duplicates.Name -join ', ')"
|
||||
}
|
||||
foreach ($path in $paths) {
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $Root $path) -PathType Leaf)) {
|
||||
throw "Indexed skill does not exist: $path"
|
||||
}
|
||||
}
|
||||
|
||||
$expectedLeaves = @(
|
||||
'microsoft/skills/review/al-performance-review.md',
|
||||
'microsoft/skills/review/al-security-review.md',
|
||||
'microsoft/skills/review/al-privacy-review.md',
|
||||
'microsoft/skills/review/al-upgrade-review.md',
|
||||
'microsoft/skills/review/al-style-review.md',
|
||||
'microsoft/skills/review/al-ui-review.md',
|
||||
'microsoft/skills/review/al-error-handling-review.md',
|
||||
'microsoft/skills/review/al-events-review.md',
|
||||
'microsoft/skills/review/al-interfaces-review.md',
|
||||
'microsoft/skills/review/al-breaking-changes-review.md',
|
||||
'microsoft/skills/review/al-web-services-review.md',
|
||||
'microsoft/skills/review/al-testing-review.md',
|
||||
'microsoft/skills/review/al-data-modeling-review.md',
|
||||
'microsoft/skills/review/al-query-review.md',
|
||||
'microsoft/skills/review/al-appsource-review.md',
|
||||
'microsoft/skills/review/al-telemetry-review.md'
|
||||
)
|
||||
$review = @($skills | Where-Object id -eq 'al-code-review')
|
||||
if ($review.Count -ne 1) {
|
||||
throw "Expected exactly one al-code-review record, found $($review.Count)."
|
||||
}
|
||||
if ((@($review[0].subSkills) -join "`n") -cne ($expectedLeaves -join "`n")) {
|
||||
throw 'al-code-review subSkills did not preserve the declared 16-leaf order.'
|
||||
}
|
||||
foreach ($leafPath in $expectedLeaves) {
|
||||
$leaf = @($skills | Where-Object path -ceq $leafPath)
|
||||
if ($leaf.Count -ne 1 -or @($leaf[0].subSkills).Count -ne 0) {
|
||||
throw "Expected '$leafPath' to resolve to exactly one leaf action skill."
|
||||
}
|
||||
}
|
||||
|
||||
$minimalReport = @{
|
||||
skill = @{ id = 'al-style-review'; version = 1 }
|
||||
outcome = 'completed'
|
||||
summary = @{
|
||||
counts = @{ blocker = 0; major = 0; minor = 0; info = 0 }
|
||||
coverage = @{ 'worklist-size' = 0; 'items-evaluated' = 0 }
|
||||
}
|
||||
findings = @()
|
||||
suppressed = @()
|
||||
} | ConvertTo-Json -Depth 8
|
||||
if (-not ($minimalReport | Test-Json -SchemaFile $reportSchema -ErrorAction Stop)) {
|
||||
throw 'Minimal findings report does not satisfy schemas/findings-report.schema.json.'
|
||||
}
|
||||
|
||||
$reviewSkillText = Get-Content -LiteralPath (
|
||||
Join-Path -Path $Root -ChildPath 'microsoft/skills/review/al-code-review.md'
|
||||
) -Raw
|
||||
$reportExamples = [regex]::Matches($reviewSkillText, '(?s)```json\s*(\{.*?\})\s*```')
|
||||
if ($reportExamples.Count -ne 2) {
|
||||
throw "Expected two al-code-review JSON examples, found $($reportExamples.Count)."
|
||||
}
|
||||
foreach ($example in $reportExamples) {
|
||||
if (-not ($example.Groups[1].Value | Test-Json -SchemaFile $reportSchema -ErrorAction Stop)) {
|
||||
throw 'An al-code-review output example does not satisfy schemas/findings-report.schema.json.'
|
||||
}
|
||||
}
|
||||
|
||||
$fixtureRoot = Join-Path -Path $tmp -ChildPath 'fixture'
|
||||
$fixtureSkills = Join-Path -Path $fixtureRoot -ChildPath 'microsoft/skills/review'
|
||||
New-Item -ItemType Directory -Path $fixtureSkills -Force | Out-Null
|
||||
$leaf = @'
|
||||
---
|
||||
kind: action-skill
|
||||
id: al-leaf-review
|
||||
version: 1
|
||||
title: Leaf
|
||||
description: Test leaf.
|
||||
inputs: [file-path]
|
||||
outputs: [findings-report]
|
||||
---
|
||||
|
||||
# Leaf
|
||||
|
||||
## Source
|
||||
Source.
|
||||
## Relevance
|
||||
Relevance.
|
||||
## Worklist
|
||||
Worklist.
|
||||
## Action
|
||||
Action.
|
||||
## Output
|
||||
Output.
|
||||
'@
|
||||
Set-Content -LiteralPath (Join-Path $fixtureSkills 'al-leaf-review.md') -Value $leaf -Encoding utf8NoBOM
|
||||
|
||||
$duplicateSuper = @'
|
||||
---
|
||||
kind: action-skill
|
||||
id: al-code-review
|
||||
version: 1
|
||||
title: Review
|
||||
description: Test super-skill.
|
||||
inputs: [file-path]
|
||||
outputs: [findings-report]
|
||||
sub-skills:
|
||||
- microsoft/skills/review/al-leaf-review.md
|
||||
- microsoft/skills/review/al-leaf-review.md
|
||||
---
|
||||
|
||||
# Review
|
||||
|
||||
## Source
|
||||
Source.
|
||||
## Relevance
|
||||
Relevance.
|
||||
## Worklist
|
||||
Worklist.
|
||||
## Action
|
||||
Action.
|
||||
## Output
|
||||
Output.
|
||||
'@
|
||||
$superPath = Join-Path $fixtureSkills 'al-code-review.md'
|
||||
Set-Content -LiteralPath $superPath -Value $duplicateSuper -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*duplicate sub-skill*' -Action {
|
||||
& $generator -BCQualityRoot $fixtureRoot -IndexPath (Join-Path $tmp 'invalid.json')
|
||||
}
|
||||
|
||||
$nestedLeaf = $leaf.Replace('id: al-leaf-review', 'id: al-nested-review').Replace(
|
||||
'outputs: [findings-report]',
|
||||
"outputs: [findings-report]`nsub-skills:`n - microsoft/skills/review/al-leaf-review.md"
|
||||
)
|
||||
Set-Content -LiteralPath (Join-Path $fixtureSkills 'al-nested-review.md') -Value $nestedLeaf -Encoding utf8NoBOM
|
||||
$nestedSuper = @'
|
||||
---
|
||||
kind: action-skill
|
||||
id: al-code-review
|
||||
version: 1
|
||||
title: Review
|
||||
description: Test super-skill.
|
||||
inputs: [file-path]
|
||||
outputs: [findings-report]
|
||||
sub-skills:
|
||||
- microsoft/skills/review/al-nested-review.md
|
||||
---
|
||||
|
||||
# Review
|
||||
|
||||
## Source
|
||||
Source.
|
||||
## Relevance
|
||||
Relevance.
|
||||
## Worklist
|
||||
Worklist.
|
||||
## Action
|
||||
Action.
|
||||
## Output
|
||||
Output.
|
||||
'@
|
||||
Set-Content -LiteralPath $superPath -Value $nestedSuper -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*Nested super-skills are not supported*' -Action {
|
||||
& $generator -BCQualityRoot $fixtureRoot -IndexPath (Join-Path $tmp 'nested.json')
|
||||
}
|
||||
}
|
||||
finally {
|
||||
Remove-Item -LiteralPath $tmp -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
Write-Output 'Skill-index check PASSED: deterministic, schema-valid, and all 16 review leaves preserved in order.'
|
||||
41
.github/scripts/validate_frontmatter.py
vendored
41
.github/scripts/validate_frontmatter.py
vendored
|
|
@ -381,6 +381,20 @@ def validate_action_skill(path: Path, parsed: Parsed, report: Report) -> None:
|
|||
bad = [x for x in ss if not x.endswith(".md")]
|
||||
if bad:
|
||||
report.error(path, "R20", f"sub-skills entries must end in '.md': {bad}", 1)
|
||||
non_canonical = [
|
||||
x for x in ss
|
||||
if "\\" in x or x.startswith("/") or ".." in Path(x).parts or x.startswith("./")
|
||||
]
|
||||
if non_canonical:
|
||||
report.error(
|
||||
path,
|
||||
"R20",
|
||||
f"sub-skills entries must be canonical repo-relative paths: {non_canonical}",
|
||||
1,
|
||||
)
|
||||
duplicates = sorted({x for x in ss if ss.count(x) > 1})
|
||||
if duplicates:
|
||||
report.error(path, "R20", f"sub-skills contains duplicate paths: {duplicates}", 1)
|
||||
|
||||
# R21 five required sections, in order, each exactly once
|
||||
heads = [h for h, _ in headings_in_order(parsed.body)]
|
||||
|
|
@ -565,7 +579,13 @@ class SkillRecord:
|
|||
skill_id: str | None
|
||||
|
||||
|
||||
def validate_sub_skills_registry(path: Path, fm: dict[str, Any], root: Path, report: Report) -> None:
|
||||
def validate_sub_skills_registry(
|
||||
path: Path,
|
||||
fm: dict[str, Any],
|
||||
root: Path,
|
||||
action_skills_by_path: dict[str, dict[str, Any]],
|
||||
report: Report,
|
||||
) -> None:
|
||||
"""R26: a super-skill's declared `sub-skills` must exactly match the
|
||||
`al-*-review.md` leaf files present in the same directory (set equality,
|
||||
ordering-agnostic). This keeps the registered leaf list the single source
|
||||
|
|
@ -598,6 +618,17 @@ def validate_sub_skills_registry(path: Path, fm: dict[str, Any], root: Path, rep
|
|||
f"sub-skills entry is not a sibling 'al-*-review.md' leaf: {entry}", 1,
|
||||
)
|
||||
|
||||
for entry in ss:
|
||||
leaf = action_skills_by_path.get(entry)
|
||||
if leaf is None:
|
||||
if (root / entry).exists():
|
||||
report.error(path, "R26", f"sub-skills entry is not an action skill: {entry}", 1)
|
||||
continue
|
||||
if is_non_empty_list_of_str(leaf.get("sub-skills")):
|
||||
report.error(path, "R26", f"nested super-skill is not permitted in v1 composition: {entry}", 1)
|
||||
if leaf.get("outputs") != ["findings-report"]:
|
||||
report.error(path, "R26", f"sub-skill must produce findings-report: {entry}", 1)
|
||||
|
||||
# Sibling leaves on disk that were never registered ('forgot to wire it up').
|
||||
for leaf in sorted(leaves - declared):
|
||||
report.error(path, "R26", f"leaf not registered in sub-skills: {leaf}", 1)
|
||||
|
|
@ -670,9 +701,13 @@ def run(root: Path) -> Report:
|
|||
others = [q.relative_to(root).as_posix() for q in paths if q != p]
|
||||
report.error(p, "R24", f"skill id '{sid}' ({kind}) is not unique; also defined in: {others}")
|
||||
|
||||
# Fourth pass: R26 sub-skills registry matches leaf files on disk
|
||||
# Fourth pass: R26 sub-skills registry matches compatible leaf files on disk
|
||||
action_skills_by_path = {
|
||||
path.relative_to(root).as_posix(): fm
|
||||
for path, fm in action_skill_fms
|
||||
}
|
||||
for path, fm in action_skill_fms:
|
||||
validate_sub_skills_registry(path, fm, root, report)
|
||||
validate_sub_skills_registry(path, fm, root, action_skills_by_path, report)
|
||||
|
||||
return report
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue